diff --git a/lexer/BUILD b/lexer/BUILD new file mode 100644 index 0000000000000..b6f0ae7339bf8 --- /dev/null +++ b/lexer/BUILD @@ -0,0 +1,75 @@ +# Part of the Carbon Language project, under the Apache License v2.0 with LLVM +# Exceptions. See /LICENSE for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test") +load("//bazel/fuzzing:rules.bzl", "cc_fuzz_test") + +package(default_visibility = ["//visibility:public"]) + +cc_library( + name = "token_kind", + srcs = ["token_kind.cpp"], + hdrs = ["token_kind.h"], + textual_hdrs = ["token_registry.def"], + deps = ["@llvm-project//llvm:Support"], +) + +cc_test( + name = "token_kind_test", + srcs = ["token_kind_test.cpp"], + deps = [ + ":token_kind", + "@llvm-project//llvm:Support", + "@llvm-project//llvm:gtest", + "@llvm-project//llvm:gtest_main", + ], +) + +cc_library( + name = "tokenized_buffer", + srcs = ["tokenized_buffer.cpp"], + hdrs = ["tokenized_buffer.h"], + deps = [ + ":token_kind", + "//diagnostics:diagnostic_emitter", + "//source:source_buffer", + "@llvm-project//llvm:Support", + ], +) + +cc_library( + name = "tokenized_buffer_test_helpers", + testonly = 1, + hdrs = ["tokenized_buffer_test_helpers.h"], + deps = [ + ":tokenized_buffer", + "@llvm-project//llvm:Support", + "@llvm-project//llvm:gmock", + ], +) + +cc_test( + name = "tokenized_buffer_test", + srcs = ["tokenized_buffer_test.cpp"], + deps = [ + ":tokenized_buffer", + ":tokenized_buffer_test_helpers", + "//diagnostics:diagnostic_emitter", + "@llvm-project//llvm:Support", + "@llvm-project//llvm:gmock", + "@llvm-project//llvm:gtest", + "@llvm-project//llvm:gtest_main", + ], +) + +cc_fuzz_test( + name = "tokenized_buffer_fuzzer", + srcs = ["tokenized_buffer_fuzzer.cpp"], + corpus = glob(["fuzzer_corpus/*"]), + deps = [ + ":tokenized_buffer", + "//diagnostics:diagnostic_emitter", + "@llvm-project//llvm:Support", + ], +) diff --git a/lexer/fuzzer_corpus/00b9598434d1cb0aca2446e5d8059f955898661b b/lexer/fuzzer_corpus/00b9598434d1cb0aca2446e5d8059f955898661b new file mode 100644 index 0000000000000..739aefd291770 Binary files /dev/null and b/lexer/fuzzer_corpus/00b9598434d1cb0aca2446e5d8059f955898661b differ diff --git a/lexer/fuzzer_corpus/01096186d4e1347ff16a01491d6ef629f5f5fd42 b/lexer/fuzzer_corpus/01096186d4e1347ff16a01491d6ef629f5f5fd42 new file mode 100644 index 0000000000000..140dc8308fcf3 Binary files /dev/null and b/lexer/fuzzer_corpus/01096186d4e1347ff16a01491d6ef629f5f5fd42 differ diff --git a/lexer/fuzzer_corpus/011f63c979b2de6c44a0f974437ff176fb309cf7 b/lexer/fuzzer_corpus/011f63c979b2de6c44a0f974437ff176fb309cf7 new file mode 100644 index 0000000000000..e72781519e904 Binary files /dev/null and b/lexer/fuzzer_corpus/011f63c979b2de6c44a0f974437ff176fb309cf7 differ diff --git a/lexer/fuzzer_corpus/013a89a4caf353b43c203a2cbd270857c46863c8 b/lexer/fuzzer_corpus/013a89a4caf353b43c203a2cbd270857c46863c8 new file mode 100644 index 0000000000000..b063c01c1d583 Binary files /dev/null and b/lexer/fuzzer_corpus/013a89a4caf353b43c203a2cbd270857c46863c8 differ diff --git a/lexer/fuzzer_corpus/02ddec8b4855127d5616e521bc826659cb04a796 b/lexer/fuzzer_corpus/02ddec8b4855127d5616e521bc826659cb04a796 new file mode 100644 index 0000000000000..61e4c3d38f8c6 Binary files /dev/null and b/lexer/fuzzer_corpus/02ddec8b4855127d5616e521bc826659cb04a796 differ diff --git a/lexer/fuzzer_corpus/0327df1d12e0abf8a355b7ed09e0228dc396d787 b/lexer/fuzzer_corpus/0327df1d12e0abf8a355b7ed09e0228dc396d787 new file mode 100644 index 0000000000000..547045ce09d0b Binary files /dev/null and b/lexer/fuzzer_corpus/0327df1d12e0abf8a355b7ed09e0228dc396d787 differ diff --git a/lexer/fuzzer_corpus/037dc35353d675c9f4d2f03c9db8186422050b2d b/lexer/fuzzer_corpus/037dc35353d675c9f4d2f03c9db8186422050b2d new file mode 100644 index 0000000000000..081159b73ff40 Binary files /dev/null and b/lexer/fuzzer_corpus/037dc35353d675c9f4d2f03c9db8186422050b2d differ diff --git a/lexer/fuzzer_corpus/04030ac2f55537459994c38435a6b7d00ae264bf b/lexer/fuzzer_corpus/04030ac2f55537459994c38435a6b7d00ae264bf new file mode 100644 index 0000000000000..ec2d47e79be53 Binary files /dev/null and b/lexer/fuzzer_corpus/04030ac2f55537459994c38435a6b7d00ae264bf differ diff --git a/lexer/fuzzer_corpus/0543ca422553bae7a2e8b46a81c8fee7eb12c9a7 b/lexer/fuzzer_corpus/0543ca422553bae7a2e8b46a81c8fee7eb12c9a7 new file mode 100644 index 0000000000000..2d2798ae43929 Binary files /dev/null and b/lexer/fuzzer_corpus/0543ca422553bae7a2e8b46a81c8fee7eb12c9a7 differ diff --git a/lexer/fuzzer_corpus/0555830a8ac12955924bf235d97792583fb8d6da b/lexer/fuzzer_corpus/0555830a8ac12955924bf235d97792583fb8d6da new file mode 100644 index 0000000000000..5afc801db25ee Binary files /dev/null and b/lexer/fuzzer_corpus/0555830a8ac12955924bf235d97792583fb8d6da differ diff --git a/lexer/fuzzer_corpus/05666d20f461b2d4a93030744abd619ce0f25fe7 b/lexer/fuzzer_corpus/05666d20f461b2d4a93030744abd619ce0f25fe7 new file mode 100644 index 0000000000000..d149c11d7f4df Binary files /dev/null and b/lexer/fuzzer_corpus/05666d20f461b2d4a93030744abd619ce0f25fe7 differ diff --git a/lexer/fuzzer_corpus/058baf2eca3ac04d3382c94dc213af644b547796 b/lexer/fuzzer_corpus/058baf2eca3ac04d3382c94dc213af644b547796 new file mode 100644 index 0000000000000..8ac251b2daf86 Binary files /dev/null and b/lexer/fuzzer_corpus/058baf2eca3ac04d3382c94dc213af644b547796 differ diff --git a/lexer/fuzzer_corpus/05910a7c99851cbb657cfbf8850e1e67e98408b1 b/lexer/fuzzer_corpus/05910a7c99851cbb657cfbf8850e1e67e98408b1 new file mode 100644 index 0000000000000..a03527e2bf7c1 Binary files /dev/null and b/lexer/fuzzer_corpus/05910a7c99851cbb657cfbf8850e1e67e98408b1 differ diff --git a/lexer/fuzzer_corpus/062fe100147aaaf3a5f3f0a67c64991480597cfc b/lexer/fuzzer_corpus/062fe100147aaaf3a5f3f0a67c64991480597cfc new file mode 100644 index 0000000000000..b564a4ceeb79b Binary files /dev/null and b/lexer/fuzzer_corpus/062fe100147aaaf3a5f3f0a67c64991480597cfc differ diff --git a/lexer/fuzzer_corpus/0685140d284f7c98815eb844fe6445ac2e4d25da b/lexer/fuzzer_corpus/0685140d284f7c98815eb844fe6445ac2e4d25da new file mode 100644 index 0000000000000..45164dd9bb4b8 Binary files /dev/null and b/lexer/fuzzer_corpus/0685140d284f7c98815eb844fe6445ac2e4d25da differ diff --git a/lexer/fuzzer_corpus/06b6c1a09f2ec62f93b7a3f1b03a2188700c8117 b/lexer/fuzzer_corpus/06b6c1a09f2ec62f93b7a3f1b03a2188700c8117 new file mode 100644 index 0000000000000..d9e1a810ee408 Binary files /dev/null and b/lexer/fuzzer_corpus/06b6c1a09f2ec62f93b7a3f1b03a2188700c8117 differ diff --git a/lexer/fuzzer_corpus/075e66efafa8882c10f46de45be654ad1f930c1f b/lexer/fuzzer_corpus/075e66efafa8882c10f46de45be654ad1f930c1f new file mode 100644 index 0000000000000..17fa5b42c06e9 Binary files /dev/null and b/lexer/fuzzer_corpus/075e66efafa8882c10f46de45be654ad1f930c1f differ diff --git a/lexer/fuzzer_corpus/07f95640a7ba78bebf24bea549bd129d81d03f14 b/lexer/fuzzer_corpus/07f95640a7ba78bebf24bea549bd129d81d03f14 new file mode 100644 index 0000000000000..4a1f797719193 Binary files /dev/null and b/lexer/fuzzer_corpus/07f95640a7ba78bebf24bea549bd129d81d03f14 differ diff --git a/lexer/fuzzer_corpus/085e5e94c0193bc313159a1aac208bc7232fa24b b/lexer/fuzzer_corpus/085e5e94c0193bc313159a1aac208bc7232fa24b new file mode 100644 index 0000000000000..87ecc5be25795 Binary files /dev/null and b/lexer/fuzzer_corpus/085e5e94c0193bc313159a1aac208bc7232fa24b differ diff --git a/lexer/fuzzer_corpus/08988f30662dbfc7a95bee73cef111dafbcd64af b/lexer/fuzzer_corpus/08988f30662dbfc7a95bee73cef111dafbcd64af new file mode 100644 index 0000000000000..cfbf523014230 Binary files /dev/null and b/lexer/fuzzer_corpus/08988f30662dbfc7a95bee73cef111dafbcd64af differ diff --git a/lexer/fuzzer_corpus/08a189c1daec7835d24c69454657ea460add9e57 b/lexer/fuzzer_corpus/08a189c1daec7835d24c69454657ea460add9e57 new file mode 100644 index 0000000000000..f21cc098df28a Binary files /dev/null and b/lexer/fuzzer_corpus/08a189c1daec7835d24c69454657ea460add9e57 differ diff --git a/lexer/fuzzer_corpus/0928b559c298c803b98e59564c8b6b5443ec1697 b/lexer/fuzzer_corpus/0928b559c298c803b98e59564c8b6b5443ec1697 new file mode 100644 index 0000000000000..8956115b3443d Binary files /dev/null and b/lexer/fuzzer_corpus/0928b559c298c803b98e59564c8b6b5443ec1697 differ diff --git a/lexer/fuzzer_corpus/09de090fce19575b878349e5b08e44774cf901b3 b/lexer/fuzzer_corpus/09de090fce19575b878349e5b08e44774cf901b3 new file mode 100644 index 0000000000000..964469c2c6ffe Binary files /dev/null and b/lexer/fuzzer_corpus/09de090fce19575b878349e5b08e44774cf901b3 differ diff --git a/lexer/fuzzer_corpus/0a44775851faa48a0293c03c5823526a05432a41 b/lexer/fuzzer_corpus/0a44775851faa48a0293c03c5823526a05432a41 new file mode 100644 index 0000000000000..b290effed1196 Binary files /dev/null and b/lexer/fuzzer_corpus/0a44775851faa48a0293c03c5823526a05432a41 differ diff --git a/lexer/fuzzer_corpus/0a73a50a6c6470643bc6ac4731df70e168a1d72a b/lexer/fuzzer_corpus/0a73a50a6c6470643bc6ac4731df70e168a1d72a new file mode 100644 index 0000000000000..d8383ed8deabb Binary files /dev/null and b/lexer/fuzzer_corpus/0a73a50a6c6470643bc6ac4731df70e168a1d72a differ diff --git a/lexer/fuzzer_corpus/0b02094a3c9673c0ca1a0f0ebb8a2a1cd334bd5d b/lexer/fuzzer_corpus/0b02094a3c9673c0ca1a0f0ebb8a2a1cd334bd5d new file mode 100644 index 0000000000000..619517a64ab85 Binary files /dev/null and b/lexer/fuzzer_corpus/0b02094a3c9673c0ca1a0f0ebb8a2a1cd334bd5d differ diff --git a/lexer/fuzzer_corpus/0b50b447ee13e7ea31515670dee9cfe0159182a3 b/lexer/fuzzer_corpus/0b50b447ee13e7ea31515670dee9cfe0159182a3 new file mode 100644 index 0000000000000..bce6612919482 Binary files /dev/null and b/lexer/fuzzer_corpus/0b50b447ee13e7ea31515670dee9cfe0159182a3 differ diff --git a/lexer/fuzzer_corpus/0b656451855e16143574319c2c02c7278916e53d b/lexer/fuzzer_corpus/0b656451855e16143574319c2c02c7278916e53d new file mode 100644 index 0000000000000..194b924993938 Binary files /dev/null and b/lexer/fuzzer_corpus/0b656451855e16143574319c2c02c7278916e53d differ diff --git a/lexer/fuzzer_corpus/0b7b607b163793a78dd80383f3dfc1283ae77822 b/lexer/fuzzer_corpus/0b7b607b163793a78dd80383f3dfc1283ae77822 new file mode 100644 index 0000000000000..8e2ee121ff8e2 Binary files /dev/null and b/lexer/fuzzer_corpus/0b7b607b163793a78dd80383f3dfc1283ae77822 differ diff --git a/lexer/fuzzer_corpus/0b832ffe11d2337ca3efd79aa32140076d90dc67 b/lexer/fuzzer_corpus/0b832ffe11d2337ca3efd79aa32140076d90dc67 new file mode 100644 index 0000000000000..4d0d40121c66b Binary files /dev/null and b/lexer/fuzzer_corpus/0b832ffe11d2337ca3efd79aa32140076d90dc67 differ diff --git a/lexer/fuzzer_corpus/0b9b08eca9652ba3a3e389f051e0d19e5e46fd3c b/lexer/fuzzer_corpus/0b9b08eca9652ba3a3e389f051e0d19e5e46fd3c new file mode 100644 index 0000000000000..8d1afe6a22a69 Binary files /dev/null and b/lexer/fuzzer_corpus/0b9b08eca9652ba3a3e389f051e0d19e5e46fd3c differ diff --git a/lexer/fuzzer_corpus/0beed1e9ae76d37f24c30c22c0b254f8efb30744 b/lexer/fuzzer_corpus/0beed1e9ae76d37f24c30c22c0b254f8efb30744 new file mode 100644 index 0000000000000..e746b634eba30 Binary files /dev/null and b/lexer/fuzzer_corpus/0beed1e9ae76d37f24c30c22c0b254f8efb30744 differ diff --git a/lexer/fuzzer_corpus/0c38e5795a262ad27f8130c77a4bc34ffe7eaa82 b/lexer/fuzzer_corpus/0c38e5795a262ad27f8130c77a4bc34ffe7eaa82 new file mode 100644 index 0000000000000..3b10a15ee88d5 Binary files /dev/null and b/lexer/fuzzer_corpus/0c38e5795a262ad27f8130c77a4bc34ffe7eaa82 differ diff --git a/lexer/fuzzer_corpus/0c5a294e646b32fc10624947911d167a7b550cf4 b/lexer/fuzzer_corpus/0c5a294e646b32fc10624947911d167a7b550cf4 new file mode 100644 index 0000000000000..c8ddef3b0729d Binary files /dev/null and b/lexer/fuzzer_corpus/0c5a294e646b32fc10624947911d167a7b550cf4 differ diff --git a/lexer/fuzzer_corpus/0ccd028acb41f2d1902f9ce943084cf855977282 b/lexer/fuzzer_corpus/0ccd028acb41f2d1902f9ce943084cf855977282 new file mode 100644 index 0000000000000..96f1d97633585 Binary files /dev/null and b/lexer/fuzzer_corpus/0ccd028acb41f2d1902f9ce943084cf855977282 differ diff --git a/lexer/fuzzer_corpus/0ce0d735a0d8e0c637030b8bed3bbcc3a3864866 b/lexer/fuzzer_corpus/0ce0d735a0d8e0c637030b8bed3bbcc3a3864866 new file mode 100644 index 0000000000000..1a108e3eeb9c9 Binary files /dev/null and b/lexer/fuzzer_corpus/0ce0d735a0d8e0c637030b8bed3bbcc3a3864866 differ diff --git a/lexer/fuzzer_corpus/0d16aa665e382f2118a4788ac09401f0de985a00 b/lexer/fuzzer_corpus/0d16aa665e382f2118a4788ac09401f0de985a00 new file mode 100644 index 0000000000000..53b2d93af220c Binary files /dev/null and b/lexer/fuzzer_corpus/0d16aa665e382f2118a4788ac09401f0de985a00 differ diff --git a/lexer/fuzzer_corpus/0d968cc12e55af8d4306e667c5c817b16b8e1b79 b/lexer/fuzzer_corpus/0d968cc12e55af8d4306e667c5c817b16b8e1b79 new file mode 100644 index 0000000000000..bd84d2ab94320 Binary files /dev/null and b/lexer/fuzzer_corpus/0d968cc12e55af8d4306e667c5c817b16b8e1b79 differ diff --git a/lexer/fuzzer_corpus/0dfaf14bb1c6bbaf15c4840df4bbcba70727fb2f b/lexer/fuzzer_corpus/0dfaf14bb1c6bbaf15c4840df4bbcba70727fb2f new file mode 100644 index 0000000000000..18d843f5a36e9 Binary files /dev/null and b/lexer/fuzzer_corpus/0dfaf14bb1c6bbaf15c4840df4bbcba70727fb2f differ diff --git a/lexer/fuzzer_corpus/0e3efc3ff89fc9956def38fa002510e8f408e4e4 b/lexer/fuzzer_corpus/0e3efc3ff89fc9956def38fa002510e8f408e4e4 new file mode 100644 index 0000000000000..1d352ab577b31 Binary files /dev/null and b/lexer/fuzzer_corpus/0e3efc3ff89fc9956def38fa002510e8f408e4e4 differ diff --git a/lexer/fuzzer_corpus/0ea6b54fd89072271de9d6db7291db91e412ba0c b/lexer/fuzzer_corpus/0ea6b54fd89072271de9d6db7291db91e412ba0c new file mode 100644 index 0000000000000..63ac2c468351a Binary files /dev/null and b/lexer/fuzzer_corpus/0ea6b54fd89072271de9d6db7291db91e412ba0c differ diff --git a/lexer/fuzzer_corpus/0eb01abf75b501eab2e9f8cf7c0a46d150e8035e b/lexer/fuzzer_corpus/0eb01abf75b501eab2e9f8cf7c0a46d150e8035e new file mode 100644 index 0000000000000..7a93f9f1df6ce Binary files /dev/null and b/lexer/fuzzer_corpus/0eb01abf75b501eab2e9f8cf7c0a46d150e8035e differ diff --git a/lexer/fuzzer_corpus/0ec0f5fdb11fb62a0552c7e13a49c164b757b049 b/lexer/fuzzer_corpus/0ec0f5fdb11fb62a0552c7e13a49c164b757b049 new file mode 100644 index 0000000000000..b227b36283b06 Binary files /dev/null and b/lexer/fuzzer_corpus/0ec0f5fdb11fb62a0552c7e13a49c164b757b049 differ diff --git a/lexer/fuzzer_corpus/0f1ba78b09883fc942f3108c0d0238e5bbba7261 b/lexer/fuzzer_corpus/0f1ba78b09883fc942f3108c0d0238e5bbba7261 new file mode 100644 index 0000000000000..106feefadd171 Binary files /dev/null and b/lexer/fuzzer_corpus/0f1ba78b09883fc942f3108c0d0238e5bbba7261 differ diff --git a/lexer/fuzzer_corpus/0f88a6fcd8459f99ca2009cc17572e4ee6c57949 b/lexer/fuzzer_corpus/0f88a6fcd8459f99ca2009cc17572e4ee6c57949 new file mode 100644 index 0000000000000..9a8e6f84b6d80 Binary files /dev/null and b/lexer/fuzzer_corpus/0f88a6fcd8459f99ca2009cc17572e4ee6c57949 differ diff --git a/lexer/fuzzer_corpus/0fa6870ba61bffbcf4331293a0c36abf2a53745d b/lexer/fuzzer_corpus/0fa6870ba61bffbcf4331293a0c36abf2a53745d new file mode 100644 index 0000000000000..8785f1ded78e9 Binary files /dev/null and b/lexer/fuzzer_corpus/0fa6870ba61bffbcf4331293a0c36abf2a53745d differ diff --git a/lexer/fuzzer_corpus/0fb249d887ffa905f90fe43c97cb0dea4277f682 b/lexer/fuzzer_corpus/0fb249d887ffa905f90fe43c97cb0dea4277f682 new file mode 100644 index 0000000000000..adab4b9e8a47c Binary files /dev/null and b/lexer/fuzzer_corpus/0fb249d887ffa905f90fe43c97cb0dea4277f682 differ diff --git a/lexer/fuzzer_corpus/0ff985bb50cd92e0904ae0c3bf562a665a9761c8 b/lexer/fuzzer_corpus/0ff985bb50cd92e0904ae0c3bf562a665a9761c8 new file mode 100644 index 0000000000000..84b53f8f261c7 Binary files /dev/null and b/lexer/fuzzer_corpus/0ff985bb50cd92e0904ae0c3bf562a665a9761c8 differ diff --git a/lexer/fuzzer_corpus/107c59baea92d5aa0b7daf7acc49d20b6a25d180 b/lexer/fuzzer_corpus/107c59baea92d5aa0b7daf7acc49d20b6a25d180 new file mode 100644 index 0000000000000..96960eb76c869 Binary files /dev/null and b/lexer/fuzzer_corpus/107c59baea92d5aa0b7daf7acc49d20b6a25d180 differ diff --git a/lexer/fuzzer_corpus/108fb5fc2033bcad402775791f7d3a83a434d402 b/lexer/fuzzer_corpus/108fb5fc2033bcad402775791f7d3a83a434d402 new file mode 100644 index 0000000000000..9847fed625c84 Binary files /dev/null and b/lexer/fuzzer_corpus/108fb5fc2033bcad402775791f7d3a83a434d402 differ diff --git a/lexer/fuzzer_corpus/10d4b9dc18446170ba10afb28ca65744f9010ed1 b/lexer/fuzzer_corpus/10d4b9dc18446170ba10afb28ca65744f9010ed1 new file mode 100644 index 0000000000000..f1beeab069bee Binary files /dev/null and b/lexer/fuzzer_corpus/10d4b9dc18446170ba10afb28ca65744f9010ed1 differ diff --git a/lexer/fuzzer_corpus/11c0e4ea13d10a785b7298b416b5d1ec70cb9097 b/lexer/fuzzer_corpus/11c0e4ea13d10a785b7298b416b5d1ec70cb9097 new file mode 100644 index 0000000000000..ab3b82d9d0144 Binary files /dev/null and b/lexer/fuzzer_corpus/11c0e4ea13d10a785b7298b416b5d1ec70cb9097 differ diff --git a/lexer/fuzzer_corpus/11d6f314e0ee9b6b319c3e4961ad4b2c041e01f1 b/lexer/fuzzer_corpus/11d6f314e0ee9b6b319c3e4961ad4b2c041e01f1 new file mode 100644 index 0000000000000..51968981a42d0 Binary files /dev/null and b/lexer/fuzzer_corpus/11d6f314e0ee9b6b319c3e4961ad4b2c041e01f1 differ diff --git a/lexer/fuzzer_corpus/12033d109a3ab4a4068fb4f5283aa8ef2b4b2939 b/lexer/fuzzer_corpus/12033d109a3ab4a4068fb4f5283aa8ef2b4b2939 new file mode 100644 index 0000000000000..2197349452cca Binary files /dev/null and b/lexer/fuzzer_corpus/12033d109a3ab4a4068fb4f5283aa8ef2b4b2939 differ diff --git a/lexer/fuzzer_corpus/1388ff1c64cf1523471677300bcc78f2cb057178 b/lexer/fuzzer_corpus/1388ff1c64cf1523471677300bcc78f2cb057178 new file mode 100644 index 0000000000000..7446957983040 Binary files /dev/null and b/lexer/fuzzer_corpus/1388ff1c64cf1523471677300bcc78f2cb057178 differ diff --git a/lexer/fuzzer_corpus/139cf5f51919c2f88794b3d6afecd894d3795bc4 b/lexer/fuzzer_corpus/139cf5f51919c2f88794b3d6afecd894d3795bc4 new file mode 100644 index 0000000000000..be8e7335b1e48 Binary files /dev/null and b/lexer/fuzzer_corpus/139cf5f51919c2f88794b3d6afecd894d3795bc4 differ diff --git a/lexer/fuzzer_corpus/13a567de304d3293826a5915b124c898d905c533 b/lexer/fuzzer_corpus/13a567de304d3293826a5915b124c898d905c533 new file mode 100644 index 0000000000000..c091d1109cc42 Binary files /dev/null and b/lexer/fuzzer_corpus/13a567de304d3293826a5915b124c898d905c533 differ diff --git a/lexer/fuzzer_corpus/13a7d12471b67d795eeb620fd9bcd23e611716b4 b/lexer/fuzzer_corpus/13a7d12471b67d795eeb620fd9bcd23e611716b4 new file mode 100644 index 0000000000000..483d91ffcd08f Binary files /dev/null and b/lexer/fuzzer_corpus/13a7d12471b67d795eeb620fd9bcd23e611716b4 differ diff --git a/lexer/fuzzer_corpus/143df8b19029156bef78ed64f99f6a29847fd707 b/lexer/fuzzer_corpus/143df8b19029156bef78ed64f99f6a29847fd707 new file mode 100644 index 0000000000000..d9fce1442b94d Binary files /dev/null and b/lexer/fuzzer_corpus/143df8b19029156bef78ed64f99f6a29847fd707 differ diff --git a/lexer/fuzzer_corpus/1489f923c4dca729178b3e3233458550d8dddf29 b/lexer/fuzzer_corpus/1489f923c4dca729178b3e3233458550d8dddf29 new file mode 100644 index 0000000000000..09f370e38f498 Binary files /dev/null and b/lexer/fuzzer_corpus/1489f923c4dca729178b3e3233458550d8dddf29 differ diff --git a/lexer/fuzzer_corpus/14b4cd2980fd137c24ea87f505d3ee4cadfde097 b/lexer/fuzzer_corpus/14b4cd2980fd137c24ea87f505d3ee4cadfde097 new file mode 100644 index 0000000000000..9c0b38b2380f5 Binary files /dev/null and b/lexer/fuzzer_corpus/14b4cd2980fd137c24ea87f505d3ee4cadfde097 differ diff --git a/lexer/fuzzer_corpus/14c55494f5bfead510993f7aa6b19f2a99d78e64 b/lexer/fuzzer_corpus/14c55494f5bfead510993f7aa6b19f2a99d78e64 new file mode 100644 index 0000000000000..0e91b31704b0d Binary files /dev/null and b/lexer/fuzzer_corpus/14c55494f5bfead510993f7aa6b19f2a99d78e64 differ diff --git a/lexer/fuzzer_corpus/15d136091465e9311ea3fff6a81f3a16cd472b9a b/lexer/fuzzer_corpus/15d136091465e9311ea3fff6a81f3a16cd472b9a new file mode 100644 index 0000000000000..c1af64cde97e2 Binary files /dev/null and b/lexer/fuzzer_corpus/15d136091465e9311ea3fff6a81f3a16cd472b9a differ diff --git a/lexer/fuzzer_corpus/15f060997fb0309d15af20cf592235d47e6653cc b/lexer/fuzzer_corpus/15f060997fb0309d15af20cf592235d47e6653cc new file mode 100644 index 0000000000000..2792d423fae5e Binary files /dev/null and b/lexer/fuzzer_corpus/15f060997fb0309d15af20cf592235d47e6653cc differ diff --git a/lexer/fuzzer_corpus/167feb1fda57dac588688aa0d0b7a50683620144 b/lexer/fuzzer_corpus/167feb1fda57dac588688aa0d0b7a50683620144 new file mode 100644 index 0000000000000..f83363c2c8866 Binary files /dev/null and b/lexer/fuzzer_corpus/167feb1fda57dac588688aa0d0b7a50683620144 differ diff --git a/lexer/fuzzer_corpus/1941ef6ffb0e99428fec8f63ce04c9eac4953bf4 b/lexer/fuzzer_corpus/1941ef6ffb0e99428fec8f63ce04c9eac4953bf4 new file mode 100644 index 0000000000000..4e95ab72dec4d Binary files /dev/null and b/lexer/fuzzer_corpus/1941ef6ffb0e99428fec8f63ce04c9eac4953bf4 differ diff --git a/lexer/fuzzer_corpus/19c2134264582d9b56d13f26eb96e92d374574c6 b/lexer/fuzzer_corpus/19c2134264582d9b56d13f26eb96e92d374574c6 new file mode 100644 index 0000000000000..793bf9f37ba91 Binary files /dev/null and b/lexer/fuzzer_corpus/19c2134264582d9b56d13f26eb96e92d374574c6 differ diff --git a/lexer/fuzzer_corpus/1a51e55c866f939f2d132aef8ff5e600242ec29a b/lexer/fuzzer_corpus/1a51e55c866f939f2d132aef8ff5e600242ec29a new file mode 100644 index 0000000000000..781682f8ffe58 Binary files /dev/null and b/lexer/fuzzer_corpus/1a51e55c866f939f2d132aef8ff5e600242ec29a differ diff --git a/lexer/fuzzer_corpus/1a7d939aee936be5d25799c3a57f8e8f3c851c89 b/lexer/fuzzer_corpus/1a7d939aee936be5d25799c3a57f8e8f3c851c89 new file mode 100644 index 0000000000000..617fbf33c8ffc Binary files /dev/null and b/lexer/fuzzer_corpus/1a7d939aee936be5d25799c3a57f8e8f3c851c89 differ diff --git a/lexer/fuzzer_corpus/1a9160bf021e36d9c2bf8f783e5b24211dbc3a28 b/lexer/fuzzer_corpus/1a9160bf021e36d9c2bf8f783e5b24211dbc3a28 new file mode 100644 index 0000000000000..f7c885eec41c1 Binary files /dev/null and b/lexer/fuzzer_corpus/1a9160bf021e36d9c2bf8f783e5b24211dbc3a28 differ diff --git a/lexer/fuzzer_corpus/1abb3bd834e95e4044a7eeca6e54a49d3c6ef802 b/lexer/fuzzer_corpus/1abb3bd834e95e4044a7eeca6e54a49d3c6ef802 new file mode 100644 index 0000000000000..022f8bfacd3c9 Binary files /dev/null and b/lexer/fuzzer_corpus/1abb3bd834e95e4044a7eeca6e54a49d3c6ef802 differ diff --git a/lexer/fuzzer_corpus/1af2d87f0c834bdfe46af5d5f18f014c3294c466 b/lexer/fuzzer_corpus/1af2d87f0c834bdfe46af5d5f18f014c3294c466 new file mode 100644 index 0000000000000..9b1bcee1013e5 Binary files /dev/null and b/lexer/fuzzer_corpus/1af2d87f0c834bdfe46af5d5f18f014c3294c466 differ diff --git a/lexer/fuzzer_corpus/1afe41bb4bfe01753e61e7aac1449e6fa2e96fe6 b/lexer/fuzzer_corpus/1afe41bb4bfe01753e61e7aac1449e6fa2e96fe6 new file mode 100644 index 0000000000000..6424e27068af5 Binary files /dev/null and b/lexer/fuzzer_corpus/1afe41bb4bfe01753e61e7aac1449e6fa2e96fe6 differ diff --git a/lexer/fuzzer_corpus/1bb9f4d20b726e743eb6af284a699ce4afa36583 b/lexer/fuzzer_corpus/1bb9f4d20b726e743eb6af284a699ce4afa36583 new file mode 100644 index 0000000000000..64b808ea47e85 Binary files /dev/null and b/lexer/fuzzer_corpus/1bb9f4d20b726e743eb6af284a699ce4afa36583 differ diff --git a/lexer/fuzzer_corpus/1c62dba4b7e1567683b1824278e7686e26d52834 b/lexer/fuzzer_corpus/1c62dba4b7e1567683b1824278e7686e26d52834 new file mode 100644 index 0000000000000..028edde1e3b11 Binary files /dev/null and b/lexer/fuzzer_corpus/1c62dba4b7e1567683b1824278e7686e26d52834 differ diff --git a/lexer/fuzzer_corpus/1da4e90919ecbf001b876a4972845144cf042f3a b/lexer/fuzzer_corpus/1da4e90919ecbf001b876a4972845144cf042f3a new file mode 100644 index 0000000000000..784dc6c677735 Binary files /dev/null and b/lexer/fuzzer_corpus/1da4e90919ecbf001b876a4972845144cf042f3a differ diff --git a/lexer/fuzzer_corpus/1ec7532018f463cad2ba3d6928fb3c6b46df398a b/lexer/fuzzer_corpus/1ec7532018f463cad2ba3d6928fb3c6b46df398a new file mode 100644 index 0000000000000..79d84859b02ac Binary files /dev/null and b/lexer/fuzzer_corpus/1ec7532018f463cad2ba3d6928fb3c6b46df398a differ diff --git a/lexer/fuzzer_corpus/1f1e84248afa238a7417b29be35fee5af6ba57cc b/lexer/fuzzer_corpus/1f1e84248afa238a7417b29be35fee5af6ba57cc new file mode 100644 index 0000000000000..e23a983194521 Binary files /dev/null and b/lexer/fuzzer_corpus/1f1e84248afa238a7417b29be35fee5af6ba57cc differ diff --git a/lexer/fuzzer_corpus/1ff7f0b20cb7d2be88fe82f1779f71a9b860b7b8 b/lexer/fuzzer_corpus/1ff7f0b20cb7d2be88fe82f1779f71a9b860b7b8 new file mode 100644 index 0000000000000..131176c74d853 Binary files /dev/null and b/lexer/fuzzer_corpus/1ff7f0b20cb7d2be88fe82f1779f71a9b860b7b8 differ diff --git a/lexer/fuzzer_corpus/20e233dd43706cd6dff2c7b774cdc47ead78de71 b/lexer/fuzzer_corpus/20e233dd43706cd6dff2c7b774cdc47ead78de71 new file mode 100644 index 0000000000000..ad6bfa8246c3c Binary files /dev/null and b/lexer/fuzzer_corpus/20e233dd43706cd6dff2c7b774cdc47ead78de71 differ diff --git a/lexer/fuzzer_corpus/213764f0401b8e8a9040c21bc153edff9286d315 b/lexer/fuzzer_corpus/213764f0401b8e8a9040c21bc153edff9286d315 new file mode 100644 index 0000000000000..59d2bc7deba64 Binary files /dev/null and b/lexer/fuzzer_corpus/213764f0401b8e8a9040c21bc153edff9286d315 differ diff --git a/lexer/fuzzer_corpus/213be3df2b6b577edcbb6f913298974fbb8d7324 b/lexer/fuzzer_corpus/213be3df2b6b577edcbb6f913298974fbb8d7324 new file mode 100644 index 0000000000000..a111e2d265d6c Binary files /dev/null and b/lexer/fuzzer_corpus/213be3df2b6b577edcbb6f913298974fbb8d7324 differ diff --git a/lexer/fuzzer_corpus/21ae4af22c4c258550b92d3cea9ce479a88c12a2 b/lexer/fuzzer_corpus/21ae4af22c4c258550b92d3cea9ce479a88c12a2 new file mode 100644 index 0000000000000..a1dcd9939eda0 Binary files /dev/null and b/lexer/fuzzer_corpus/21ae4af22c4c258550b92d3cea9ce479a88c12a2 differ diff --git a/lexer/fuzzer_corpus/21c85dd996a874a885825773fe3ae6f66272ae77 b/lexer/fuzzer_corpus/21c85dd996a874a885825773fe3ae6f66272ae77 new file mode 100644 index 0000000000000..df04010e25528 Binary files /dev/null and b/lexer/fuzzer_corpus/21c85dd996a874a885825773fe3ae6f66272ae77 differ diff --git a/lexer/fuzzer_corpus/21d6931263d831c0aed540fd1bab0b60799af038 b/lexer/fuzzer_corpus/21d6931263d831c0aed540fd1bab0b60799af038 new file mode 100644 index 0000000000000..21fec2366525b Binary files /dev/null and b/lexer/fuzzer_corpus/21d6931263d831c0aed540fd1bab0b60799af038 differ diff --git a/lexer/fuzzer_corpus/21dc828f45d060ee0455004297b50d2b64d17e02 b/lexer/fuzzer_corpus/21dc828f45d060ee0455004297b50d2b64d17e02 new file mode 100644 index 0000000000000..0c7c93b2de938 Binary files /dev/null and b/lexer/fuzzer_corpus/21dc828f45d060ee0455004297b50d2b64d17e02 differ diff --git a/lexer/fuzzer_corpus/221c22da6c2c0fd71d5ebc21876fd57cb77b1b00 b/lexer/fuzzer_corpus/221c22da6c2c0fd71d5ebc21876fd57cb77b1b00 new file mode 100644 index 0000000000000..668dbb044c280 Binary files /dev/null and b/lexer/fuzzer_corpus/221c22da6c2c0fd71d5ebc21876fd57cb77b1b00 differ diff --git a/lexer/fuzzer_corpus/22812a981f0458f49a0cae159de66a24631464a3 b/lexer/fuzzer_corpus/22812a981f0458f49a0cae159de66a24631464a3 new file mode 100644 index 0000000000000..4ae545b287310 Binary files /dev/null and b/lexer/fuzzer_corpus/22812a981f0458f49a0cae159de66a24631464a3 differ diff --git a/lexer/fuzzer_corpus/22854bee28c089600335e2c16d6c6adad03fba54 b/lexer/fuzzer_corpus/22854bee28c089600335e2c16d6c6adad03fba54 new file mode 100644 index 0000000000000..ce5048df466bd Binary files /dev/null and b/lexer/fuzzer_corpus/22854bee28c089600335e2c16d6c6adad03fba54 differ diff --git a/lexer/fuzzer_corpus/234563c77ded0c75e0583be09a1f4a8a016b8024 b/lexer/fuzzer_corpus/234563c77ded0c75e0583be09a1f4a8a016b8024 new file mode 100644 index 0000000000000..c903c49efec8d Binary files /dev/null and b/lexer/fuzzer_corpus/234563c77ded0c75e0583be09a1f4a8a016b8024 differ diff --git a/lexer/fuzzer_corpus/236e3801dd0cf62a835705da142af1587e498516 b/lexer/fuzzer_corpus/236e3801dd0cf62a835705da142af1587e498516 new file mode 100644 index 0000000000000..9565ed98e2601 Binary files /dev/null and b/lexer/fuzzer_corpus/236e3801dd0cf62a835705da142af1587e498516 differ diff --git a/lexer/fuzzer_corpus/24426f52ba7ad6d178e9e20cb28cf9182d51871f b/lexer/fuzzer_corpus/24426f52ba7ad6d178e9e20cb28cf9182d51871f new file mode 100644 index 0000000000000..0f031070cc2bd Binary files /dev/null and b/lexer/fuzzer_corpus/24426f52ba7ad6d178e9e20cb28cf9182d51871f differ diff --git a/lexer/fuzzer_corpus/24586a887ef47d547a4bea0a89972fcf3c19c454 b/lexer/fuzzer_corpus/24586a887ef47d547a4bea0a89972fcf3c19c454 new file mode 100644 index 0000000000000..2c11fba382ea2 Binary files /dev/null and b/lexer/fuzzer_corpus/24586a887ef47d547a4bea0a89972fcf3c19c454 differ diff --git a/lexer/fuzzer_corpus/2600c18db42bde973e72b07f0dadf5d32fbfc48a b/lexer/fuzzer_corpus/2600c18db42bde973e72b07f0dadf5d32fbfc48a new file mode 100644 index 0000000000000..1906acc4542d5 Binary files /dev/null and b/lexer/fuzzer_corpus/2600c18db42bde973e72b07f0dadf5d32fbfc48a differ diff --git a/lexer/fuzzer_corpus/26514fd3aa2e7d09114d7b5c971e9cbc96e49aba b/lexer/fuzzer_corpus/26514fd3aa2e7d09114d7b5c971e9cbc96e49aba new file mode 100644 index 0000000000000..dff5af643fd90 Binary files /dev/null and b/lexer/fuzzer_corpus/26514fd3aa2e7d09114d7b5c971e9cbc96e49aba differ diff --git a/lexer/fuzzer_corpus/26ffdd8987ca20c37b2e8b30866e534342d1c175 b/lexer/fuzzer_corpus/26ffdd8987ca20c37b2e8b30866e534342d1c175 new file mode 100644 index 0000000000000..d95feedca75c3 Binary files /dev/null and b/lexer/fuzzer_corpus/26ffdd8987ca20c37b2e8b30866e534342d1c175 differ diff --git a/lexer/fuzzer_corpus/274399a85ea96a507d87d6199c58ea701113ec73 b/lexer/fuzzer_corpus/274399a85ea96a507d87d6199c58ea701113ec73 new file mode 100644 index 0000000000000..ea788c121c7ef Binary files /dev/null and b/lexer/fuzzer_corpus/274399a85ea96a507d87d6199c58ea701113ec73 differ diff --git a/lexer/fuzzer_corpus/28135abd5f66044cb0fd6f82a3b0dd81b2059593 b/lexer/fuzzer_corpus/28135abd5f66044cb0fd6f82a3b0dd81b2059593 new file mode 100644 index 0000000000000..70b3c2f4653f0 Binary files /dev/null and b/lexer/fuzzer_corpus/28135abd5f66044cb0fd6f82a3b0dd81b2059593 differ diff --git a/lexer/fuzzer_corpus/281bc1b2b4a20d112c7ea17951397b5a789b5ee8 b/lexer/fuzzer_corpus/281bc1b2b4a20d112c7ea17951397b5a789b5ee8 new file mode 100644 index 0000000000000..c9199dc5a808a Binary files /dev/null and b/lexer/fuzzer_corpus/281bc1b2b4a20d112c7ea17951397b5a789b5ee8 differ diff --git a/lexer/fuzzer_corpus/28434409ed42ff7b5f6cf4c0731765f70aae12ba b/lexer/fuzzer_corpus/28434409ed42ff7b5f6cf4c0731765f70aae12ba new file mode 100644 index 0000000000000..ef121ea907f3d Binary files /dev/null and b/lexer/fuzzer_corpus/28434409ed42ff7b5f6cf4c0731765f70aae12ba differ diff --git a/lexer/fuzzer_corpus/29b8dee814dce4d94d388d035c10d4f6149b934b b/lexer/fuzzer_corpus/29b8dee814dce4d94d388d035c10d4f6149b934b new file mode 100644 index 0000000000000..bddb79f5a2513 Binary files /dev/null and b/lexer/fuzzer_corpus/29b8dee814dce4d94d388d035c10d4f6149b934b differ diff --git a/lexer/fuzzer_corpus/29c07937042029d42ded435734bb17a9da1a13ef b/lexer/fuzzer_corpus/29c07937042029d42ded435734bb17a9da1a13ef new file mode 100644 index 0000000000000..176bf4677d81a Binary files /dev/null and b/lexer/fuzzer_corpus/29c07937042029d42ded435734bb17a9da1a13ef differ diff --git a/lexer/fuzzer_corpus/29dd4eab6410e5cad789f8ad21b564f5ff580b62 b/lexer/fuzzer_corpus/29dd4eab6410e5cad789f8ad21b564f5ff580b62 new file mode 100644 index 0000000000000..88ce8d1d865fe Binary files /dev/null and b/lexer/fuzzer_corpus/29dd4eab6410e5cad789f8ad21b564f5ff580b62 differ diff --git a/lexer/fuzzer_corpus/2a6775c7917e01ee00754918264977fdd60f15ea b/lexer/fuzzer_corpus/2a6775c7917e01ee00754918264977fdd60f15ea new file mode 100644 index 0000000000000..94c6d820f8f8f Binary files /dev/null and b/lexer/fuzzer_corpus/2a6775c7917e01ee00754918264977fdd60f15ea differ diff --git a/lexer/fuzzer_corpus/2ad3087a982396bd302d9d445d6c5423c248d3c2 b/lexer/fuzzer_corpus/2ad3087a982396bd302d9d445d6c5423c248d3c2 new file mode 100644 index 0000000000000..20c1fcac2ab4c Binary files /dev/null and b/lexer/fuzzer_corpus/2ad3087a982396bd302d9d445d6c5423c248d3c2 differ diff --git a/lexer/fuzzer_corpus/2b58f8d18c8ddfb98618711a7da2afd0efc9bcf8 b/lexer/fuzzer_corpus/2b58f8d18c8ddfb98618711a7da2afd0efc9bcf8 new file mode 100644 index 0000000000000..16cf2c5ad7f5b Binary files /dev/null and b/lexer/fuzzer_corpus/2b58f8d18c8ddfb98618711a7da2afd0efc9bcf8 differ diff --git a/lexer/fuzzer_corpus/2bea1061181244323cdf95ad6a489b72ee38d225 b/lexer/fuzzer_corpus/2bea1061181244323cdf95ad6a489b72ee38d225 new file mode 100644 index 0000000000000..ff5495519f447 Binary files /dev/null and b/lexer/fuzzer_corpus/2bea1061181244323cdf95ad6a489b72ee38d225 differ diff --git a/lexer/fuzzer_corpus/2cce2108f1d888fdecf0c406fd5fb2ab0448d88d b/lexer/fuzzer_corpus/2cce2108f1d888fdecf0c406fd5fb2ab0448d88d new file mode 100644 index 0000000000000..3a156a06226e9 Binary files /dev/null and b/lexer/fuzzer_corpus/2cce2108f1d888fdecf0c406fd5fb2ab0448d88d differ diff --git a/lexer/fuzzer_corpus/2d1f6ee26e62965c25a2540c47cda839e86cbc9d b/lexer/fuzzer_corpus/2d1f6ee26e62965c25a2540c47cda839e86cbc9d new file mode 100644 index 0000000000000..5a41da8da48cc Binary files /dev/null and b/lexer/fuzzer_corpus/2d1f6ee26e62965c25a2540c47cda839e86cbc9d differ diff --git a/lexer/fuzzer_corpus/2e047eb1a04b386c558490f7634175199fb86d2b b/lexer/fuzzer_corpus/2e047eb1a04b386c558490f7634175199fb86d2b new file mode 100644 index 0000000000000..e9fb7f046d1b9 Binary files /dev/null and b/lexer/fuzzer_corpus/2e047eb1a04b386c558490f7634175199fb86d2b differ diff --git a/lexer/fuzzer_corpus/2eca067d0407b2ce79b7fe3aa0abf38b48178e16 b/lexer/fuzzer_corpus/2eca067d0407b2ce79b7fe3aa0abf38b48178e16 new file mode 100644 index 0000000000000..e49f1c02698aa Binary files /dev/null and b/lexer/fuzzer_corpus/2eca067d0407b2ce79b7fe3aa0abf38b48178e16 differ diff --git a/lexer/fuzzer_corpus/2eca430ce72ba8cfe14a36dae22c3af594455d78 b/lexer/fuzzer_corpus/2eca430ce72ba8cfe14a36dae22c3af594455d78 new file mode 100644 index 0000000000000..1ec00b3b0ad5f Binary files /dev/null and b/lexer/fuzzer_corpus/2eca430ce72ba8cfe14a36dae22c3af594455d78 differ diff --git a/lexer/fuzzer_corpus/300268d606563a63b1fa078deedf83afcaaae4f0 b/lexer/fuzzer_corpus/300268d606563a63b1fa078deedf83afcaaae4f0 new file mode 100644 index 0000000000000..97fce95451869 Binary files /dev/null and b/lexer/fuzzer_corpus/300268d606563a63b1fa078deedf83afcaaae4f0 differ diff --git a/lexer/fuzzer_corpus/30122a51022d504d0bd883ec9083db0ebc122f3a b/lexer/fuzzer_corpus/30122a51022d504d0bd883ec9083db0ebc122f3a new file mode 100644 index 0000000000000..2e59967502b8d Binary files /dev/null and b/lexer/fuzzer_corpus/30122a51022d504d0bd883ec9083db0ebc122f3a differ diff --git a/lexer/fuzzer_corpus/3055e6fc37e2846c56a0a7ef9c910807325c2df1 b/lexer/fuzzer_corpus/3055e6fc37e2846c56a0a7ef9c910807325c2df1 new file mode 100644 index 0000000000000..16c31a2925c4e Binary files /dev/null and b/lexer/fuzzer_corpus/3055e6fc37e2846c56a0a7ef9c910807325c2df1 differ diff --git a/lexer/fuzzer_corpus/30a79f74923a4a0450c38b8d78c32dd0ec49fb1e b/lexer/fuzzer_corpus/30a79f74923a4a0450c38b8d78c32dd0ec49fb1e new file mode 100644 index 0000000000000..932e776d40a18 Binary files /dev/null and b/lexer/fuzzer_corpus/30a79f74923a4a0450c38b8d78c32dd0ec49fb1e differ diff --git a/lexer/fuzzer_corpus/30d2affb78dce801693964e7aa7a776e46764142 b/lexer/fuzzer_corpus/30d2affb78dce801693964e7aa7a776e46764142 new file mode 100644 index 0000000000000..d9bbf5d988ed4 Binary files /dev/null and b/lexer/fuzzer_corpus/30d2affb78dce801693964e7aa7a776e46764142 differ diff --git a/lexer/fuzzer_corpus/33970181a654f47a1b7350a040f66460573c2b37 b/lexer/fuzzer_corpus/33970181a654f47a1b7350a040f66460573c2b37 new file mode 100644 index 0000000000000..4d383411ba819 Binary files /dev/null and b/lexer/fuzzer_corpus/33970181a654f47a1b7350a040f66460573c2b37 differ diff --git a/lexer/fuzzer_corpus/33fbd678677fdcc9ff26ce50d77282550ed7b152 b/lexer/fuzzer_corpus/33fbd678677fdcc9ff26ce50d77282550ed7b152 new file mode 100644 index 0000000000000..8353f74c7d513 Binary files /dev/null and b/lexer/fuzzer_corpus/33fbd678677fdcc9ff26ce50d77282550ed7b152 differ diff --git a/lexer/fuzzer_corpus/3410eb3f248569d94b015591f12cf76034f8c64d b/lexer/fuzzer_corpus/3410eb3f248569d94b015591f12cf76034f8c64d new file mode 100644 index 0000000000000..47e9e6d846232 Binary files /dev/null and b/lexer/fuzzer_corpus/3410eb3f248569d94b015591f12cf76034f8c64d differ diff --git a/lexer/fuzzer_corpus/34c37b9bcc5ca133876f590f4225bd361357eb13 b/lexer/fuzzer_corpus/34c37b9bcc5ca133876f590f4225bd361357eb13 new file mode 100644 index 0000000000000..e0671ee2e5981 Binary files /dev/null and b/lexer/fuzzer_corpus/34c37b9bcc5ca133876f590f4225bd361357eb13 differ diff --git a/lexer/fuzzer_corpus/36388f570f44f86bc535dc8e510721d1c28a0637 b/lexer/fuzzer_corpus/36388f570f44f86bc535dc8e510721d1c28a0637 new file mode 100644 index 0000000000000..766acf8ded72d Binary files /dev/null and b/lexer/fuzzer_corpus/36388f570f44f86bc535dc8e510721d1c28a0637 differ diff --git a/lexer/fuzzer_corpus/36444b4480602bbc82c421d3a5dca926ee56f17d b/lexer/fuzzer_corpus/36444b4480602bbc82c421d3a5dca926ee56f17d new file mode 100644 index 0000000000000..e9a2d0753a339 Binary files /dev/null and b/lexer/fuzzer_corpus/36444b4480602bbc82c421d3a5dca926ee56f17d differ diff --git a/lexer/fuzzer_corpus/378f7e36a287439cdbcb171241c946981517e6b7 b/lexer/fuzzer_corpus/378f7e36a287439cdbcb171241c946981517e6b7 new file mode 100644 index 0000000000000..fda26ba2cd943 Binary files /dev/null and b/lexer/fuzzer_corpus/378f7e36a287439cdbcb171241c946981517e6b7 differ diff --git a/lexer/fuzzer_corpus/37d5919815fa0c3579c90f5882bc78d2175f2658 b/lexer/fuzzer_corpus/37d5919815fa0c3579c90f5882bc78d2175f2658 new file mode 100644 index 0000000000000..f2910ecccfd24 Binary files /dev/null and b/lexer/fuzzer_corpus/37d5919815fa0c3579c90f5882bc78d2175f2658 differ diff --git a/lexer/fuzzer_corpus/37d7800daba869cfb014eb10e5f46a67fbbbf14d b/lexer/fuzzer_corpus/37d7800daba869cfb014eb10e5f46a67fbbbf14d new file mode 100644 index 0000000000000..0ea445a911324 Binary files /dev/null and b/lexer/fuzzer_corpus/37d7800daba869cfb014eb10e5f46a67fbbbf14d differ diff --git a/lexer/fuzzer_corpus/38197ad1d6330f1836fecfc754c19dcfe8d8da6d b/lexer/fuzzer_corpus/38197ad1d6330f1836fecfc754c19dcfe8d8da6d new file mode 100644 index 0000000000000..c124c462d6431 Binary files /dev/null and b/lexer/fuzzer_corpus/38197ad1d6330f1836fecfc754c19dcfe8d8da6d differ diff --git a/lexer/fuzzer_corpus/383ecf2dd4fee1f6fad7a2fc6a1014ea612cc705 b/lexer/fuzzer_corpus/383ecf2dd4fee1f6fad7a2fc6a1014ea612cc705 new file mode 100644 index 0000000000000..4934f87de9b15 Binary files /dev/null and b/lexer/fuzzer_corpus/383ecf2dd4fee1f6fad7a2fc6a1014ea612cc705 differ diff --git a/lexer/fuzzer_corpus/38aae1be66ab0f6d35121b4372baa54f3a734cd1 b/lexer/fuzzer_corpus/38aae1be66ab0f6d35121b4372baa54f3a734cd1 new file mode 100644 index 0000000000000..c065f4708f68d Binary files /dev/null and b/lexer/fuzzer_corpus/38aae1be66ab0f6d35121b4372baa54f3a734cd1 differ diff --git a/lexer/fuzzer_corpus/38d2de4fb4421617c287855ede415e253edf27fc b/lexer/fuzzer_corpus/38d2de4fb4421617c287855ede415e253edf27fc new file mode 100644 index 0000000000000..e5fc7bc47d6cf Binary files /dev/null and b/lexer/fuzzer_corpus/38d2de4fb4421617c287855ede415e253edf27fc differ diff --git a/lexer/fuzzer_corpus/39cbaf1cff432636f99ae2086461c48f081f3608 b/lexer/fuzzer_corpus/39cbaf1cff432636f99ae2086461c48f081f3608 new file mode 100644 index 0000000000000..00ef342f01a4c Binary files /dev/null and b/lexer/fuzzer_corpus/39cbaf1cff432636f99ae2086461c48f081f3608 differ diff --git a/lexer/fuzzer_corpus/39fbf7272789a966efbe93fc511b64c764d89513 b/lexer/fuzzer_corpus/39fbf7272789a966efbe93fc511b64c764d89513 new file mode 100644 index 0000000000000..b31dc9287e8a6 Binary files /dev/null and b/lexer/fuzzer_corpus/39fbf7272789a966efbe93fc511b64c764d89513 differ diff --git a/lexer/fuzzer_corpus/3a33064962ff92045a47b9409449cf1552d685d7 b/lexer/fuzzer_corpus/3a33064962ff92045a47b9409449cf1552d685d7 new file mode 100644 index 0000000000000..666f1e688adcc Binary files /dev/null and b/lexer/fuzzer_corpus/3a33064962ff92045a47b9409449cf1552d685d7 differ diff --git a/lexer/fuzzer_corpus/3a3fc43f4c64d0305c0d697a33acb209e48971af b/lexer/fuzzer_corpus/3a3fc43f4c64d0305c0d697a33acb209e48971af new file mode 100644 index 0000000000000..99565e5d13972 Binary files /dev/null and b/lexer/fuzzer_corpus/3a3fc43f4c64d0305c0d697a33acb209e48971af differ diff --git a/lexer/fuzzer_corpus/3a85b6e409d6180559bbaf0282fe374734ac8ab7 b/lexer/fuzzer_corpus/3a85b6e409d6180559bbaf0282fe374734ac8ab7 new file mode 100644 index 0000000000000..6fdc170de35be Binary files /dev/null and b/lexer/fuzzer_corpus/3a85b6e409d6180559bbaf0282fe374734ac8ab7 differ diff --git a/lexer/fuzzer_corpus/3aef9b8111cb335c85a241d9f59929e11707730f b/lexer/fuzzer_corpus/3aef9b8111cb335c85a241d9f59929e11707730f new file mode 100644 index 0000000000000..629e8223a31ef Binary files /dev/null and b/lexer/fuzzer_corpus/3aef9b8111cb335c85a241d9f59929e11707730f differ diff --git a/lexer/fuzzer_corpus/3d6d742aaaea3abffa078fd4c03af3cd03c5bb94 b/lexer/fuzzer_corpus/3d6d742aaaea3abffa078fd4c03af3cd03c5bb94 new file mode 100644 index 0000000000000..388026e652990 Binary files /dev/null and b/lexer/fuzzer_corpus/3d6d742aaaea3abffa078fd4c03af3cd03c5bb94 differ diff --git a/lexer/fuzzer_corpus/3e21eed3aab591d3aab15d975b2660648ce7bc9e b/lexer/fuzzer_corpus/3e21eed3aab591d3aab15d975b2660648ce7bc9e new file mode 100644 index 0000000000000..81d5b1e5f52db Binary files /dev/null and b/lexer/fuzzer_corpus/3e21eed3aab591d3aab15d975b2660648ce7bc9e differ diff --git a/lexer/fuzzer_corpus/3e23582c9fe275a17897308f4529ce965ef26639 b/lexer/fuzzer_corpus/3e23582c9fe275a17897308f4529ce965ef26639 new file mode 100644 index 0000000000000..3dcd9020616ae Binary files /dev/null and b/lexer/fuzzer_corpus/3e23582c9fe275a17897308f4529ce965ef26639 differ diff --git a/lexer/fuzzer_corpus/3f0c682d7231780f496758f26a19dea90eb0a63f b/lexer/fuzzer_corpus/3f0c682d7231780f496758f26a19dea90eb0a63f new file mode 100644 index 0000000000000..caabe06c6d903 Binary files /dev/null and b/lexer/fuzzer_corpus/3f0c682d7231780f496758f26a19dea90eb0a63f differ diff --git a/lexer/fuzzer_corpus/3f128a449c26d99e65f137b08835072d0f4fa93a b/lexer/fuzzer_corpus/3f128a449c26d99e65f137b08835072d0f4fa93a new file mode 100644 index 0000000000000..3c0a8aafd85ab Binary files /dev/null and b/lexer/fuzzer_corpus/3f128a449c26d99e65f137b08835072d0f4fa93a differ diff --git a/lexer/fuzzer_corpus/3f5305709a3b6e105cf4b8c1368a8d5c0540a2da b/lexer/fuzzer_corpus/3f5305709a3b6e105cf4b8c1368a8d5c0540a2da new file mode 100644 index 0000000000000..fbf98e21c0600 Binary files /dev/null and b/lexer/fuzzer_corpus/3f5305709a3b6e105cf4b8c1368a8d5c0540a2da differ diff --git a/lexer/fuzzer_corpus/3ffa1177da8e5aa9fe3d071384f8958da89d40a6 b/lexer/fuzzer_corpus/3ffa1177da8e5aa9fe3d071384f8958da89d40a6 new file mode 100644 index 0000000000000..6e5ca6c898e91 Binary files /dev/null and b/lexer/fuzzer_corpus/3ffa1177da8e5aa9fe3d071384f8958da89d40a6 differ diff --git a/lexer/fuzzer_corpus/401b5ffa27fd9bcc9c4c99acc3294b51b7ab5435 b/lexer/fuzzer_corpus/401b5ffa27fd9bcc9c4c99acc3294b51b7ab5435 new file mode 100644 index 0000000000000..b2bf5da66960b Binary files /dev/null and b/lexer/fuzzer_corpus/401b5ffa27fd9bcc9c4c99acc3294b51b7ab5435 differ diff --git a/lexer/fuzzer_corpus/403d7b69e46cde2a0c35e051dc0343b283943a6a b/lexer/fuzzer_corpus/403d7b69e46cde2a0c35e051dc0343b283943a6a new file mode 100644 index 0000000000000..b6c0b8cecb001 Binary files /dev/null and b/lexer/fuzzer_corpus/403d7b69e46cde2a0c35e051dc0343b283943a6a differ diff --git a/lexer/fuzzer_corpus/419ccf5d9e7c1a163b62230196e1154953a91f9f b/lexer/fuzzer_corpus/419ccf5d9e7c1a163b62230196e1154953a91f9f new file mode 100644 index 0000000000000..f45c23c4452aa Binary files /dev/null and b/lexer/fuzzer_corpus/419ccf5d9e7c1a163b62230196e1154953a91f9f differ diff --git a/lexer/fuzzer_corpus/42880c44925ec18e3a69b2274b05e0aeff25e454 b/lexer/fuzzer_corpus/42880c44925ec18e3a69b2274b05e0aeff25e454 new file mode 100644 index 0000000000000..f3365dbffe568 Binary files /dev/null and b/lexer/fuzzer_corpus/42880c44925ec18e3a69b2274b05e0aeff25e454 differ diff --git a/lexer/fuzzer_corpus/434337b14da4a5fb3cdc96cdf47f17517583815c b/lexer/fuzzer_corpus/434337b14da4a5fb3cdc96cdf47f17517583815c new file mode 100644 index 0000000000000..10daccd797325 Binary files /dev/null and b/lexer/fuzzer_corpus/434337b14da4a5fb3cdc96cdf47f17517583815c differ diff --git a/lexer/fuzzer_corpus/4351e9a0bb6b53c76164f05125a112cb283cfd81 b/lexer/fuzzer_corpus/4351e9a0bb6b53c76164f05125a112cb283cfd81 new file mode 100644 index 0000000000000..72f7dcfc664ce Binary files /dev/null and b/lexer/fuzzer_corpus/4351e9a0bb6b53c76164f05125a112cb283cfd81 differ diff --git a/lexer/fuzzer_corpus/4480089669b1f8da9065f5eb9491687d9e31e9da b/lexer/fuzzer_corpus/4480089669b1f8da9065f5eb9491687d9e31e9da new file mode 100644 index 0000000000000..cba12705e046b Binary files /dev/null and b/lexer/fuzzer_corpus/4480089669b1f8da9065f5eb9491687d9e31e9da differ diff --git a/lexer/fuzzer_corpus/4487fe428cf14de470fa39e874acece46fcfcbe8 b/lexer/fuzzer_corpus/4487fe428cf14de470fa39e874acece46fcfcbe8 new file mode 100644 index 0000000000000..7a4913556edc7 Binary files /dev/null and b/lexer/fuzzer_corpus/4487fe428cf14de470fa39e874acece46fcfcbe8 differ diff --git a/lexer/fuzzer_corpus/44df8bdb51a76972b66deb94dfdf5783de99a64e b/lexer/fuzzer_corpus/44df8bdb51a76972b66deb94dfdf5783de99a64e new file mode 100644 index 0000000000000..0c822c0282606 Binary files /dev/null and b/lexer/fuzzer_corpus/44df8bdb51a76972b66deb94dfdf5783de99a64e differ diff --git a/lexer/fuzzer_corpus/467d884be458d07e7fd5bdda5fff78c9133a1212 b/lexer/fuzzer_corpus/467d884be458d07e7fd5bdda5fff78c9133a1212 new file mode 100644 index 0000000000000..87a98f1c7a57e Binary files /dev/null and b/lexer/fuzzer_corpus/467d884be458d07e7fd5bdda5fff78c9133a1212 differ diff --git a/lexer/fuzzer_corpus/4852364d47db4ef47740f9184a39e0631aa69cd0 b/lexer/fuzzer_corpus/4852364d47db4ef47740f9184a39e0631aa69cd0 new file mode 100644 index 0000000000000..e6090a5537f2f Binary files /dev/null and b/lexer/fuzzer_corpus/4852364d47db4ef47740f9184a39e0631aa69cd0 differ diff --git a/lexer/fuzzer_corpus/487b9de35feaf3e349a411438344e46c870120d5 b/lexer/fuzzer_corpus/487b9de35feaf3e349a411438344e46c870120d5 new file mode 100644 index 0000000000000..124a6b7225410 Binary files /dev/null and b/lexer/fuzzer_corpus/487b9de35feaf3e349a411438344e46c870120d5 differ diff --git a/lexer/fuzzer_corpus/48ba5eedfbf3c64e54d39fb78dcef8081d4cafc2 b/lexer/fuzzer_corpus/48ba5eedfbf3c64e54d39fb78dcef8081d4cafc2 new file mode 100644 index 0000000000000..22bec37c5a382 Binary files /dev/null and b/lexer/fuzzer_corpus/48ba5eedfbf3c64e54d39fb78dcef8081d4cafc2 differ diff --git a/lexer/fuzzer_corpus/49af95277336d8c0ed0d7a7522e73f8c87d89020 b/lexer/fuzzer_corpus/49af95277336d8c0ed0d7a7522e73f8c87d89020 new file mode 100644 index 0000000000000..8fe164d2c78fb Binary files /dev/null and b/lexer/fuzzer_corpus/49af95277336d8c0ed0d7a7522e73f8c87d89020 differ diff --git a/lexer/fuzzer_corpus/49d400fc7de4b5392935638c26ce2e62778d416e b/lexer/fuzzer_corpus/49d400fc7de4b5392935638c26ce2e62778d416e new file mode 100644 index 0000000000000..8c93bae617460 Binary files /dev/null and b/lexer/fuzzer_corpus/49d400fc7de4b5392935638c26ce2e62778d416e differ diff --git a/lexer/fuzzer_corpus/4a0034191d5d0e629583d8b06b6a4a81ce024bee b/lexer/fuzzer_corpus/4a0034191d5d0e629583d8b06b6a4a81ce024bee new file mode 100644 index 0000000000000..af0f25b743b7f Binary files /dev/null and b/lexer/fuzzer_corpus/4a0034191d5d0e629583d8b06b6a4a81ce024bee differ diff --git a/lexer/fuzzer_corpus/4b32fa9e7beb0500c4f76cdf27a4d1be933ddc68 b/lexer/fuzzer_corpus/4b32fa9e7beb0500c4f76cdf27a4d1be933ddc68 new file mode 100644 index 0000000000000..560af9a8e9807 Binary files /dev/null and b/lexer/fuzzer_corpus/4b32fa9e7beb0500c4f76cdf27a4d1be933ddc68 differ diff --git a/lexer/fuzzer_corpus/4b53f47724ca22ac2b26e7b2191779c9fe74fc4c b/lexer/fuzzer_corpus/4b53f47724ca22ac2b26e7b2191779c9fe74fc4c new file mode 100644 index 0000000000000..a055d6e13ca59 Binary files /dev/null and b/lexer/fuzzer_corpus/4b53f47724ca22ac2b26e7b2191779c9fe74fc4c differ diff --git a/lexer/fuzzer_corpus/4c5e150ebdd825a7ad25ac960eed57c54838c614 b/lexer/fuzzer_corpus/4c5e150ebdd825a7ad25ac960eed57c54838c614 new file mode 100644 index 0000000000000..0486ee7404f96 Binary files /dev/null and b/lexer/fuzzer_corpus/4c5e150ebdd825a7ad25ac960eed57c54838c614 differ diff --git a/lexer/fuzzer_corpus/4cb14c262491d3c0e6d47cef077c5ff05353e43d b/lexer/fuzzer_corpus/4cb14c262491d3c0e6d47cef077c5ff05353e43d new file mode 100644 index 0000000000000..832a87de7fcf5 Binary files /dev/null and b/lexer/fuzzer_corpus/4cb14c262491d3c0e6d47cef077c5ff05353e43d differ diff --git a/lexer/fuzzer_corpus/4d042b03473298b5ef025639d90062272204d6b9 b/lexer/fuzzer_corpus/4d042b03473298b5ef025639d90062272204d6b9 new file mode 100644 index 0000000000000..e79f968dd98f6 Binary files /dev/null and b/lexer/fuzzer_corpus/4d042b03473298b5ef025639d90062272204d6b9 differ diff --git a/lexer/fuzzer_corpus/4dd2cd4a75976db57d91729517748dc8f1e9eb39 b/lexer/fuzzer_corpus/4dd2cd4a75976db57d91729517748dc8f1e9eb39 new file mode 100644 index 0000000000000..08bd1ebb83ec0 Binary files /dev/null and b/lexer/fuzzer_corpus/4dd2cd4a75976db57d91729517748dc8f1e9eb39 differ diff --git a/lexer/fuzzer_corpus/4df44f90bc0f13fb557f109e46647b4d91a5014e b/lexer/fuzzer_corpus/4df44f90bc0f13fb557f109e46647b4d91a5014e new file mode 100644 index 0000000000000..8f45cfa13dfb5 Binary files /dev/null and b/lexer/fuzzer_corpus/4df44f90bc0f13fb557f109e46647b4d91a5014e differ diff --git a/lexer/fuzzer_corpus/4e5560d451aa505e253511657c115baf69d8e6f8 b/lexer/fuzzer_corpus/4e5560d451aa505e253511657c115baf69d8e6f8 new file mode 100644 index 0000000000000..6908c0d1c27f3 Binary files /dev/null and b/lexer/fuzzer_corpus/4e5560d451aa505e253511657c115baf69d8e6f8 differ diff --git a/lexer/fuzzer_corpus/4e6876ea3989c69057b0a81a029f83e5246f70dc b/lexer/fuzzer_corpus/4e6876ea3989c69057b0a81a029f83e5246f70dc new file mode 100644 index 0000000000000..5bb50f7ca22c8 Binary files /dev/null and b/lexer/fuzzer_corpus/4e6876ea3989c69057b0a81a029f83e5246f70dc differ diff --git a/lexer/fuzzer_corpus/4fbc30b7354b6bd455608e7fb5bb7f6feb37941e b/lexer/fuzzer_corpus/4fbc30b7354b6bd455608e7fb5bb7f6feb37941e new file mode 100644 index 0000000000000..c651b16d59030 Binary files /dev/null and b/lexer/fuzzer_corpus/4fbc30b7354b6bd455608e7fb5bb7f6feb37941e differ diff --git a/lexer/fuzzer_corpus/532fd1978e5444b41c97d1639fe7b8efc8da14e9 b/lexer/fuzzer_corpus/532fd1978e5444b41c97d1639fe7b8efc8da14e9 new file mode 100644 index 0000000000000..808e09fe9df36 Binary files /dev/null and b/lexer/fuzzer_corpus/532fd1978e5444b41c97d1639fe7b8efc8da14e9 differ diff --git a/lexer/fuzzer_corpus/5378a1d0f6db372c2e26926f3ab692aaff281d5d b/lexer/fuzzer_corpus/5378a1d0f6db372c2e26926f3ab692aaff281d5d new file mode 100644 index 0000000000000..354752d296327 Binary files /dev/null and b/lexer/fuzzer_corpus/5378a1d0f6db372c2e26926f3ab692aaff281d5d differ diff --git a/lexer/fuzzer_corpus/53f0d72af41e99e951722231524ec437759422f2 b/lexer/fuzzer_corpus/53f0d72af41e99e951722231524ec437759422f2 new file mode 100644 index 0000000000000..7afd8da9b4f60 Binary files /dev/null and b/lexer/fuzzer_corpus/53f0d72af41e99e951722231524ec437759422f2 differ diff --git a/lexer/fuzzer_corpus/5458cc3caca58cf412198f6d5085f370b7d1cb97 b/lexer/fuzzer_corpus/5458cc3caca58cf412198f6d5085f370b7d1cb97 new file mode 100644 index 0000000000000..0763d851ac9af Binary files /dev/null and b/lexer/fuzzer_corpus/5458cc3caca58cf412198f6d5085f370b7d1cb97 differ diff --git a/lexer/fuzzer_corpus/5577956e2ca1e38f1ece81a6c2190490c3273f20 b/lexer/fuzzer_corpus/5577956e2ca1e38f1ece81a6c2190490c3273f20 new file mode 100644 index 0000000000000..6aeece34771cf Binary files /dev/null and b/lexer/fuzzer_corpus/5577956e2ca1e38f1ece81a6c2190490c3273f20 differ diff --git a/lexer/fuzzer_corpus/568e139f25140400eefdf6d4e2a38bc21e61d4c0 b/lexer/fuzzer_corpus/568e139f25140400eefdf6d4e2a38bc21e61d4c0 new file mode 100644 index 0000000000000..322a3b56a7e89 Binary files /dev/null and b/lexer/fuzzer_corpus/568e139f25140400eefdf6d4e2a38bc21e61d4c0 differ diff --git a/lexer/fuzzer_corpus/56bc35408f7d618b0b2ad0cc28738fa9d38a0bd2 b/lexer/fuzzer_corpus/56bc35408f7d618b0b2ad0cc28738fa9d38a0bd2 new file mode 100644 index 0000000000000..702b7635f83d4 Binary files /dev/null and b/lexer/fuzzer_corpus/56bc35408f7d618b0b2ad0cc28738fa9d38a0bd2 differ diff --git a/lexer/fuzzer_corpus/57e42e4edb7e862c38e24209fea0d89c06034e4d b/lexer/fuzzer_corpus/57e42e4edb7e862c38e24209fea0d89c06034e4d new file mode 100644 index 0000000000000..04d0e139f9fbc Binary files /dev/null and b/lexer/fuzzer_corpus/57e42e4edb7e862c38e24209fea0d89c06034e4d differ diff --git a/lexer/fuzzer_corpus/59837c18261ef4bf33c51437a98d25cebf414840 b/lexer/fuzzer_corpus/59837c18261ef4bf33c51437a98d25cebf414840 new file mode 100644 index 0000000000000..8d2247e5348c8 Binary files /dev/null and b/lexer/fuzzer_corpus/59837c18261ef4bf33c51437a98d25cebf414840 differ diff --git a/lexer/fuzzer_corpus/598b22649052ab5cf9c8f2d9e43c97c0ca1ecb6f b/lexer/fuzzer_corpus/598b22649052ab5cf9c8f2d9e43c97c0ca1ecb6f new file mode 100644 index 0000000000000..a73270d12a3a3 Binary files /dev/null and b/lexer/fuzzer_corpus/598b22649052ab5cf9c8f2d9e43c97c0ca1ecb6f differ diff --git a/lexer/fuzzer_corpus/59f33d4f075c2cf6448f94f623fac85834bdb2cf b/lexer/fuzzer_corpus/59f33d4f075c2cf6448f94f623fac85834bdb2cf new file mode 100644 index 0000000000000..7a2a6ebe49de3 Binary files /dev/null and b/lexer/fuzzer_corpus/59f33d4f075c2cf6448f94f623fac85834bdb2cf differ diff --git a/lexer/fuzzer_corpus/59fe75149f202f3e7920a1fd87959628855929a8 b/lexer/fuzzer_corpus/59fe75149f202f3e7920a1fd87959628855929a8 new file mode 100644 index 0000000000000..67c9e852f9c8c Binary files /dev/null and b/lexer/fuzzer_corpus/59fe75149f202f3e7920a1fd87959628855929a8 differ diff --git a/lexer/fuzzer_corpus/5a02c95e0a7d591f275f196eb5274f1b2209e244 b/lexer/fuzzer_corpus/5a02c95e0a7d591f275f196eb5274f1b2209e244 new file mode 100644 index 0000000000000..741669d0c4b88 Binary files /dev/null and b/lexer/fuzzer_corpus/5a02c95e0a7d591f275f196eb5274f1b2209e244 differ diff --git a/lexer/fuzzer_corpus/5aa449aa73c5c05cf57a957914e47a4d2adbd4c9 b/lexer/fuzzer_corpus/5aa449aa73c5c05cf57a957914e47a4d2adbd4c9 new file mode 100644 index 0000000000000..7f4710010922a Binary files /dev/null and b/lexer/fuzzer_corpus/5aa449aa73c5c05cf57a957914e47a4d2adbd4c9 differ diff --git a/lexer/fuzzer_corpus/5adf5dd22c409ad57c9f6b29eeb92aaafdf5372c b/lexer/fuzzer_corpus/5adf5dd22c409ad57c9f6b29eeb92aaafdf5372c new file mode 100644 index 0000000000000..8c696e3acb57c Binary files /dev/null and b/lexer/fuzzer_corpus/5adf5dd22c409ad57c9f6b29eeb92aaafdf5372c differ diff --git a/lexer/fuzzer_corpus/5b6e66c2980a4e9eae54c9996bd413b58e564150 b/lexer/fuzzer_corpus/5b6e66c2980a4e9eae54c9996bd413b58e564150 new file mode 100644 index 0000000000000..1b28f379de43f Binary files /dev/null and b/lexer/fuzzer_corpus/5b6e66c2980a4e9eae54c9996bd413b58e564150 differ diff --git a/lexer/fuzzer_corpus/5bfebbdbbfb4f41a5d5d388504bd062d7a99c3e0 b/lexer/fuzzer_corpus/5bfebbdbbfb4f41a5d5d388504bd062d7a99c3e0 new file mode 100644 index 0000000000000..d853216ac41be Binary files /dev/null and b/lexer/fuzzer_corpus/5bfebbdbbfb4f41a5d5d388504bd062d7a99c3e0 differ diff --git a/lexer/fuzzer_corpus/5c3a3ba5513cfcdc49f76b2009d806141588f85f b/lexer/fuzzer_corpus/5c3a3ba5513cfcdc49f76b2009d806141588f85f new file mode 100644 index 0000000000000..87a08cfd5bd3c Binary files /dev/null and b/lexer/fuzzer_corpus/5c3a3ba5513cfcdc49f76b2009d806141588f85f differ diff --git a/lexer/fuzzer_corpus/5d31e2a814a923c713cd3533f1cf397c902d304e b/lexer/fuzzer_corpus/5d31e2a814a923c713cd3533f1cf397c902d304e new file mode 100644 index 0000000000000..6d31093f2386b Binary files /dev/null and b/lexer/fuzzer_corpus/5d31e2a814a923c713cd3533f1cf397c902d304e differ diff --git a/lexer/fuzzer_corpus/5d6c6fa498ecdef22875bdb7b8d212f146cc387b b/lexer/fuzzer_corpus/5d6c6fa498ecdef22875bdb7b8d212f146cc387b new file mode 100644 index 0000000000000..6170abfcb15ea Binary files /dev/null and b/lexer/fuzzer_corpus/5d6c6fa498ecdef22875bdb7b8d212f146cc387b differ diff --git a/lexer/fuzzer_corpus/5e05747916bcae6d1d14627260fa82de4caea0d7 b/lexer/fuzzer_corpus/5e05747916bcae6d1d14627260fa82de4caea0d7 new file mode 100644 index 0000000000000..909332b7db049 Binary files /dev/null and b/lexer/fuzzer_corpus/5e05747916bcae6d1d14627260fa82de4caea0d7 differ diff --git a/lexer/fuzzer_corpus/5e399a625d3bb3c6447dafe42e4553fc16950f6e b/lexer/fuzzer_corpus/5e399a625d3bb3c6447dafe42e4553fc16950f6e new file mode 100644 index 0000000000000..50dbfb27eae05 Binary files /dev/null and b/lexer/fuzzer_corpus/5e399a625d3bb3c6447dafe42e4553fc16950f6e differ diff --git a/lexer/fuzzer_corpus/5eb107e4dc5d78891aac3f0c1e83883b7a663fdf b/lexer/fuzzer_corpus/5eb107e4dc5d78891aac3f0c1e83883b7a663fdf new file mode 100644 index 0000000000000..9b706d2389125 Binary files /dev/null and b/lexer/fuzzer_corpus/5eb107e4dc5d78891aac3f0c1e83883b7a663fdf differ diff --git a/lexer/fuzzer_corpus/6015eb2c2f9da9ac11698c60c65c0bb094a66657 b/lexer/fuzzer_corpus/6015eb2c2f9da9ac11698c60c65c0bb094a66657 new file mode 100644 index 0000000000000..753fe958e8787 Binary files /dev/null and b/lexer/fuzzer_corpus/6015eb2c2f9da9ac11698c60c65c0bb094a66657 differ diff --git a/lexer/fuzzer_corpus/603588985367299d4b4e56c81186715fc8ab5298 b/lexer/fuzzer_corpus/603588985367299d4b4e56c81186715fc8ab5298 new file mode 100644 index 0000000000000..9627254292c36 Binary files /dev/null and b/lexer/fuzzer_corpus/603588985367299d4b4e56c81186715fc8ab5298 differ diff --git a/lexer/fuzzer_corpus/60603d960234eaa4cf2748d2f8b43e1303171ec7 b/lexer/fuzzer_corpus/60603d960234eaa4cf2748d2f8b43e1303171ec7 new file mode 100644 index 0000000000000..3be4dcec5e05e Binary files /dev/null and b/lexer/fuzzer_corpus/60603d960234eaa4cf2748d2f8b43e1303171ec7 differ diff --git a/lexer/fuzzer_corpus/6235a1b078cf86f65be51d3e4d952688c0cf4027 b/lexer/fuzzer_corpus/6235a1b078cf86f65be51d3e4d952688c0cf4027 new file mode 100644 index 0000000000000..69f7a4e5bbe5e Binary files /dev/null and b/lexer/fuzzer_corpus/6235a1b078cf86f65be51d3e4d952688c0cf4027 differ diff --git a/lexer/fuzzer_corpus/62618378c0f30271418e65d9642b681cf5445c1e b/lexer/fuzzer_corpus/62618378c0f30271418e65d9642b681cf5445c1e new file mode 100644 index 0000000000000..5513111268e66 Binary files /dev/null and b/lexer/fuzzer_corpus/62618378c0f30271418e65d9642b681cf5445c1e differ diff --git a/lexer/fuzzer_corpus/62d8ba9122d4a19be633439c04f949e0683f417d b/lexer/fuzzer_corpus/62d8ba9122d4a19be633439c04f949e0683f417d new file mode 100644 index 0000000000000..0cd8debd4cf0b Binary files /dev/null and b/lexer/fuzzer_corpus/62d8ba9122d4a19be633439c04f949e0683f417d differ diff --git a/lexer/fuzzer_corpus/63fbf21dc690999b09177bf4ac4216676ed26b6e b/lexer/fuzzer_corpus/63fbf21dc690999b09177bf4ac4216676ed26b6e new file mode 100644 index 0000000000000..542475849b7b7 Binary files /dev/null and b/lexer/fuzzer_corpus/63fbf21dc690999b09177bf4ac4216676ed26b6e differ diff --git a/lexer/fuzzer_corpus/64ea37e683f8f7ad3eb0a9c4ad19d82cfbdc9774 b/lexer/fuzzer_corpus/64ea37e683f8f7ad3eb0a9c4ad19d82cfbdc9774 new file mode 100644 index 0000000000000..edd37b331ec6c Binary files /dev/null and b/lexer/fuzzer_corpus/64ea37e683f8f7ad3eb0a9c4ad19d82cfbdc9774 differ diff --git a/lexer/fuzzer_corpus/658886802343686540cb41c5499a46f1705b9d5c b/lexer/fuzzer_corpus/658886802343686540cb41c5499a46f1705b9d5c new file mode 100644 index 0000000000000..784c6962ba67d Binary files /dev/null and b/lexer/fuzzer_corpus/658886802343686540cb41c5499a46f1705b9d5c differ diff --git a/lexer/fuzzer_corpus/6614bca5723ef48592ebb2efee5bc73e17760aee b/lexer/fuzzer_corpus/6614bca5723ef48592ebb2efee5bc73e17760aee new file mode 100644 index 0000000000000..965b3ff139b9d Binary files /dev/null and b/lexer/fuzzer_corpus/6614bca5723ef48592ebb2efee5bc73e17760aee differ diff --git a/lexer/fuzzer_corpus/664bd1be9730de5ce106f7b2d87774e6549c6ada b/lexer/fuzzer_corpus/664bd1be9730de5ce106f7b2d87774e6549c6ada new file mode 100644 index 0000000000000..a78dd53639b9b Binary files /dev/null and b/lexer/fuzzer_corpus/664bd1be9730de5ce106f7b2d87774e6549c6ada differ diff --git a/lexer/fuzzer_corpus/66a6d4bff6fa7a62ec7a16d9bf5adf706efea00f b/lexer/fuzzer_corpus/66a6d4bff6fa7a62ec7a16d9bf5adf706efea00f new file mode 100644 index 0000000000000..328d09ee0ad82 Binary files /dev/null and b/lexer/fuzzer_corpus/66a6d4bff6fa7a62ec7a16d9bf5adf706efea00f differ diff --git a/lexer/fuzzer_corpus/66b622ca15899fc16af4040369d852faf77fd604 b/lexer/fuzzer_corpus/66b622ca15899fc16af4040369d852faf77fd604 new file mode 100644 index 0000000000000..fa070e0448fcf Binary files /dev/null and b/lexer/fuzzer_corpus/66b622ca15899fc16af4040369d852faf77fd604 differ diff --git a/lexer/fuzzer_corpus/66cee929f075d88bfa46887e565b5d3d148f9154 b/lexer/fuzzer_corpus/66cee929f075d88bfa46887e565b5d3d148f9154 new file mode 100644 index 0000000000000..a40352b242f81 Binary files /dev/null and b/lexer/fuzzer_corpus/66cee929f075d88bfa46887e565b5d3d148f9154 differ diff --git a/lexer/fuzzer_corpus/6a8a9d4ec0485d6a585db4b56ecfc8779e7888fd b/lexer/fuzzer_corpus/6a8a9d4ec0485d6a585db4b56ecfc8779e7888fd new file mode 100644 index 0000000000000..48cf0718e02e3 Binary files /dev/null and b/lexer/fuzzer_corpus/6a8a9d4ec0485d6a585db4b56ecfc8779e7888fd differ diff --git a/lexer/fuzzer_corpus/6af377722221a297079f87c07a93c2189ac24fb1 b/lexer/fuzzer_corpus/6af377722221a297079f87c07a93c2189ac24fb1 new file mode 100644 index 0000000000000..5ae871261b71d Binary files /dev/null and b/lexer/fuzzer_corpus/6af377722221a297079f87c07a93c2189ac24fb1 differ diff --git a/lexer/fuzzer_corpus/6b557d729ebff7f4da9b746da200980115618627 b/lexer/fuzzer_corpus/6b557d729ebff7f4da9b746da200980115618627 new file mode 100644 index 0000000000000..3a4ba687c7c71 Binary files /dev/null and b/lexer/fuzzer_corpus/6b557d729ebff7f4da9b746da200980115618627 differ diff --git a/lexer/fuzzer_corpus/6c4fb24e430367960280c066b72fb107b1c424ab b/lexer/fuzzer_corpus/6c4fb24e430367960280c066b72fb107b1c424ab new file mode 100644 index 0000000000000..2e555db1ed51f Binary files /dev/null and b/lexer/fuzzer_corpus/6c4fb24e430367960280c066b72fb107b1c424ab differ diff --git a/lexer/fuzzer_corpus/6c6e255ee96eef814112752fe189fae0dfe78743 b/lexer/fuzzer_corpus/6c6e255ee96eef814112752fe189fae0dfe78743 new file mode 100644 index 0000000000000..83150082fff7d Binary files /dev/null and b/lexer/fuzzer_corpus/6c6e255ee96eef814112752fe189fae0dfe78743 differ diff --git a/lexer/fuzzer_corpus/6d44e5997f87b024c53eab22f5912e1988cb66f3 b/lexer/fuzzer_corpus/6d44e5997f87b024c53eab22f5912e1988cb66f3 new file mode 100644 index 0000000000000..ae7cddb5beab1 Binary files /dev/null and b/lexer/fuzzer_corpus/6d44e5997f87b024c53eab22f5912e1988cb66f3 differ diff --git a/lexer/fuzzer_corpus/6d512f956c75ca835d38cbcb6e26e1a2473abd08 b/lexer/fuzzer_corpus/6d512f956c75ca835d38cbcb6e26e1a2473abd08 new file mode 100644 index 0000000000000..154072f31b942 Binary files /dev/null and b/lexer/fuzzer_corpus/6d512f956c75ca835d38cbcb6e26e1a2473abd08 differ diff --git a/lexer/fuzzer_corpus/6d72ae632ce79e9c197f5ca799f667d61d2a3df8 b/lexer/fuzzer_corpus/6d72ae632ce79e9c197f5ca799f667d61d2a3df8 new file mode 100644 index 0000000000000..3bdc06efb32dd Binary files /dev/null and b/lexer/fuzzer_corpus/6d72ae632ce79e9c197f5ca799f667d61d2a3df8 differ diff --git a/lexer/fuzzer_corpus/6e62c2d775c1c825125cab26e5a4b004a06a4ba7 b/lexer/fuzzer_corpus/6e62c2d775c1c825125cab26e5a4b004a06a4ba7 new file mode 100644 index 0000000000000..4257f192306b9 Binary files /dev/null and b/lexer/fuzzer_corpus/6e62c2d775c1c825125cab26e5a4b004a06a4ba7 differ diff --git a/lexer/fuzzer_corpus/6fcdcf3781990fdceb9b08ba304283f1fd8f9b4e b/lexer/fuzzer_corpus/6fcdcf3781990fdceb9b08ba304283f1fd8f9b4e new file mode 100644 index 0000000000000..2041fbe283a77 Binary files /dev/null and b/lexer/fuzzer_corpus/6fcdcf3781990fdceb9b08ba304283f1fd8f9b4e differ diff --git a/lexer/fuzzer_corpus/7016224c7c5f50c8e8aeb6e43851f5e958aa36da b/lexer/fuzzer_corpus/7016224c7c5f50c8e8aeb6e43851f5e958aa36da new file mode 100644 index 0000000000000..5cb74a104c8fe Binary files /dev/null and b/lexer/fuzzer_corpus/7016224c7c5f50c8e8aeb6e43851f5e958aa36da differ diff --git a/lexer/fuzzer_corpus/709dcef4bfe4c6162c6a983a3d12eb20ac5cbed7 b/lexer/fuzzer_corpus/709dcef4bfe4c6162c6a983a3d12eb20ac5cbed7 new file mode 100644 index 0000000000000..dbfa25113e1e1 Binary files /dev/null and b/lexer/fuzzer_corpus/709dcef4bfe4c6162c6a983a3d12eb20ac5cbed7 differ diff --git a/lexer/fuzzer_corpus/70aed57684596e5887c493a5a79dc1d6bcb5eb43 b/lexer/fuzzer_corpus/70aed57684596e5887c493a5a79dc1d6bcb5eb43 new file mode 100644 index 0000000000000..7e82bb564feea Binary files /dev/null and b/lexer/fuzzer_corpus/70aed57684596e5887c493a5a79dc1d6bcb5eb43 differ diff --git a/lexer/fuzzer_corpus/716d1769833e864f111085008d8322d53a4daf30 b/lexer/fuzzer_corpus/716d1769833e864f111085008d8322d53a4daf30 new file mode 100644 index 0000000000000..a00c30c4659e8 Binary files /dev/null and b/lexer/fuzzer_corpus/716d1769833e864f111085008d8322d53a4daf30 differ diff --git a/lexer/fuzzer_corpus/7194db810c5947f300d6035d2a4a743090210e21 b/lexer/fuzzer_corpus/7194db810c5947f300d6035d2a4a743090210e21 new file mode 100644 index 0000000000000..b4664d08e64a1 Binary files /dev/null and b/lexer/fuzzer_corpus/7194db810c5947f300d6035d2a4a743090210e21 differ diff --git a/lexer/fuzzer_corpus/71a5590952fd84047c6d6e9be45bf1f045355625 b/lexer/fuzzer_corpus/71a5590952fd84047c6d6e9be45bf1f045355625 new file mode 100644 index 0000000000000..ad9a166b1a8a2 Binary files /dev/null and b/lexer/fuzzer_corpus/71a5590952fd84047c6d6e9be45bf1f045355625 differ diff --git a/lexer/fuzzer_corpus/73a854c8805585f984236fa44232cb965b9083ae b/lexer/fuzzer_corpus/73a854c8805585f984236fa44232cb965b9083ae new file mode 100644 index 0000000000000..34f7cc23220aa Binary files /dev/null and b/lexer/fuzzer_corpus/73a854c8805585f984236fa44232cb965b9083ae differ diff --git a/lexer/fuzzer_corpus/74bd0a894550c238cee609bb561591118137bc37 b/lexer/fuzzer_corpus/74bd0a894550c238cee609bb561591118137bc37 new file mode 100644 index 0000000000000..c23248162c569 Binary files /dev/null and b/lexer/fuzzer_corpus/74bd0a894550c238cee609bb561591118137bc37 differ diff --git a/lexer/fuzzer_corpus/75977e3e56167889e781b65443e91eb49902cfbf b/lexer/fuzzer_corpus/75977e3e56167889e781b65443e91eb49902cfbf new file mode 100644 index 0000000000000..8bee33e2382c2 Binary files /dev/null and b/lexer/fuzzer_corpus/75977e3e56167889e781b65443e91eb49902cfbf differ diff --git a/lexer/fuzzer_corpus/767654d5cbd98934fc45a7ba38f606c4efe7e359 b/lexer/fuzzer_corpus/767654d5cbd98934fc45a7ba38f606c4efe7e359 new file mode 100644 index 0000000000000..15942afbe6329 Binary files /dev/null and b/lexer/fuzzer_corpus/767654d5cbd98934fc45a7ba38f606c4efe7e359 differ diff --git a/lexer/fuzzer_corpus/7737f7fabd55d3fff123dbc65fee826311c2580d b/lexer/fuzzer_corpus/7737f7fabd55d3fff123dbc65fee826311c2580d new file mode 100644 index 0000000000000..ad7fc57bae943 Binary files /dev/null and b/lexer/fuzzer_corpus/7737f7fabd55d3fff123dbc65fee826311c2580d differ diff --git a/lexer/fuzzer_corpus/77453d8dc39b44fdf3c80a0ca80c7ead9149de84 b/lexer/fuzzer_corpus/77453d8dc39b44fdf3c80a0ca80c7ead9149de84 new file mode 100644 index 0000000000000..9c16509e66fa3 Binary files /dev/null and b/lexer/fuzzer_corpus/77453d8dc39b44fdf3c80a0ca80c7ead9149de84 differ diff --git a/lexer/fuzzer_corpus/788e547a1eb3cab3960e638885457a0fe112904c b/lexer/fuzzer_corpus/788e547a1eb3cab3960e638885457a0fe112904c new file mode 100644 index 0000000000000..577881667dde3 Binary files /dev/null and b/lexer/fuzzer_corpus/788e547a1eb3cab3960e638885457a0fe112904c differ diff --git a/lexer/fuzzer_corpus/7897d38321a88305b14831654cdb32ba40665cfd b/lexer/fuzzer_corpus/7897d38321a88305b14831654cdb32ba40665cfd new file mode 100644 index 0000000000000..01a43576b61ac Binary files /dev/null and b/lexer/fuzzer_corpus/7897d38321a88305b14831654cdb32ba40665cfd differ diff --git a/lexer/fuzzer_corpus/799ee6b016fe616eec6ee67f1977744aee58aee0 b/lexer/fuzzer_corpus/799ee6b016fe616eec6ee67f1977744aee58aee0 new file mode 100644 index 0000000000000..9f47fa51051b5 Binary files /dev/null and b/lexer/fuzzer_corpus/799ee6b016fe616eec6ee67f1977744aee58aee0 differ diff --git a/lexer/fuzzer_corpus/7a0cf48bba4dafb8f490124a89977a93c73988b4 b/lexer/fuzzer_corpus/7a0cf48bba4dafb8f490124a89977a93c73988b4 new file mode 100644 index 0000000000000..f20f4a2790ccc Binary files /dev/null and b/lexer/fuzzer_corpus/7a0cf48bba4dafb8f490124a89977a93c73988b4 differ diff --git a/lexer/fuzzer_corpus/7a26e6bcc857b6e9b797b3c918f2bcb1a569ddde b/lexer/fuzzer_corpus/7a26e6bcc857b6e9b797b3c918f2bcb1a569ddde new file mode 100644 index 0000000000000..d5dc2123bc234 Binary files /dev/null and b/lexer/fuzzer_corpus/7a26e6bcc857b6e9b797b3c918f2bcb1a569ddde differ diff --git a/lexer/fuzzer_corpus/7a34b6c09a1ec38856874d086cf899709fa20650 b/lexer/fuzzer_corpus/7a34b6c09a1ec38856874d086cf899709fa20650 new file mode 100644 index 0000000000000..9bae760676a36 Binary files /dev/null and b/lexer/fuzzer_corpus/7a34b6c09a1ec38856874d086cf899709fa20650 differ diff --git a/lexer/fuzzer_corpus/7ae9e827c496473c836ad4de36f8b029cbb005c8 b/lexer/fuzzer_corpus/7ae9e827c496473c836ad4de36f8b029cbb005c8 new file mode 100644 index 0000000000000..3c6c770c60433 Binary files /dev/null and b/lexer/fuzzer_corpus/7ae9e827c496473c836ad4de36f8b029cbb005c8 differ diff --git a/lexer/fuzzer_corpus/7bc478ddfb9efcd7b4650c00e27da372d2cbee86 b/lexer/fuzzer_corpus/7bc478ddfb9efcd7b4650c00e27da372d2cbee86 new file mode 100644 index 0000000000000..7dfb796d8eee2 Binary files /dev/null and b/lexer/fuzzer_corpus/7bc478ddfb9efcd7b4650c00e27da372d2cbee86 differ diff --git a/lexer/fuzzer_corpus/7c9291251d2b1628c0bf6b31e2074445408df3e9 b/lexer/fuzzer_corpus/7c9291251d2b1628c0bf6b31e2074445408df3e9 new file mode 100644 index 0000000000000..e778df0a8a9dc Binary files /dev/null and b/lexer/fuzzer_corpus/7c9291251d2b1628c0bf6b31e2074445408df3e9 differ diff --git a/lexer/fuzzer_corpus/7d7bcfb836b397d537c2b3649c86e9a9c48303e0 b/lexer/fuzzer_corpus/7d7bcfb836b397d537c2b3649c86e9a9c48303e0 new file mode 100644 index 0000000000000..221a8ade07dc7 Binary files /dev/null and b/lexer/fuzzer_corpus/7d7bcfb836b397d537c2b3649c86e9a9c48303e0 differ diff --git a/lexer/fuzzer_corpus/7dcb764405ae26b71cb127b30336b45fd802640a b/lexer/fuzzer_corpus/7dcb764405ae26b71cb127b30336b45fd802640a new file mode 100644 index 0000000000000..078e1a4c49e3b Binary files /dev/null and b/lexer/fuzzer_corpus/7dcb764405ae26b71cb127b30336b45fd802640a differ diff --git a/lexer/fuzzer_corpus/7dde4f40a01dfa3b917b86b8bd035929132f32db b/lexer/fuzzer_corpus/7dde4f40a01dfa3b917b86b8bd035929132f32db new file mode 100644 index 0000000000000..18368599248f6 Binary files /dev/null and b/lexer/fuzzer_corpus/7dde4f40a01dfa3b917b86b8bd035929132f32db differ diff --git a/lexer/fuzzer_corpus/7e01c8cdc6b1904415e575240588c4759047692d b/lexer/fuzzer_corpus/7e01c8cdc6b1904415e575240588c4759047692d new file mode 100644 index 0000000000000..d40e9e60a4b6c Binary files /dev/null and b/lexer/fuzzer_corpus/7e01c8cdc6b1904415e575240588c4759047692d differ diff --git a/lexer/fuzzer_corpus/7e10b31e7041e9432b39bb1ae327355d96dfd62c b/lexer/fuzzer_corpus/7e10b31e7041e9432b39bb1ae327355d96dfd62c new file mode 100644 index 0000000000000..70ad36c5b52ae Binary files /dev/null and b/lexer/fuzzer_corpus/7e10b31e7041e9432b39bb1ae327355d96dfd62c differ diff --git a/lexer/fuzzer_corpus/7e479935227c90be28bb8a6261fe2141a12821e5 b/lexer/fuzzer_corpus/7e479935227c90be28bb8a6261fe2141a12821e5 new file mode 100644 index 0000000000000..7c999410d6955 Binary files /dev/null and b/lexer/fuzzer_corpus/7e479935227c90be28bb8a6261fe2141a12821e5 differ diff --git a/lexer/fuzzer_corpus/7e6775e595a452e61552a68facb7aa884fc579e5 b/lexer/fuzzer_corpus/7e6775e595a452e61552a68facb7aa884fc579e5 new file mode 100644 index 0000000000000..47cc44b30bcb2 Binary files /dev/null and b/lexer/fuzzer_corpus/7e6775e595a452e61552a68facb7aa884fc579e5 differ diff --git a/lexer/fuzzer_corpus/7fbd42707b95e15ea0584236b080b01a86041f0a b/lexer/fuzzer_corpus/7fbd42707b95e15ea0584236b080b01a86041f0a new file mode 100644 index 0000000000000..5ddd9dbc19186 Binary files /dev/null and b/lexer/fuzzer_corpus/7fbd42707b95e15ea0584236b080b01a86041f0a differ diff --git a/lexer/fuzzer_corpus/80347f920096695efda11b33e2c2d06024c0c82a b/lexer/fuzzer_corpus/80347f920096695efda11b33e2c2d06024c0c82a new file mode 100644 index 0000000000000..68611b3f094bd Binary files /dev/null and b/lexer/fuzzer_corpus/80347f920096695efda11b33e2c2d06024c0c82a differ diff --git a/lexer/fuzzer_corpus/808863bfb504bd1a7ba89b4f8eb8c118a9125798 b/lexer/fuzzer_corpus/808863bfb504bd1a7ba89b4f8eb8c118a9125798 new file mode 100644 index 0000000000000..bb24de5ff9452 Binary files /dev/null and b/lexer/fuzzer_corpus/808863bfb504bd1a7ba89b4f8eb8c118a9125798 differ diff --git a/lexer/fuzzer_corpus/80a0247ebe3fd2aceff058d956c1c921ed7681a4 b/lexer/fuzzer_corpus/80a0247ebe3fd2aceff058d956c1c921ed7681a4 new file mode 100644 index 0000000000000..0b271626bd432 Binary files /dev/null and b/lexer/fuzzer_corpus/80a0247ebe3fd2aceff058d956c1c921ed7681a4 differ diff --git a/lexer/fuzzer_corpus/80d94c313466452d7b50fd144838a527809f2c7f b/lexer/fuzzer_corpus/80d94c313466452d7b50fd144838a527809f2c7f new file mode 100644 index 0000000000000..93dbae56b9b3b Binary files /dev/null and b/lexer/fuzzer_corpus/80d94c313466452d7b50fd144838a527809f2c7f differ diff --git a/lexer/fuzzer_corpus/80f348579a9a803a72d4a4cccd54824488b55aac b/lexer/fuzzer_corpus/80f348579a9a803a72d4a4cccd54824488b55aac new file mode 100644 index 0000000000000..ab68b19c87586 Binary files /dev/null and b/lexer/fuzzer_corpus/80f348579a9a803a72d4a4cccd54824488b55aac differ diff --git a/lexer/fuzzer_corpus/8211bb03478a5a26c1fb40236780cbee8da45a06 b/lexer/fuzzer_corpus/8211bb03478a5a26c1fb40236780cbee8da45a06 new file mode 100644 index 0000000000000..0dc1ad07f66fc Binary files /dev/null and b/lexer/fuzzer_corpus/8211bb03478a5a26c1fb40236780cbee8da45a06 differ diff --git a/lexer/fuzzer_corpus/833ca5a7d4cdf2c1867491c242ece799a01d77a9 b/lexer/fuzzer_corpus/833ca5a7d4cdf2c1867491c242ece799a01d77a9 new file mode 100644 index 0000000000000..eaa3cceb277fb Binary files /dev/null and b/lexer/fuzzer_corpus/833ca5a7d4cdf2c1867491c242ece799a01d77a9 differ diff --git a/lexer/fuzzer_corpus/83a58c921e3f326f742c80f8b885c6c6dcc68803 b/lexer/fuzzer_corpus/83a58c921e3f326f742c80f8b885c6c6dcc68803 new file mode 100644 index 0000000000000..354c17fb67508 Binary files /dev/null and b/lexer/fuzzer_corpus/83a58c921e3f326f742c80f8b885c6c6dcc68803 differ diff --git a/lexer/fuzzer_corpus/83a9bf595bc7dea203b912007f70743b88f7af95 b/lexer/fuzzer_corpus/83a9bf595bc7dea203b912007f70743b88f7af95 new file mode 100644 index 0000000000000..e5333f8d20346 Binary files /dev/null and b/lexer/fuzzer_corpus/83a9bf595bc7dea203b912007f70743b88f7af95 differ diff --git a/lexer/fuzzer_corpus/83cd63c7765c3f1d6f7f4f1f9b032f8efd46b3ef b/lexer/fuzzer_corpus/83cd63c7765c3f1d6f7f4f1f9b032f8efd46b3ef new file mode 100644 index 0000000000000..4c3ec98e53d34 Binary files /dev/null and b/lexer/fuzzer_corpus/83cd63c7765c3f1d6f7f4f1f9b032f8efd46b3ef differ diff --git a/lexer/fuzzer_corpus/8476e6af7e75aa144bad68b33b499cdac5da2bf4 b/lexer/fuzzer_corpus/8476e6af7e75aa144bad68b33b499cdac5da2bf4 new file mode 100644 index 0000000000000..8ae722284a7df Binary files /dev/null and b/lexer/fuzzer_corpus/8476e6af7e75aa144bad68b33b499cdac5da2bf4 differ diff --git a/lexer/fuzzer_corpus/8477039b36c6e7e796ce4036aa78d32e00ce0103 b/lexer/fuzzer_corpus/8477039b36c6e7e796ce4036aa78d32e00ce0103 new file mode 100644 index 0000000000000..6bc49c15ac80d Binary files /dev/null and b/lexer/fuzzer_corpus/8477039b36c6e7e796ce4036aa78d32e00ce0103 differ diff --git a/lexer/fuzzer_corpus/85232cbda473f24e2195b75d84e387f25956a358 b/lexer/fuzzer_corpus/85232cbda473f24e2195b75d84e387f25956a358 new file mode 100644 index 0000000000000..2815cd671c2fd Binary files /dev/null and b/lexer/fuzzer_corpus/85232cbda473f24e2195b75d84e387f25956a358 differ diff --git a/lexer/fuzzer_corpus/860f19159238baa82dd6680e5f00cd818d869fca b/lexer/fuzzer_corpus/860f19159238baa82dd6680e5f00cd818d869fca new file mode 100644 index 0000000000000..45467dd05236c Binary files /dev/null and b/lexer/fuzzer_corpus/860f19159238baa82dd6680e5f00cd818d869fca differ diff --git a/lexer/fuzzer_corpus/86c5f1d7668cd16af68c6901ef37d10a929b937e b/lexer/fuzzer_corpus/86c5f1d7668cd16af68c6901ef37d10a929b937e new file mode 100644 index 0000000000000..ffe20be9e0055 Binary files /dev/null and b/lexer/fuzzer_corpus/86c5f1d7668cd16af68c6901ef37d10a929b937e differ diff --git a/lexer/fuzzer_corpus/875d260c6259b312bf7adc25aa6d37de04cacef9 b/lexer/fuzzer_corpus/875d260c6259b312bf7adc25aa6d37de04cacef9 new file mode 100644 index 0000000000000..ba5c272a78bfc Binary files /dev/null and b/lexer/fuzzer_corpus/875d260c6259b312bf7adc25aa6d37de04cacef9 differ diff --git a/lexer/fuzzer_corpus/87870c833802ae370b72ded06d6b5466283e5189 b/lexer/fuzzer_corpus/87870c833802ae370b72ded06d6b5466283e5189 new file mode 100644 index 0000000000000..6bd99a4c5d72c Binary files /dev/null and b/lexer/fuzzer_corpus/87870c833802ae370b72ded06d6b5466283e5189 differ diff --git a/lexer/fuzzer_corpus/882f2aaac218ec0d84736ac2c55910a4594a1eae b/lexer/fuzzer_corpus/882f2aaac218ec0d84736ac2c55910a4594a1eae new file mode 100644 index 0000000000000..c5eef8ad446e6 Binary files /dev/null and b/lexer/fuzzer_corpus/882f2aaac218ec0d84736ac2c55910a4594a1eae differ diff --git a/lexer/fuzzer_corpus/8840b9e11ea5d3aea427c815ebfe1896dc3b765e b/lexer/fuzzer_corpus/8840b9e11ea5d3aea427c815ebfe1896dc3b765e new file mode 100644 index 0000000000000..abf11fc154652 Binary files /dev/null and b/lexer/fuzzer_corpus/8840b9e11ea5d3aea427c815ebfe1896dc3b765e differ diff --git a/lexer/fuzzer_corpus/8873098a4246f81ee6e5576070d0a7244b3ac9f3 b/lexer/fuzzer_corpus/8873098a4246f81ee6e5576070d0a7244b3ac9f3 new file mode 100644 index 0000000000000..9f97f469d7899 Binary files /dev/null and b/lexer/fuzzer_corpus/8873098a4246f81ee6e5576070d0a7244b3ac9f3 differ diff --git a/lexer/fuzzer_corpus/8904890ebc3c9798fabda724359e14379698ebd4 b/lexer/fuzzer_corpus/8904890ebc3c9798fabda724359e14379698ebd4 new file mode 100644 index 0000000000000..d2288b53905da Binary files /dev/null and b/lexer/fuzzer_corpus/8904890ebc3c9798fabda724359e14379698ebd4 differ diff --git a/lexer/fuzzer_corpus/891f275cd08dde2776d7140e46cb047faf9f0f8d b/lexer/fuzzer_corpus/891f275cd08dde2776d7140e46cb047faf9f0f8d new file mode 100644 index 0000000000000..9cc2c48fa8591 Binary files /dev/null and b/lexer/fuzzer_corpus/891f275cd08dde2776d7140e46cb047faf9f0f8d differ diff --git a/lexer/fuzzer_corpus/89371ce91fe1a6d01ba115e5bf54bf577340b8eb b/lexer/fuzzer_corpus/89371ce91fe1a6d01ba115e5bf54bf577340b8eb new file mode 100644 index 0000000000000..7fa555015fabf Binary files /dev/null and b/lexer/fuzzer_corpus/89371ce91fe1a6d01ba115e5bf54bf577340b8eb differ diff --git a/lexer/fuzzer_corpus/895e8fd679feb6a39068aba3d31fb206089a064b b/lexer/fuzzer_corpus/895e8fd679feb6a39068aba3d31fb206089a064b new file mode 100644 index 0000000000000..b04294a69bbb5 Binary files /dev/null and b/lexer/fuzzer_corpus/895e8fd679feb6a39068aba3d31fb206089a064b differ diff --git a/lexer/fuzzer_corpus/8a00c440fa5a60b20d2c8085a38baeaaf3c5670c b/lexer/fuzzer_corpus/8a00c440fa5a60b20d2c8085a38baeaaf3c5670c new file mode 100644 index 0000000000000..9c8b16b04a18e Binary files /dev/null and b/lexer/fuzzer_corpus/8a00c440fa5a60b20d2c8085a38baeaaf3c5670c differ diff --git a/lexer/fuzzer_corpus/8a88097f625c33cc9cbe29343feb52ab248b5c1b b/lexer/fuzzer_corpus/8a88097f625c33cc9cbe29343feb52ab248b5c1b new file mode 100644 index 0000000000000..4dbc9d564eeb5 Binary files /dev/null and b/lexer/fuzzer_corpus/8a88097f625c33cc9cbe29343feb52ab248b5c1b differ diff --git a/lexer/fuzzer_corpus/8b0a819cd3fc4d68cf3e0e0855f7ebcd89a20317 b/lexer/fuzzer_corpus/8b0a819cd3fc4d68cf3e0e0855f7ebcd89a20317 new file mode 100644 index 0000000000000..cdcc4ecba00df Binary files /dev/null and b/lexer/fuzzer_corpus/8b0a819cd3fc4d68cf3e0e0855f7ebcd89a20317 differ diff --git a/lexer/fuzzer_corpus/8b98620ebbf7bab27f5ca3fc0ec663cc58c76db9 b/lexer/fuzzer_corpus/8b98620ebbf7bab27f5ca3fc0ec663cc58c76db9 new file mode 100644 index 0000000000000..7e1d793a6fb19 Binary files /dev/null and b/lexer/fuzzer_corpus/8b98620ebbf7bab27f5ca3fc0ec663cc58c76db9 differ diff --git a/lexer/fuzzer_corpus/8c623105d5f0f8ac252343c858286d65b992687c b/lexer/fuzzer_corpus/8c623105d5f0f8ac252343c858286d65b992687c new file mode 100644 index 0000000000000..20bb277dba010 Binary files /dev/null and b/lexer/fuzzer_corpus/8c623105d5f0f8ac252343c858286d65b992687c differ diff --git a/lexer/fuzzer_corpus/8c74dc0dc68a7e61878314355623c887ace5eefe b/lexer/fuzzer_corpus/8c74dc0dc68a7e61878314355623c887ace5eefe new file mode 100644 index 0000000000000..d30f58dbcbde3 Binary files /dev/null and b/lexer/fuzzer_corpus/8c74dc0dc68a7e61878314355623c887ace5eefe differ diff --git a/lexer/fuzzer_corpus/8d19aefc358ee1284e3a594474f26015586dd9e9 b/lexer/fuzzer_corpus/8d19aefc358ee1284e3a594474f26015586dd9e9 new file mode 100644 index 0000000000000..b857b0e345f79 Binary files /dev/null and b/lexer/fuzzer_corpus/8d19aefc358ee1284e3a594474f26015586dd9e9 differ diff --git a/lexer/fuzzer_corpus/8dc12bb4d6f47bdce0349d159084f7a90ab432a4 b/lexer/fuzzer_corpus/8dc12bb4d6f47bdce0349d159084f7a90ab432a4 new file mode 100644 index 0000000000000..d3e0190fb425b Binary files /dev/null and b/lexer/fuzzer_corpus/8dc12bb4d6f47bdce0349d159084f7a90ab432a4 differ diff --git a/lexer/fuzzer_corpus/8df1c55d2c899a91d4de757ad55abeaee00718e5 b/lexer/fuzzer_corpus/8df1c55d2c899a91d4de757ad55abeaee00718e5 new file mode 100644 index 0000000000000..2b3aed318de75 Binary files /dev/null and b/lexer/fuzzer_corpus/8df1c55d2c899a91d4de757ad55abeaee00718e5 differ diff --git a/lexer/fuzzer_corpus/8ea0ce1a9c4c228095a13e76e7c43657f8bd6e23 b/lexer/fuzzer_corpus/8ea0ce1a9c4c228095a13e76e7c43657f8bd6e23 new file mode 100644 index 0000000000000..75d02bf95f419 Binary files /dev/null and b/lexer/fuzzer_corpus/8ea0ce1a9c4c228095a13e76e7c43657f8bd6e23 differ diff --git a/lexer/fuzzer_corpus/8eeef9f346f4ffdc8e3e6456301cf775d164c42d b/lexer/fuzzer_corpus/8eeef9f346f4ffdc8e3e6456301cf775d164c42d new file mode 100644 index 0000000000000..5d00130e63085 Binary files /dev/null and b/lexer/fuzzer_corpus/8eeef9f346f4ffdc8e3e6456301cf775d164c42d differ diff --git a/lexer/fuzzer_corpus/8f1f1bdd61e03a0432e1425982bc95d4eed1dc8f b/lexer/fuzzer_corpus/8f1f1bdd61e03a0432e1425982bc95d4eed1dc8f new file mode 100644 index 0000000000000..e17fed8c926e6 Binary files /dev/null and b/lexer/fuzzer_corpus/8f1f1bdd61e03a0432e1425982bc95d4eed1dc8f differ diff --git a/lexer/fuzzer_corpus/8fc80309676fc4e9b457cba583e09623e3a50e8b b/lexer/fuzzer_corpus/8fc80309676fc4e9b457cba583e09623e3a50e8b new file mode 100644 index 0000000000000..b89e92b0fa4d3 Binary files /dev/null and b/lexer/fuzzer_corpus/8fc80309676fc4e9b457cba583e09623e3a50e8b differ diff --git a/lexer/fuzzer_corpus/9032685281013329ab00123038a24e88ee87e99f b/lexer/fuzzer_corpus/9032685281013329ab00123038a24e88ee87e99f new file mode 100644 index 0000000000000..8284de965c268 Binary files /dev/null and b/lexer/fuzzer_corpus/9032685281013329ab00123038a24e88ee87e99f differ diff --git a/lexer/fuzzer_corpus/903b7c38e142f05daeefa445f48a7da5630e1efe b/lexer/fuzzer_corpus/903b7c38e142f05daeefa445f48a7da5630e1efe new file mode 100644 index 0000000000000..4f61f0925f82c Binary files /dev/null and b/lexer/fuzzer_corpus/903b7c38e142f05daeefa445f48a7da5630e1efe differ diff --git a/lexer/fuzzer_corpus/90fb17406d6d1e431223bda418dc24452cd47cbd b/lexer/fuzzer_corpus/90fb17406d6d1e431223bda418dc24452cd47cbd new file mode 100644 index 0000000000000..52247c315a576 Binary files /dev/null and b/lexer/fuzzer_corpus/90fb17406d6d1e431223bda418dc24452cd47cbd differ diff --git a/lexer/fuzzer_corpus/910a4cd86e2bdc41e8e4ae4e06e918ec0bc91b61 b/lexer/fuzzer_corpus/910a4cd86e2bdc41e8e4ae4e06e918ec0bc91b61 new file mode 100644 index 0000000000000..c6cb8c9050ce5 Binary files /dev/null and b/lexer/fuzzer_corpus/910a4cd86e2bdc41e8e4ae4e06e918ec0bc91b61 differ diff --git a/lexer/fuzzer_corpus/92180b5585dde95b54512c0aa418844642cfbf1c b/lexer/fuzzer_corpus/92180b5585dde95b54512c0aa418844642cfbf1c new file mode 100644 index 0000000000000..e9047ccd7ac4e Binary files /dev/null and b/lexer/fuzzer_corpus/92180b5585dde95b54512c0aa418844642cfbf1c differ diff --git a/lexer/fuzzer_corpus/92a9994a35a5751f87ec0261a50c46f6cac276c1 b/lexer/fuzzer_corpus/92a9994a35a5751f87ec0261a50c46f6cac276c1 new file mode 100644 index 0000000000000..53c9627db87c7 Binary files /dev/null and b/lexer/fuzzer_corpus/92a9994a35a5751f87ec0261a50c46f6cac276c1 differ diff --git a/lexer/fuzzer_corpus/932d2c4206b09575f98aba174b4ae7eeecf9dd87 b/lexer/fuzzer_corpus/932d2c4206b09575f98aba174b4ae7eeecf9dd87 new file mode 100644 index 0000000000000..e920f35b8277b Binary files /dev/null and b/lexer/fuzzer_corpus/932d2c4206b09575f98aba174b4ae7eeecf9dd87 differ diff --git a/lexer/fuzzer_corpus/936b204683d9aace88edec6916348263f8845bd2 b/lexer/fuzzer_corpus/936b204683d9aace88edec6916348263f8845bd2 new file mode 100644 index 0000000000000..514b3d985c948 Binary files /dev/null and b/lexer/fuzzer_corpus/936b204683d9aace88edec6916348263f8845bd2 differ diff --git a/lexer/fuzzer_corpus/94992041a722e9461cb4da61dc3ca6b1c807fc52 b/lexer/fuzzer_corpus/94992041a722e9461cb4da61dc3ca6b1c807fc52 new file mode 100644 index 0000000000000..74998deeee7f6 Binary files /dev/null and b/lexer/fuzzer_corpus/94992041a722e9461cb4da61dc3ca6b1c807fc52 differ diff --git a/lexer/fuzzer_corpus/94f232a0e1b64975bc82dc84a484907795dd3c9a b/lexer/fuzzer_corpus/94f232a0e1b64975bc82dc84a484907795dd3c9a new file mode 100644 index 0000000000000..9d60c44012b6c Binary files /dev/null and b/lexer/fuzzer_corpus/94f232a0e1b64975bc82dc84a484907795dd3c9a differ diff --git a/lexer/fuzzer_corpus/94f5a62a7150c24701f9b0ccc4f003afaf33b04b b/lexer/fuzzer_corpus/94f5a62a7150c24701f9b0ccc4f003afaf33b04b new file mode 100644 index 0000000000000..3c44c7c016b91 Binary files /dev/null and b/lexer/fuzzer_corpus/94f5a62a7150c24701f9b0ccc4f003afaf33b04b differ diff --git a/lexer/fuzzer_corpus/951cfc17803941388dcb27de4933bcf3cfa4e308 b/lexer/fuzzer_corpus/951cfc17803941388dcb27de4933bcf3cfa4e308 new file mode 100644 index 0000000000000..27f962b82f590 Binary files /dev/null and b/lexer/fuzzer_corpus/951cfc17803941388dcb27de4933bcf3cfa4e308 differ diff --git a/lexer/fuzzer_corpus/96080c9605fcf493be8852845bc14430217f7f10 b/lexer/fuzzer_corpus/96080c9605fcf493be8852845bc14430217f7f10 new file mode 100644 index 0000000000000..8abd95ecb27c5 Binary files /dev/null and b/lexer/fuzzer_corpus/96080c9605fcf493be8852845bc14430217f7f10 differ diff --git a/lexer/fuzzer_corpus/9623152b8ee14d7d2be5fd3ca2eef2085125a009 b/lexer/fuzzer_corpus/9623152b8ee14d7d2be5fd3ca2eef2085125a009 new file mode 100644 index 0000000000000..22305986977a7 Binary files /dev/null and b/lexer/fuzzer_corpus/9623152b8ee14d7d2be5fd3ca2eef2085125a009 differ diff --git a/lexer/fuzzer_corpus/965d67a89f658ebc162f41d13880ceb53878398a b/lexer/fuzzer_corpus/965d67a89f658ebc162f41d13880ceb53878398a new file mode 100644 index 0000000000000..00b1122a67d52 Binary files /dev/null and b/lexer/fuzzer_corpus/965d67a89f658ebc162f41d13880ceb53878398a differ diff --git a/lexer/fuzzer_corpus/969a5335de7a36dab4b48601f83e8289d1e24e14 b/lexer/fuzzer_corpus/969a5335de7a36dab4b48601f83e8289d1e24e14 new file mode 100644 index 0000000000000..ad1165af08f92 Binary files /dev/null and b/lexer/fuzzer_corpus/969a5335de7a36dab4b48601f83e8289d1e24e14 differ diff --git a/lexer/fuzzer_corpus/971e03f3b57ac29e3328d70fb6a901e50cc1e3c3 b/lexer/fuzzer_corpus/971e03f3b57ac29e3328d70fb6a901e50cc1e3c3 new file mode 100644 index 0000000000000..5978da87ba160 Binary files /dev/null and b/lexer/fuzzer_corpus/971e03f3b57ac29e3328d70fb6a901e50cc1e3c3 differ diff --git a/lexer/fuzzer_corpus/976127759e62f70d02f40d83a2ed5b7951571ceb b/lexer/fuzzer_corpus/976127759e62f70d02f40d83a2ed5b7951571ceb new file mode 100644 index 0000000000000..b833ef9028ae2 Binary files /dev/null and b/lexer/fuzzer_corpus/976127759e62f70d02f40d83a2ed5b7951571ceb differ diff --git a/lexer/fuzzer_corpus/97846c394ced714617a96b8b35b582e232a9e7b3 b/lexer/fuzzer_corpus/97846c394ced714617a96b8b35b582e232a9e7b3 new file mode 100644 index 0000000000000..a73664657d7b4 Binary files /dev/null and b/lexer/fuzzer_corpus/97846c394ced714617a96b8b35b582e232a9e7b3 differ diff --git a/lexer/fuzzer_corpus/9888bb78613415dc88bd114be3f2633e747ae22f b/lexer/fuzzer_corpus/9888bb78613415dc88bd114be3f2633e747ae22f new file mode 100644 index 0000000000000..7d7bc4d03bb9d Binary files /dev/null and b/lexer/fuzzer_corpus/9888bb78613415dc88bd114be3f2633e747ae22f differ diff --git a/lexer/fuzzer_corpus/991945983caa4499432036384514d356d791b3be b/lexer/fuzzer_corpus/991945983caa4499432036384514d356d791b3be new file mode 100644 index 0000000000000..927102682d759 Binary files /dev/null and b/lexer/fuzzer_corpus/991945983caa4499432036384514d356d791b3be differ diff --git a/lexer/fuzzer_corpus/99a14a1d0ef9a631b975d4e3d898f1fc28634572 b/lexer/fuzzer_corpus/99a14a1d0ef9a631b975d4e3d898f1fc28634572 new file mode 100644 index 0000000000000..fbd945e1010a6 Binary files /dev/null and b/lexer/fuzzer_corpus/99a14a1d0ef9a631b975d4e3d898f1fc28634572 differ diff --git a/lexer/fuzzer_corpus/99c0f2d3879cc32cf8f28dbe298b8f51780b0691 b/lexer/fuzzer_corpus/99c0f2d3879cc32cf8f28dbe298b8f51780b0691 new file mode 100644 index 0000000000000..8afad01586140 Binary files /dev/null and b/lexer/fuzzer_corpus/99c0f2d3879cc32cf8f28dbe298b8f51780b0691 differ diff --git a/lexer/fuzzer_corpus/99c58901dd5b578c26d9f06a680cd1586d884d0b b/lexer/fuzzer_corpus/99c58901dd5b578c26d9f06a680cd1586d884d0b new file mode 100644 index 0000000000000..01e0ec1a0d8db Binary files /dev/null and b/lexer/fuzzer_corpus/99c58901dd5b578c26d9f06a680cd1586d884d0b differ diff --git a/lexer/fuzzer_corpus/99d75bc6a864abd29d7531565e692e87cd78ea7f b/lexer/fuzzer_corpus/99d75bc6a864abd29d7531565e692e87cd78ea7f new file mode 100644 index 0000000000000..dde1cb97f8371 Binary files /dev/null and b/lexer/fuzzer_corpus/99d75bc6a864abd29d7531565e692e87cd78ea7f differ diff --git a/lexer/fuzzer_corpus/9af095b9240a4b086859755798d38cd676803dd2 b/lexer/fuzzer_corpus/9af095b9240a4b086859755798d38cd676803dd2 new file mode 100644 index 0000000000000..dc9ae5a13b2e5 Binary files /dev/null and b/lexer/fuzzer_corpus/9af095b9240a4b086859755798d38cd676803dd2 differ diff --git a/lexer/fuzzer_corpus/9bef829363a2ea30f044c9521dc187bac9e518e8 b/lexer/fuzzer_corpus/9bef829363a2ea30f044c9521dc187bac9e518e8 new file mode 100644 index 0000000000000..ce9196a927c5b Binary files /dev/null and b/lexer/fuzzer_corpus/9bef829363a2ea30f044c9521dc187bac9e518e8 differ diff --git a/lexer/fuzzer_corpus/9c098fec348d1fb7ebd8f71dc3253896efec0698 b/lexer/fuzzer_corpus/9c098fec348d1fb7ebd8f71dc3253896efec0698 new file mode 100644 index 0000000000000..7c6c4c383281a Binary files /dev/null and b/lexer/fuzzer_corpus/9c098fec348d1fb7ebd8f71dc3253896efec0698 differ diff --git a/lexer/fuzzer_corpus/9c5122ab103de543630bc727b76bd7b21839c073 b/lexer/fuzzer_corpus/9c5122ab103de543630bc727b76bd7b21839c073 new file mode 100644 index 0000000000000..fa16ca0c74d0a Binary files /dev/null and b/lexer/fuzzer_corpus/9c5122ab103de543630bc727b76bd7b21839c073 differ diff --git a/lexer/fuzzer_corpus/9d89da8664157ea1b5faa5bde8caf9f31d85b4a4 b/lexer/fuzzer_corpus/9d89da8664157ea1b5faa5bde8caf9f31d85b4a4 new file mode 100644 index 0000000000000..7ed33ce041879 Binary files /dev/null and b/lexer/fuzzer_corpus/9d89da8664157ea1b5faa5bde8caf9f31d85b4a4 differ diff --git a/lexer/fuzzer_corpus/9da97ed219829e31b2b2a551b3ba371c7d7ab7f7 b/lexer/fuzzer_corpus/9da97ed219829e31b2b2a551b3ba371c7d7ab7f7 new file mode 100644 index 0000000000000..b99b06e437480 Binary files /dev/null and b/lexer/fuzzer_corpus/9da97ed219829e31b2b2a551b3ba371c7d7ab7f7 differ diff --git a/lexer/fuzzer_corpus/9e1bf8c243c01b2c171f28550c25b974eb3fc910 b/lexer/fuzzer_corpus/9e1bf8c243c01b2c171f28550c25b974eb3fc910 new file mode 100644 index 0000000000000..8025b653d84be Binary files /dev/null and b/lexer/fuzzer_corpus/9e1bf8c243c01b2c171f28550c25b974eb3fc910 differ diff --git a/lexer/fuzzer_corpus/9f915817eadb6a53e1ba53785c9e39d3addfb7a7 b/lexer/fuzzer_corpus/9f915817eadb6a53e1ba53785c9e39d3addfb7a7 new file mode 100644 index 0000000000000..da9944802cdc3 Binary files /dev/null and b/lexer/fuzzer_corpus/9f915817eadb6a53e1ba53785c9e39d3addfb7a7 differ diff --git a/lexer/fuzzer_corpus/a019adfa6af044c1f3102fd4eb35f3883eed7767 b/lexer/fuzzer_corpus/a019adfa6af044c1f3102fd4eb35f3883eed7767 new file mode 100644 index 0000000000000..a755420ef376a Binary files /dev/null and b/lexer/fuzzer_corpus/a019adfa6af044c1f3102fd4eb35f3883eed7767 differ diff --git a/lexer/fuzzer_corpus/a03d6067a14043eca56172c64d9d734bbc6d3453 b/lexer/fuzzer_corpus/a03d6067a14043eca56172c64d9d734bbc6d3453 new file mode 100644 index 0000000000000..d1f4dc4b96b79 Binary files /dev/null and b/lexer/fuzzer_corpus/a03d6067a14043eca56172c64d9d734bbc6d3453 differ diff --git a/lexer/fuzzer_corpus/a0c3025c93be787b318b0a0915efce0568f29fb4 b/lexer/fuzzer_corpus/a0c3025c93be787b318b0a0915efce0568f29fb4 new file mode 100644 index 0000000000000..4d5d4bcc1f90b Binary files /dev/null and b/lexer/fuzzer_corpus/a0c3025c93be787b318b0a0915efce0568f29fb4 differ diff --git a/lexer/fuzzer_corpus/a0e3646d2ccccb4a16225b1c4995d6053b4bdd4e b/lexer/fuzzer_corpus/a0e3646d2ccccb4a16225b1c4995d6053b4bdd4e new file mode 100644 index 0000000000000..9a46ca528a4f5 Binary files /dev/null and b/lexer/fuzzer_corpus/a0e3646d2ccccb4a16225b1c4995d6053b4bdd4e differ diff --git a/lexer/fuzzer_corpus/a0edc24b3d17348f5cb80d2b6b002ee58db6b035 b/lexer/fuzzer_corpus/a0edc24b3d17348f5cb80d2b6b002ee58db6b035 new file mode 100644 index 0000000000000..99f1f9fa614db Binary files /dev/null and b/lexer/fuzzer_corpus/a0edc24b3d17348f5cb80d2b6b002ee58db6b035 differ diff --git a/lexer/fuzzer_corpus/a101443f19af96e9408021094d007f3da407fb7f b/lexer/fuzzer_corpus/a101443f19af96e9408021094d007f3da407fb7f new file mode 100644 index 0000000000000..0eed11eedfc1b Binary files /dev/null and b/lexer/fuzzer_corpus/a101443f19af96e9408021094d007f3da407fb7f differ diff --git a/lexer/fuzzer_corpus/a116527e7791e8cfe660a8c53382415e74b1634e b/lexer/fuzzer_corpus/a116527e7791e8cfe660a8c53382415e74b1634e new file mode 100644 index 0000000000000..6461d7744f2be Binary files /dev/null and b/lexer/fuzzer_corpus/a116527e7791e8cfe660a8c53382415e74b1634e differ diff --git a/lexer/fuzzer_corpus/a1ca34f30d867511c5071bd62a0ec9532226a626 b/lexer/fuzzer_corpus/a1ca34f30d867511c5071bd62a0ec9532226a626 new file mode 100644 index 0000000000000..ca6102e8566bd Binary files /dev/null and b/lexer/fuzzer_corpus/a1ca34f30d867511c5071bd62a0ec9532226a626 differ diff --git a/lexer/fuzzer_corpus/a204e6fc1dd252eb72786637ece539fc8ccb08be b/lexer/fuzzer_corpus/a204e6fc1dd252eb72786637ece539fc8ccb08be new file mode 100644 index 0000000000000..8b2e4e199c77a Binary files /dev/null and b/lexer/fuzzer_corpus/a204e6fc1dd252eb72786637ece539fc8ccb08be differ diff --git a/lexer/fuzzer_corpus/a208cbe77067c1b6f065b09fc05a11f79843e4cf b/lexer/fuzzer_corpus/a208cbe77067c1b6f065b09fc05a11f79843e4cf new file mode 100644 index 0000000000000..2b3e1f5e3b0fa Binary files /dev/null and b/lexer/fuzzer_corpus/a208cbe77067c1b6f065b09fc05a11f79843e4cf differ diff --git a/lexer/fuzzer_corpus/a24bab38ef7386f98fc7f93be8d07c3f6ceb3ece b/lexer/fuzzer_corpus/a24bab38ef7386f98fc7f93be8d07c3f6ceb3ece new file mode 100644 index 0000000000000..4df6ae967bc34 Binary files /dev/null and b/lexer/fuzzer_corpus/a24bab38ef7386f98fc7f93be8d07c3f6ceb3ece differ diff --git a/lexer/fuzzer_corpus/a24c8bbc68f5bbc75f9e4125919196e52f3b3419 b/lexer/fuzzer_corpus/a24c8bbc68f5bbc75f9e4125919196e52f3b3419 new file mode 100644 index 0000000000000..f94550ce3c402 Binary files /dev/null and b/lexer/fuzzer_corpus/a24c8bbc68f5bbc75f9e4125919196e52f3b3419 differ diff --git a/lexer/fuzzer_corpus/a259e1a662bcc421d6834afeab3a47dc344eedba b/lexer/fuzzer_corpus/a259e1a662bcc421d6834afeab3a47dc344eedba new file mode 100644 index 0000000000000..4f6323800b8cf Binary files /dev/null and b/lexer/fuzzer_corpus/a259e1a662bcc421d6834afeab3a47dc344eedba differ diff --git a/lexer/fuzzer_corpus/a2967df1f22ff6eb94dc4837d98507c543aae3cb b/lexer/fuzzer_corpus/a2967df1f22ff6eb94dc4837d98507c543aae3cb new file mode 100644 index 0000000000000..ea22da1534233 Binary files /dev/null and b/lexer/fuzzer_corpus/a2967df1f22ff6eb94dc4837d98507c543aae3cb differ diff --git a/lexer/fuzzer_corpus/a29d6d6337656a0bd738e477cdd90231050f37cd b/lexer/fuzzer_corpus/a29d6d6337656a0bd738e477cdd90231050f37cd new file mode 100644 index 0000000000000..e6b13bcc3ad0a Binary files /dev/null and b/lexer/fuzzer_corpus/a29d6d6337656a0bd738e477cdd90231050f37cd differ diff --git a/lexer/fuzzer_corpus/a2c6ea965e4b89873c59972ad24de3521f5ec565 b/lexer/fuzzer_corpus/a2c6ea965e4b89873c59972ad24de3521f5ec565 new file mode 100644 index 0000000000000..5eeb34a9d357f Binary files /dev/null and b/lexer/fuzzer_corpus/a2c6ea965e4b89873c59972ad24de3521f5ec565 differ diff --git a/lexer/fuzzer_corpus/a3be9919f350dff90746d5bbd335e9c13e075c6b b/lexer/fuzzer_corpus/a3be9919f350dff90746d5bbd335e9c13e075c6b new file mode 100644 index 0000000000000..939aaa7b53101 Binary files /dev/null and b/lexer/fuzzer_corpus/a3be9919f350dff90746d5bbd335e9c13e075c6b differ diff --git a/lexer/fuzzer_corpus/a45e530738453fbacd12389fc0b7c6bb0b5908b5 b/lexer/fuzzer_corpus/a45e530738453fbacd12389fc0b7c6bb0b5908b5 new file mode 100644 index 0000000000000..49e6182cc8cc3 Binary files /dev/null and b/lexer/fuzzer_corpus/a45e530738453fbacd12389fc0b7c6bb0b5908b5 differ diff --git a/lexer/fuzzer_corpus/a4fdd8e51e18577132e9b1d5d9e8504e4d3c96dc b/lexer/fuzzer_corpus/a4fdd8e51e18577132e9b1d5d9e8504e4d3c96dc new file mode 100644 index 0000000000000..ce7bf297d5b3c Binary files /dev/null and b/lexer/fuzzer_corpus/a4fdd8e51e18577132e9b1d5d9e8504e4d3c96dc differ diff --git a/lexer/fuzzer_corpus/a5091dc3125fa52413368b176e86a49738c917a3 b/lexer/fuzzer_corpus/a5091dc3125fa52413368b176e86a49738c917a3 new file mode 100644 index 0000000000000..26e35f95b49b8 Binary files /dev/null and b/lexer/fuzzer_corpus/a5091dc3125fa52413368b176e86a49738c917a3 differ diff --git a/lexer/fuzzer_corpus/a5bd5b33d21e9a221872d7ea360f18433d3f3be1 b/lexer/fuzzer_corpus/a5bd5b33d21e9a221872d7ea360f18433d3f3be1 new file mode 100644 index 0000000000000..522fa3d0a77b8 Binary files /dev/null and b/lexer/fuzzer_corpus/a5bd5b33d21e9a221872d7ea360f18433d3f3be1 differ diff --git a/lexer/fuzzer_corpus/a6a2deed0d75339e09924b0f19655f6656a15f4f b/lexer/fuzzer_corpus/a6a2deed0d75339e09924b0f19655f6656a15f4f new file mode 100644 index 0000000000000..d67f6f939bab9 Binary files /dev/null and b/lexer/fuzzer_corpus/a6a2deed0d75339e09924b0f19655f6656a15f4f differ diff --git a/lexer/fuzzer_corpus/a81a394ce3ae2d25c0ee201c6578069a1b174e55 b/lexer/fuzzer_corpus/a81a394ce3ae2d25c0ee201c6578069a1b174e55 new file mode 100644 index 0000000000000..9d55ca75fa1b8 Binary files /dev/null and b/lexer/fuzzer_corpus/a81a394ce3ae2d25c0ee201c6578069a1b174e55 differ diff --git a/lexer/fuzzer_corpus/a9bcae8634ddb628111c2eedd0fcb768f85ff9f5 b/lexer/fuzzer_corpus/a9bcae8634ddb628111c2eedd0fcb768f85ff9f5 new file mode 100644 index 0000000000000..e4edb74cb95bd Binary files /dev/null and b/lexer/fuzzer_corpus/a9bcae8634ddb628111c2eedd0fcb768f85ff9f5 differ diff --git a/lexer/fuzzer_corpus/aa19f2bf66f2268aca0d9eca37dbc685625342a8 b/lexer/fuzzer_corpus/aa19f2bf66f2268aca0d9eca37dbc685625342a8 new file mode 100644 index 0000000000000..95bc3c06f2b34 Binary files /dev/null and b/lexer/fuzzer_corpus/aa19f2bf66f2268aca0d9eca37dbc685625342a8 differ diff --git a/lexer/fuzzer_corpus/aa597b4fbdaf9c8d1c9a17426569c76134ff0fb9 b/lexer/fuzzer_corpus/aa597b4fbdaf9c8d1c9a17426569c76134ff0fb9 new file mode 100644 index 0000000000000..d571a00b887f9 Binary files /dev/null and b/lexer/fuzzer_corpus/aa597b4fbdaf9c8d1c9a17426569c76134ff0fb9 differ diff --git a/lexer/fuzzer_corpus/ab437c655c7ad4f2d048b93a9bb3306d0d560e0f b/lexer/fuzzer_corpus/ab437c655c7ad4f2d048b93a9bb3306d0d560e0f new file mode 100644 index 0000000000000..e7052f2e3529f Binary files /dev/null and b/lexer/fuzzer_corpus/ab437c655c7ad4f2d048b93a9bb3306d0d560e0f differ diff --git a/lexer/fuzzer_corpus/ab7ec188984da88b4ab1fb8082a95165d2341dd1 b/lexer/fuzzer_corpus/ab7ec188984da88b4ab1fb8082a95165d2341dd1 new file mode 100644 index 0000000000000..5f1a2e682d03f Binary files /dev/null and b/lexer/fuzzer_corpus/ab7ec188984da88b4ab1fb8082a95165d2341dd1 differ diff --git a/lexer/fuzzer_corpus/ac0b4253921b9410c5dc7fb082cedf8836ccbcf6 b/lexer/fuzzer_corpus/ac0b4253921b9410c5dc7fb082cedf8836ccbcf6 new file mode 100644 index 0000000000000..5b5ea21ca7534 Binary files /dev/null and b/lexer/fuzzer_corpus/ac0b4253921b9410c5dc7fb082cedf8836ccbcf6 differ diff --git a/lexer/fuzzer_corpus/ac15ecf3a7ca5b032a266d1e9e008ee0aa60df50 b/lexer/fuzzer_corpus/ac15ecf3a7ca5b032a266d1e9e008ee0aa60df50 new file mode 100644 index 0000000000000..0cd14f2c56d9b Binary files /dev/null and b/lexer/fuzzer_corpus/ac15ecf3a7ca5b032a266d1e9e008ee0aa60df50 differ diff --git a/lexer/fuzzer_corpus/ac20951ae95689b3ead13a8095570c9a2ed30e5e b/lexer/fuzzer_corpus/ac20951ae95689b3ead13a8095570c9a2ed30e5e new file mode 100644 index 0000000000000..b1273acc8989e Binary files /dev/null and b/lexer/fuzzer_corpus/ac20951ae95689b3ead13a8095570c9a2ed30e5e differ diff --git a/lexer/fuzzer_corpus/ac7e926bb0bd586f189a08bd9887dd903b7ca7ab b/lexer/fuzzer_corpus/ac7e926bb0bd586f189a08bd9887dd903b7ca7ab new file mode 100644 index 0000000000000..e24a97dd02db7 Binary files /dev/null and b/lexer/fuzzer_corpus/ac7e926bb0bd586f189a08bd9887dd903b7ca7ab differ diff --git a/lexer/fuzzer_corpus/aca10067e6738a677ea8f18c501d1078bc89586b b/lexer/fuzzer_corpus/aca10067e6738a677ea8f18c501d1078bc89586b new file mode 100644 index 0000000000000..dddac07056b53 Binary files /dev/null and b/lexer/fuzzer_corpus/aca10067e6738a677ea8f18c501d1078bc89586b differ diff --git a/lexer/fuzzer_corpus/acd71bf955344d94564db8a7d3ab6ed13f5f30e5 b/lexer/fuzzer_corpus/acd71bf955344d94564db8a7d3ab6ed13f5f30e5 new file mode 100644 index 0000000000000..47efd22946408 Binary files /dev/null and b/lexer/fuzzer_corpus/acd71bf955344d94564db8a7d3ab6ed13f5f30e5 differ diff --git a/lexer/fuzzer_corpus/ad121b61847802cec3b334da80f661e215dd16d3 b/lexer/fuzzer_corpus/ad121b61847802cec3b334da80f661e215dd16d3 new file mode 100644 index 0000000000000..4ab5bcb0301c3 Binary files /dev/null and b/lexer/fuzzer_corpus/ad121b61847802cec3b334da80f661e215dd16d3 differ diff --git a/lexer/fuzzer_corpus/adc83b19e793491b1c6ea0fd8b46cd9f32e592fc b/lexer/fuzzer_corpus/adc83b19e793491b1c6ea0fd8b46cd9f32e592fc new file mode 100644 index 0000000000000..8b137891791fe --- /dev/null +++ b/lexer/fuzzer_corpus/adc83b19e793491b1c6ea0fd8b46cd9f32e592fc @@ -0,0 +1 @@ + diff --git a/lexer/fuzzer_corpus/adcb06d68c467bc49a484a781f3645f2e0fce25d b/lexer/fuzzer_corpus/adcb06d68c467bc49a484a781f3645f2e0fce25d new file mode 100644 index 0000000000000..6502b3b04d247 Binary files /dev/null and b/lexer/fuzzer_corpus/adcb06d68c467bc49a484a781f3645f2e0fce25d differ diff --git a/lexer/fuzzer_corpus/ae4efce72909d7c1fc095fa47f3cf98f4a7ffb34 b/lexer/fuzzer_corpus/ae4efce72909d7c1fc095fa47f3cf98f4a7ffb34 new file mode 100644 index 0000000000000..5acdf806852ab Binary files /dev/null and b/lexer/fuzzer_corpus/ae4efce72909d7c1fc095fa47f3cf98f4a7ffb34 differ diff --git a/lexer/fuzzer_corpus/ae9c533353214d987cef18fd97d4fbaa24c39c09 b/lexer/fuzzer_corpus/ae9c533353214d987cef18fd97d4fbaa24c39c09 new file mode 100644 index 0000000000000..290ad7c6f17db Binary files /dev/null and b/lexer/fuzzer_corpus/ae9c533353214d987cef18fd97d4fbaa24c39c09 differ diff --git a/lexer/fuzzer_corpus/afeb1f561aa265b0793c0cf08ffbe986ba95cccd b/lexer/fuzzer_corpus/afeb1f561aa265b0793c0cf08ffbe986ba95cccd new file mode 100644 index 0000000000000..4f4b09fde155f Binary files /dev/null and b/lexer/fuzzer_corpus/afeb1f561aa265b0793c0cf08ffbe986ba95cccd differ diff --git a/lexer/fuzzer_corpus/b02b1b59c1aaf66b44b15dfc215b921a276f23e9 b/lexer/fuzzer_corpus/b02b1b59c1aaf66b44b15dfc215b921a276f23e9 new file mode 100644 index 0000000000000..a479236390250 Binary files /dev/null and b/lexer/fuzzer_corpus/b02b1b59c1aaf66b44b15dfc215b921a276f23e9 differ diff --git a/lexer/fuzzer_corpus/b071ec550ba1741734797a724d10985838abe3f0 b/lexer/fuzzer_corpus/b071ec550ba1741734797a724d10985838abe3f0 new file mode 100644 index 0000000000000..0c12f07c08c2c Binary files /dev/null and b/lexer/fuzzer_corpus/b071ec550ba1741734797a724d10985838abe3f0 differ diff --git a/lexer/fuzzer_corpus/b1256a6b51fe7e4af001152847a41e23822312fd b/lexer/fuzzer_corpus/b1256a6b51fe7e4af001152847a41e23822312fd new file mode 100644 index 0000000000000..46a55914b8986 Binary files /dev/null and b/lexer/fuzzer_corpus/b1256a6b51fe7e4af001152847a41e23822312fd differ diff --git a/lexer/fuzzer_corpus/b1898734ba5a8617c5535e2ec02feb33764c5da7 b/lexer/fuzzer_corpus/b1898734ba5a8617c5535e2ec02feb33764c5da7 new file mode 100644 index 0000000000000..0b446340aa4e4 Binary files /dev/null and b/lexer/fuzzer_corpus/b1898734ba5a8617c5535e2ec02feb33764c5da7 differ diff --git a/lexer/fuzzer_corpus/b1ac04528499a30c79bb5a6a887c7f093ffd3d7b b/lexer/fuzzer_corpus/b1ac04528499a30c79bb5a6a887c7f093ffd3d7b new file mode 100644 index 0000000000000..4b97a85c5ac69 Binary files /dev/null and b/lexer/fuzzer_corpus/b1ac04528499a30c79bb5a6a887c7f093ffd3d7b differ diff --git a/lexer/fuzzer_corpus/b27345719ecf784b471e96fdc2407baaac45ecd0 b/lexer/fuzzer_corpus/b27345719ecf784b471e96fdc2407baaac45ecd0 new file mode 100644 index 0000000000000..219013d51239a Binary files /dev/null and b/lexer/fuzzer_corpus/b27345719ecf784b471e96fdc2407baaac45ecd0 differ diff --git a/lexer/fuzzer_corpus/b378c4f7a72199ec1da13bd6756d4487ac1a9fa3 b/lexer/fuzzer_corpus/b378c4f7a72199ec1da13bd6756d4487ac1a9fa3 new file mode 100644 index 0000000000000..d2d739f678d7f Binary files /dev/null and b/lexer/fuzzer_corpus/b378c4f7a72199ec1da13bd6756d4487ac1a9fa3 differ diff --git a/lexer/fuzzer_corpus/b399c00c30a1356525d5d39e7cfeb99b2f878bee b/lexer/fuzzer_corpus/b399c00c30a1356525d5d39e7cfeb99b2f878bee new file mode 100644 index 0000000000000..20f506ce44eaa Binary files /dev/null and b/lexer/fuzzer_corpus/b399c00c30a1356525d5d39e7cfeb99b2f878bee differ diff --git a/lexer/fuzzer_corpus/b41ca4c43c7c6700e337d07a188f3f1a8c13fe46 b/lexer/fuzzer_corpus/b41ca4c43c7c6700e337d07a188f3f1a8c13fe46 new file mode 100644 index 0000000000000..e2731b651f64c Binary files /dev/null and b/lexer/fuzzer_corpus/b41ca4c43c7c6700e337d07a188f3f1a8c13fe46 differ diff --git a/lexer/fuzzer_corpus/b41da924198c3920fdaa0b0ff572615dae93f7dd b/lexer/fuzzer_corpus/b41da924198c3920fdaa0b0ff572615dae93f7dd new file mode 100644 index 0000000000000..c03b2c2ee9a08 Binary files /dev/null and b/lexer/fuzzer_corpus/b41da924198c3920fdaa0b0ff572615dae93f7dd differ diff --git a/lexer/fuzzer_corpus/b4899afd21a1c4c6eb7459bf91ed43689919798e b/lexer/fuzzer_corpus/b4899afd21a1c4c6eb7459bf91ed43689919798e new file mode 100644 index 0000000000000..b0900ca9b04d7 Binary files /dev/null and b/lexer/fuzzer_corpus/b4899afd21a1c4c6eb7459bf91ed43689919798e differ diff --git a/lexer/fuzzer_corpus/b49b6c4ec35af87447943af9acb1a14a637ba4ae b/lexer/fuzzer_corpus/b49b6c4ec35af87447943af9acb1a14a637ba4ae new file mode 100644 index 0000000000000..7105883b0a0f8 Binary files /dev/null and b/lexer/fuzzer_corpus/b49b6c4ec35af87447943af9acb1a14a637ba4ae differ diff --git a/lexer/fuzzer_corpus/b49dddf68a09edcb754e5d95cadc35e2683df728 b/lexer/fuzzer_corpus/b49dddf68a09edcb754e5d95cadc35e2683df728 new file mode 100644 index 0000000000000..4a4a95440a500 Binary files /dev/null and b/lexer/fuzzer_corpus/b49dddf68a09edcb754e5d95cadc35e2683df728 differ diff --git a/lexer/fuzzer_corpus/b4c66e7c06855671e747f3100c9d5072b79cf83a b/lexer/fuzzer_corpus/b4c66e7c06855671e747f3100c9d5072b79cf83a new file mode 100644 index 0000000000000..7afc70f2e3702 Binary files /dev/null and b/lexer/fuzzer_corpus/b4c66e7c06855671e747f3100c9d5072b79cf83a differ diff --git a/lexer/fuzzer_corpus/b52f85fe855438edbb3177ba64ea88040c5068c3 b/lexer/fuzzer_corpus/b52f85fe855438edbb3177ba64ea88040c5068c3 new file mode 100644 index 0000000000000..2d43129532042 Binary files /dev/null and b/lexer/fuzzer_corpus/b52f85fe855438edbb3177ba64ea88040c5068c3 differ diff --git a/lexer/fuzzer_corpus/b5945b57aad75381e3a3545542330957d8af699b b/lexer/fuzzer_corpus/b5945b57aad75381e3a3545542330957d8af699b new file mode 100644 index 0000000000000..ca94d0e3334b7 Binary files /dev/null and b/lexer/fuzzer_corpus/b5945b57aad75381e3a3545542330957d8af699b differ diff --git a/lexer/fuzzer_corpus/b5eefc1ac36df7e12eaf561ea2198bd05650348f b/lexer/fuzzer_corpus/b5eefc1ac36df7e12eaf561ea2198bd05650348f new file mode 100644 index 0000000000000..b7d7412ae2b1b Binary files /dev/null and b/lexer/fuzzer_corpus/b5eefc1ac36df7e12eaf561ea2198bd05650348f differ diff --git a/lexer/fuzzer_corpus/b68b13e7dbf523d3850bfb4ee1879ee267ce1309 b/lexer/fuzzer_corpus/b68b13e7dbf523d3850bfb4ee1879ee267ce1309 new file mode 100644 index 0000000000000..c8b199635c599 Binary files /dev/null and b/lexer/fuzzer_corpus/b68b13e7dbf523d3850bfb4ee1879ee267ce1309 differ diff --git a/lexer/fuzzer_corpus/b6f15ce61e23d12e16ac972ece7ecb71a1f4c87a b/lexer/fuzzer_corpus/b6f15ce61e23d12e16ac972ece7ecb71a1f4c87a new file mode 100644 index 0000000000000..4d3d341f33fea Binary files /dev/null and b/lexer/fuzzer_corpus/b6f15ce61e23d12e16ac972ece7ecb71a1f4c87a differ diff --git a/lexer/fuzzer_corpus/b7294356ab92ecb2a8f35f0b61cb2f3fc68af2f7 b/lexer/fuzzer_corpus/b7294356ab92ecb2a8f35f0b61cb2f3fc68af2f7 new file mode 100644 index 0000000000000..adaf863928661 Binary files /dev/null and b/lexer/fuzzer_corpus/b7294356ab92ecb2a8f35f0b61cb2f3fc68af2f7 differ diff --git a/lexer/fuzzer_corpus/b7719b430c03af92aef14110ee2fbf3abba0de5d b/lexer/fuzzer_corpus/b7719b430c03af92aef14110ee2fbf3abba0de5d new file mode 100644 index 0000000000000..865807decea7d Binary files /dev/null and b/lexer/fuzzer_corpus/b7719b430c03af92aef14110ee2fbf3abba0de5d differ diff --git a/lexer/fuzzer_corpus/b7cd4bb8ad5ec4e73e07a3cda397faa3f19e4286 b/lexer/fuzzer_corpus/b7cd4bb8ad5ec4e73e07a3cda397faa3f19e4286 new file mode 100644 index 0000000000000..d5e26d956cfee Binary files /dev/null and b/lexer/fuzzer_corpus/b7cd4bb8ad5ec4e73e07a3cda397faa3f19e4286 differ diff --git a/lexer/fuzzer_corpus/b8e9125a0ae565f71d1d93249e347d56068d42bf b/lexer/fuzzer_corpus/b8e9125a0ae565f71d1d93249e347d56068d42bf new file mode 100644 index 0000000000000..a482f9b058e2b Binary files /dev/null and b/lexer/fuzzer_corpus/b8e9125a0ae565f71d1d93249e347d56068d42bf differ diff --git a/lexer/fuzzer_corpus/b9a93efe3d76aecec38a121521ae64810a0769c2 b/lexer/fuzzer_corpus/b9a93efe3d76aecec38a121521ae64810a0769c2 new file mode 100644 index 0000000000000..63f0b68d4d492 Binary files /dev/null and b/lexer/fuzzer_corpus/b9a93efe3d76aecec38a121521ae64810a0769c2 differ diff --git a/lexer/fuzzer_corpus/bb1e5e53f6a1253fe1e896b50e9edc0095e2932b b/lexer/fuzzer_corpus/bb1e5e53f6a1253fe1e896b50e9edc0095e2932b new file mode 100644 index 0000000000000..ec6097b198a4e Binary files /dev/null and b/lexer/fuzzer_corpus/bb1e5e53f6a1253fe1e896b50e9edc0095e2932b differ diff --git a/lexer/fuzzer_corpus/bd7c4c25399bf8f278eb615f58b75b8f72349bf6 b/lexer/fuzzer_corpus/bd7c4c25399bf8f278eb615f58b75b8f72349bf6 new file mode 100644 index 0000000000000..61f4efbea8717 Binary files /dev/null and b/lexer/fuzzer_corpus/bd7c4c25399bf8f278eb615f58b75b8f72349bf6 differ diff --git a/lexer/fuzzer_corpus/be7f31b717f86fda96b3148fb706908303e17f8f b/lexer/fuzzer_corpus/be7f31b717f86fda96b3148fb706908303e17f8f new file mode 100644 index 0000000000000..a9790b18b838d Binary files /dev/null and b/lexer/fuzzer_corpus/be7f31b717f86fda96b3148fb706908303e17f8f differ diff --git a/lexer/fuzzer_corpus/bee6d10492fd61954b05c5108f958c5406b30fc9 b/lexer/fuzzer_corpus/bee6d10492fd61954b05c5108f958c5406b30fc9 new file mode 100644 index 0000000000000..7aa04705a1ede Binary files /dev/null and b/lexer/fuzzer_corpus/bee6d10492fd61954b05c5108f958c5406b30fc9 differ diff --git a/lexer/fuzzer_corpus/befde2b724d2370f61d7dc2da95d1be21a9c862b b/lexer/fuzzer_corpus/befde2b724d2370f61d7dc2da95d1be21a9c862b new file mode 100644 index 0000000000000..fe1a052f5908d Binary files /dev/null and b/lexer/fuzzer_corpus/befde2b724d2370f61d7dc2da95d1be21a9c862b differ diff --git a/lexer/fuzzer_corpus/c02bf5d08db57f3aae88172e17cad08ebd076c22 b/lexer/fuzzer_corpus/c02bf5d08db57f3aae88172e17cad08ebd076c22 new file mode 100644 index 0000000000000..1725660c3e0ee Binary files /dev/null and b/lexer/fuzzer_corpus/c02bf5d08db57f3aae88172e17cad08ebd076c22 differ diff --git a/lexer/fuzzer_corpus/c18fefd5d4d05d1ee9f162801aa6e17f428ba8da b/lexer/fuzzer_corpus/c18fefd5d4d05d1ee9f162801aa6e17f428ba8da new file mode 100644 index 0000000000000..3d1966813c4d8 Binary files /dev/null and b/lexer/fuzzer_corpus/c18fefd5d4d05d1ee9f162801aa6e17f428ba8da differ diff --git a/lexer/fuzzer_corpus/c1b2bcb63da8041ce87b328132217ba70d920fca b/lexer/fuzzer_corpus/c1b2bcb63da8041ce87b328132217ba70d920fca new file mode 100644 index 0000000000000..4ad2545a86b55 Binary files /dev/null and b/lexer/fuzzer_corpus/c1b2bcb63da8041ce87b328132217ba70d920fca differ diff --git a/lexer/fuzzer_corpus/c1df12a64dd8abefb0e6590527cb10ed692cf324 b/lexer/fuzzer_corpus/c1df12a64dd8abefb0e6590527cb10ed692cf324 new file mode 100644 index 0000000000000..a16a93b68fc77 Binary files /dev/null and b/lexer/fuzzer_corpus/c1df12a64dd8abefb0e6590527cb10ed692cf324 differ diff --git a/lexer/fuzzer_corpus/c21a4d67d547c0845676886c6892c8c6827d75e2 b/lexer/fuzzer_corpus/c21a4d67d547c0845676886c6892c8c6827d75e2 new file mode 100644 index 0000000000000..dcbdd1cfa6cd3 Binary files /dev/null and b/lexer/fuzzer_corpus/c21a4d67d547c0845676886c6892c8c6827d75e2 differ diff --git a/lexer/fuzzer_corpus/c2bec5c726a06b456cdd88f28a32f48831044a11 b/lexer/fuzzer_corpus/c2bec5c726a06b456cdd88f28a32f48831044a11 new file mode 100644 index 0000000000000..e1ded6082090d Binary files /dev/null and b/lexer/fuzzer_corpus/c2bec5c726a06b456cdd88f28a32f48831044a11 differ diff --git a/lexer/fuzzer_corpus/c32505224c93f972f500f44afb188d3889dfa159 b/lexer/fuzzer_corpus/c32505224c93f972f500f44afb188d3889dfa159 new file mode 100644 index 0000000000000..369664575f1c5 Binary files /dev/null and b/lexer/fuzzer_corpus/c32505224c93f972f500f44afb188d3889dfa159 differ diff --git a/lexer/fuzzer_corpus/c3670cccd1b3582fb1ec1b742f6e2898cfe26555 b/lexer/fuzzer_corpus/c3670cccd1b3582fb1ec1b742f6e2898cfe26555 new file mode 100644 index 0000000000000..f109405a581ab Binary files /dev/null and b/lexer/fuzzer_corpus/c3670cccd1b3582fb1ec1b742f6e2898cfe26555 differ diff --git a/lexer/fuzzer_corpus/c3ed53cf882409c54cc6c988aae34deacecf057a b/lexer/fuzzer_corpus/c3ed53cf882409c54cc6c988aae34deacecf057a new file mode 100644 index 0000000000000..0d29dd658099a Binary files /dev/null and b/lexer/fuzzer_corpus/c3ed53cf882409c54cc6c988aae34deacecf057a differ diff --git a/lexer/fuzzer_corpus/c63072e6cf44fde5c8ec3e7ea1a5328497ea5479 b/lexer/fuzzer_corpus/c63072e6cf44fde5c8ec3e7ea1a5328497ea5479 new file mode 100644 index 0000000000000..0a5df510dc44c Binary files /dev/null and b/lexer/fuzzer_corpus/c63072e6cf44fde5c8ec3e7ea1a5328497ea5479 differ diff --git a/lexer/fuzzer_corpus/c6435d401d81e17e25e273719e4f1fa7fe1b53b7 b/lexer/fuzzer_corpus/c6435d401d81e17e25e273719e4f1fa7fe1b53b7 new file mode 100644 index 0000000000000..504f74e43de49 Binary files /dev/null and b/lexer/fuzzer_corpus/c6435d401d81e17e25e273719e4f1fa7fe1b53b7 differ diff --git a/lexer/fuzzer_corpus/c70643d26ec7090724a54c6e87ea68c89d4c4e91 b/lexer/fuzzer_corpus/c70643d26ec7090724a54c6e87ea68c89d4c4e91 new file mode 100644 index 0000000000000..29e44e89f129c Binary files /dev/null and b/lexer/fuzzer_corpus/c70643d26ec7090724a54c6e87ea68c89d4c4e91 differ diff --git a/lexer/fuzzer_corpus/c7fb3f5be75a6dacfc3e28abbf73fe5e58495d30 b/lexer/fuzzer_corpus/c7fb3f5be75a6dacfc3e28abbf73fe5e58495d30 new file mode 100644 index 0000000000000..8263326a9a38a Binary files /dev/null and b/lexer/fuzzer_corpus/c7fb3f5be75a6dacfc3e28abbf73fe5e58495d30 differ diff --git a/lexer/fuzzer_corpus/ca576072e488152f50e6f1158a1396f73ddd2c07 b/lexer/fuzzer_corpus/ca576072e488152f50e6f1158a1396f73ddd2c07 new file mode 100644 index 0000000000000..4ffe592cbaa7e Binary files /dev/null and b/lexer/fuzzer_corpus/ca576072e488152f50e6f1158a1396f73ddd2c07 differ diff --git a/lexer/fuzzer_corpus/cb2edb9e0c977a705a6f29fae0934079a1b60c92 b/lexer/fuzzer_corpus/cb2edb9e0c977a705a6f29fae0934079a1b60c92 new file mode 100644 index 0000000000000..4c488ef17bd73 Binary files /dev/null and b/lexer/fuzzer_corpus/cb2edb9e0c977a705a6f29fae0934079a1b60c92 differ diff --git a/lexer/fuzzer_corpus/cb864d68d0b90ecc000956495f457f2da8cc0181 b/lexer/fuzzer_corpus/cb864d68d0b90ecc000956495f457f2da8cc0181 new file mode 100644 index 0000000000000..2e116a9a4d6c5 Binary files /dev/null and b/lexer/fuzzer_corpus/cb864d68d0b90ecc000956495f457f2da8cc0181 differ diff --git a/lexer/fuzzer_corpus/cbe9914a17fe70a1883c2dab7491fd0211a5ee38 b/lexer/fuzzer_corpus/cbe9914a17fe70a1883c2dab7491fd0211a5ee38 new file mode 100644 index 0000000000000..49d9b4c472f18 Binary files /dev/null and b/lexer/fuzzer_corpus/cbe9914a17fe70a1883c2dab7491fd0211a5ee38 differ diff --git a/lexer/fuzzer_corpus/cc34b4cfee9dc462dcc95d7560a21cbf4adcf7b0 b/lexer/fuzzer_corpus/cc34b4cfee9dc462dcc95d7560a21cbf4adcf7b0 new file mode 100644 index 0000000000000..1907d3e665c87 Binary files /dev/null and b/lexer/fuzzer_corpus/cc34b4cfee9dc462dcc95d7560a21cbf4adcf7b0 differ diff --git a/lexer/fuzzer_corpus/cc652bd0ebdb4b1726182372be30c691099f7e8c b/lexer/fuzzer_corpus/cc652bd0ebdb4b1726182372be30c691099f7e8c new file mode 100644 index 0000000000000..8896e0bebd5e5 Binary files /dev/null and b/lexer/fuzzer_corpus/cc652bd0ebdb4b1726182372be30c691099f7e8c differ diff --git a/lexer/fuzzer_corpus/ccaac59aa2a93544b76f073311b350dcf65fbf5e b/lexer/fuzzer_corpus/ccaac59aa2a93544b76f073311b350dcf65fbf5e new file mode 100644 index 0000000000000..9b7ba2ca8e09a Binary files /dev/null and b/lexer/fuzzer_corpus/ccaac59aa2a93544b76f073311b350dcf65fbf5e differ diff --git a/lexer/fuzzer_corpus/ccc73c223aabaa38c58624ea80eb778c645346f2 b/lexer/fuzzer_corpus/ccc73c223aabaa38c58624ea80eb778c645346f2 new file mode 100644 index 0000000000000..77690850be768 Binary files /dev/null and b/lexer/fuzzer_corpus/ccc73c223aabaa38c58624ea80eb778c645346f2 differ diff --git a/lexer/fuzzer_corpus/cd04f9a383899d9640ff1e9707e47e77afdd9562 b/lexer/fuzzer_corpus/cd04f9a383899d9640ff1e9707e47e77afdd9562 new file mode 100644 index 0000000000000..db521616e270e Binary files /dev/null and b/lexer/fuzzer_corpus/cd04f9a383899d9640ff1e9707e47e77afdd9562 differ diff --git a/lexer/fuzzer_corpus/cd2ef6d312ee80a3d4c377361fddbb793889f4c8 b/lexer/fuzzer_corpus/cd2ef6d312ee80a3d4c377361fddbb793889f4c8 new file mode 100644 index 0000000000000..eb991762c4ff3 Binary files /dev/null and b/lexer/fuzzer_corpus/cd2ef6d312ee80a3d4c377361fddbb793889f4c8 differ diff --git a/lexer/fuzzer_corpus/ce010e3e4729a79b9793c64a4532b53c54c9430c b/lexer/fuzzer_corpus/ce010e3e4729a79b9793c64a4532b53c54c9430c new file mode 100644 index 0000000000000..9db3a093364a1 Binary files /dev/null and b/lexer/fuzzer_corpus/ce010e3e4729a79b9793c64a4532b53c54c9430c differ diff --git a/lexer/fuzzer_corpus/ce3e985cb8f77fcd48e3ea06f922260a2dd36fda b/lexer/fuzzer_corpus/ce3e985cb8f77fcd48e3ea06f922260a2dd36fda new file mode 100644 index 0000000000000..faac4ec96c407 Binary files /dev/null and b/lexer/fuzzer_corpus/ce3e985cb8f77fcd48e3ea06f922260a2dd36fda differ diff --git a/lexer/fuzzer_corpus/cf7e04eb0562f842999ecc3f4ec6877fa4f1e342 b/lexer/fuzzer_corpus/cf7e04eb0562f842999ecc3f4ec6877fa4f1e342 new file mode 100644 index 0000000000000..9f3796f5e0aa6 Binary files /dev/null and b/lexer/fuzzer_corpus/cf7e04eb0562f842999ecc3f4ec6877fa4f1e342 differ diff --git a/lexer/fuzzer_corpus/d1020364827748c3a6e1bf09c6e75a5edf44079e b/lexer/fuzzer_corpus/d1020364827748c3a6e1bf09c6e75a5edf44079e new file mode 100644 index 0000000000000..ae80374f0698a Binary files /dev/null and b/lexer/fuzzer_corpus/d1020364827748c3a6e1bf09c6e75a5edf44079e differ diff --git a/lexer/fuzzer_corpus/d19cd0207ab449752da6b55a82975be7f056a70b b/lexer/fuzzer_corpus/d19cd0207ab449752da6b55a82975be7f056a70b new file mode 100644 index 0000000000000..fa7136b457ef6 Binary files /dev/null and b/lexer/fuzzer_corpus/d19cd0207ab449752da6b55a82975be7f056a70b differ diff --git a/lexer/fuzzer_corpus/d1ee881e19534be9fc12fd862bb3f2269a09bdd7 b/lexer/fuzzer_corpus/d1ee881e19534be9fc12fd862bb3f2269a09bdd7 new file mode 100644 index 0000000000000..ea7e6ea0b3dad Binary files /dev/null and b/lexer/fuzzer_corpus/d1ee881e19534be9fc12fd862bb3f2269a09bdd7 differ diff --git a/lexer/fuzzer_corpus/d26ba8c01b80dbb3ea1a50d429b832a33b6af614 b/lexer/fuzzer_corpus/d26ba8c01b80dbb3ea1a50d429b832a33b6af614 new file mode 100644 index 0000000000000..fd60487508b76 Binary files /dev/null and b/lexer/fuzzer_corpus/d26ba8c01b80dbb3ea1a50d429b832a33b6af614 differ diff --git a/lexer/fuzzer_corpus/d2b69ab4a6972e182cdef5c319be940062388300 b/lexer/fuzzer_corpus/d2b69ab4a6972e182cdef5c319be940062388300 new file mode 100644 index 0000000000000..7c19668cc8238 Binary files /dev/null and b/lexer/fuzzer_corpus/d2b69ab4a6972e182cdef5c319be940062388300 differ diff --git a/lexer/fuzzer_corpus/d38a79abc527773b20840f0ac21b618049746c32 b/lexer/fuzzer_corpus/d38a79abc527773b20840f0ac21b618049746c32 new file mode 100644 index 0000000000000..93fc8823b37a4 Binary files /dev/null and b/lexer/fuzzer_corpus/d38a79abc527773b20840f0ac21b618049746c32 differ diff --git a/lexer/fuzzer_corpus/d3ae4114281302b6810c02cee903e2f05aea79ce b/lexer/fuzzer_corpus/d3ae4114281302b6810c02cee903e2f05aea79ce new file mode 100644 index 0000000000000..ba5c95ab5adc6 Binary files /dev/null and b/lexer/fuzzer_corpus/d3ae4114281302b6810c02cee903e2f05aea79ce differ diff --git a/lexer/fuzzer_corpus/d3c908f06f91576b96d60c7b165811e69993ae78 b/lexer/fuzzer_corpus/d3c908f06f91576b96d60c7b165811e69993ae78 new file mode 100644 index 0000000000000..17d0c6e98b709 Binary files /dev/null and b/lexer/fuzzer_corpus/d3c908f06f91576b96d60c7b165811e69993ae78 differ diff --git a/lexer/fuzzer_corpus/d42d277470474b4dbf7a5181f298b299d3028a1a b/lexer/fuzzer_corpus/d42d277470474b4dbf7a5181f298b299d3028a1a new file mode 100644 index 0000000000000..dfc270bb17d42 Binary files /dev/null and b/lexer/fuzzer_corpus/d42d277470474b4dbf7a5181f298b299d3028a1a differ diff --git a/lexer/fuzzer_corpus/d488eca7d059c07b0f9e8ad7d7206ace887cda85 b/lexer/fuzzer_corpus/d488eca7d059c07b0f9e8ad7d7206ace887cda85 new file mode 100644 index 0000000000000..65cc9d3d3bd29 Binary files /dev/null and b/lexer/fuzzer_corpus/d488eca7d059c07b0f9e8ad7d7206ace887cda85 differ diff --git a/lexer/fuzzer_corpus/d51ab1559f11d168cd4cdf123602113f60732e79 b/lexer/fuzzer_corpus/d51ab1559f11d168cd4cdf123602113f60732e79 new file mode 100644 index 0000000000000..606724982a3ab Binary files /dev/null and b/lexer/fuzzer_corpus/d51ab1559f11d168cd4cdf123602113f60732e79 differ diff --git a/lexer/fuzzer_corpus/d8130b7018da880f1090b166df7bb13810f101bd b/lexer/fuzzer_corpus/d8130b7018da880f1090b166df7bb13810f101bd new file mode 100644 index 0000000000000..b1da6c943574d Binary files /dev/null and b/lexer/fuzzer_corpus/d8130b7018da880f1090b166df7bb13810f101bd differ diff --git a/lexer/fuzzer_corpus/d8e267ac5306c3dae043f15df37dae5b76a44a52 b/lexer/fuzzer_corpus/d8e267ac5306c3dae043f15df37dae5b76a44a52 new file mode 100644 index 0000000000000..268740129b761 Binary files /dev/null and b/lexer/fuzzer_corpus/d8e267ac5306c3dae043f15df37dae5b76a44a52 differ diff --git a/lexer/fuzzer_corpus/d9437de0ae53e80a335a15a00ec9e622f2fb89d3 b/lexer/fuzzer_corpus/d9437de0ae53e80a335a15a00ec9e622f2fb89d3 new file mode 100644 index 0000000000000..b14006dea1bb0 Binary files /dev/null and b/lexer/fuzzer_corpus/d9437de0ae53e80a335a15a00ec9e622f2fb89d3 differ diff --git a/lexer/fuzzer_corpus/da5dc3160083ac54ec11a162d23293ffef7029c8 b/lexer/fuzzer_corpus/da5dc3160083ac54ec11a162d23293ffef7029c8 new file mode 100644 index 0000000000000..4a9ea5ec03def Binary files /dev/null and b/lexer/fuzzer_corpus/da5dc3160083ac54ec11a162d23293ffef7029c8 differ diff --git a/lexer/fuzzer_corpus/daa6495ffb6c5cbf5cd3dbf3b7b8bfc42803d1d2 b/lexer/fuzzer_corpus/daa6495ffb6c5cbf5cd3dbf3b7b8bfc42803d1d2 new file mode 100644 index 0000000000000..a49bcda770456 Binary files /dev/null and b/lexer/fuzzer_corpus/daa6495ffb6c5cbf5cd3dbf3b7b8bfc42803d1d2 differ diff --git a/lexer/fuzzer_corpus/daed34b7f0f3a2f9b0716b14514473d6efa38c3f b/lexer/fuzzer_corpus/daed34b7f0f3a2f9b0716b14514473d6efa38c3f new file mode 100644 index 0000000000000..06fa49e50f2cd Binary files /dev/null and b/lexer/fuzzer_corpus/daed34b7f0f3a2f9b0716b14514473d6efa38c3f differ diff --git a/lexer/fuzzer_corpus/dba6c808eafe79949543a1f50ee39f82de48ccee b/lexer/fuzzer_corpus/dba6c808eafe79949543a1f50ee39f82de48ccee new file mode 100644 index 0000000000000..973bf2c5250ba Binary files /dev/null and b/lexer/fuzzer_corpus/dba6c808eafe79949543a1f50ee39f82de48ccee differ diff --git a/lexer/fuzzer_corpus/dbb41d9522e68d0faf7f5a03f41aed64e7b9d7ee b/lexer/fuzzer_corpus/dbb41d9522e68d0faf7f5a03f41aed64e7b9d7ee new file mode 100644 index 0000000000000..b7efa0a22de93 Binary files /dev/null and b/lexer/fuzzer_corpus/dbb41d9522e68d0faf7f5a03f41aed64e7b9d7ee differ diff --git a/lexer/fuzzer_corpus/dc119760a36e1c532dc48cf83cb4ff18cbef6c15 b/lexer/fuzzer_corpus/dc119760a36e1c532dc48cf83cb4ff18cbef6c15 new file mode 100644 index 0000000000000..30c8cd08f73ff Binary files /dev/null and b/lexer/fuzzer_corpus/dc119760a36e1c532dc48cf83cb4ff18cbef6c15 differ diff --git a/lexer/fuzzer_corpus/dcc6432dae6ed005aaa9f060c4909bb89c1d98d8 b/lexer/fuzzer_corpus/dcc6432dae6ed005aaa9f060c4909bb89c1d98d8 new file mode 100644 index 0000000000000..1e174a78ad313 Binary files /dev/null and b/lexer/fuzzer_corpus/dcc6432dae6ed005aaa9f060c4909bb89c1d98d8 differ diff --git a/lexer/fuzzer_corpus/dcf81fe92865368756d9729c63739c37b38f037b b/lexer/fuzzer_corpus/dcf81fe92865368756d9729c63739c37b38f037b new file mode 100644 index 0000000000000..b5318c4c08410 Binary files /dev/null and b/lexer/fuzzer_corpus/dcf81fe92865368756d9729c63739c37b38f037b differ diff --git a/lexer/fuzzer_corpus/dd0da9f51f2783d95680664efb3f5a322ee885b5 b/lexer/fuzzer_corpus/dd0da9f51f2783d95680664efb3f5a322ee885b5 new file mode 100644 index 0000000000000..d967972068737 Binary files /dev/null and b/lexer/fuzzer_corpus/dd0da9f51f2783d95680664efb3f5a322ee885b5 differ diff --git a/lexer/fuzzer_corpus/dd3f58c65c86aea5827d99b4fe213b183fc79107 b/lexer/fuzzer_corpus/dd3f58c65c86aea5827d99b4fe213b183fc79107 new file mode 100644 index 0000000000000..22be20d5add14 Binary files /dev/null and b/lexer/fuzzer_corpus/dd3f58c65c86aea5827d99b4fe213b183fc79107 differ diff --git a/lexer/fuzzer_corpus/de676ef0bc27e3411664f351ea13ee0f2901a8f1 b/lexer/fuzzer_corpus/de676ef0bc27e3411664f351ea13ee0f2901a8f1 new file mode 100644 index 0000000000000..eda1feb654418 Binary files /dev/null and b/lexer/fuzzer_corpus/de676ef0bc27e3411664f351ea13ee0f2901a8f1 differ diff --git a/lexer/fuzzer_corpus/df9bc143232105f71dbc4cd6f29f1faf40adc348 b/lexer/fuzzer_corpus/df9bc143232105f71dbc4cd6f29f1faf40adc348 new file mode 100644 index 0000000000000..7f03fd84c2dbb Binary files /dev/null and b/lexer/fuzzer_corpus/df9bc143232105f71dbc4cd6f29f1faf40adc348 differ diff --git a/lexer/fuzzer_corpus/e01aeb76d2ded444ad0373a68f2c784db09ca0bd b/lexer/fuzzer_corpus/e01aeb76d2ded444ad0373a68f2c784db09ca0bd new file mode 100644 index 0000000000000..9d674127a8288 Binary files /dev/null and b/lexer/fuzzer_corpus/e01aeb76d2ded444ad0373a68f2c784db09ca0bd differ diff --git a/lexer/fuzzer_corpus/e0938f8484d9bc11fc2e6c6a95b95bcda639c25a b/lexer/fuzzer_corpus/e0938f8484d9bc11fc2e6c6a95b95bcda639c25a new file mode 100644 index 0000000000000..fe17b49e1e147 Binary files /dev/null and b/lexer/fuzzer_corpus/e0938f8484d9bc11fc2e6c6a95b95bcda639c25a differ diff --git a/lexer/fuzzer_corpus/e0db3d59e7b298f022cace1fd52d3875e5fc9694 b/lexer/fuzzer_corpus/e0db3d59e7b298f022cace1fd52d3875e5fc9694 new file mode 100644 index 0000000000000..32fbe4feffc87 Binary files /dev/null and b/lexer/fuzzer_corpus/e0db3d59e7b298f022cace1fd52d3875e5fc9694 differ diff --git a/lexer/fuzzer_corpus/e255f1112e76bf142d77ee3c42cda90853fefeaf b/lexer/fuzzer_corpus/e255f1112e76bf142d77ee3c42cda90853fefeaf new file mode 100644 index 0000000000000..7f474f79cddb6 Binary files /dev/null and b/lexer/fuzzer_corpus/e255f1112e76bf142d77ee3c42cda90853fefeaf differ diff --git a/lexer/fuzzer_corpus/e26caaba5987ef0e4c8e8960a65d163eb739b2a2 b/lexer/fuzzer_corpus/e26caaba5987ef0e4c8e8960a65d163eb739b2a2 new file mode 100644 index 0000000000000..4cf0664ea8855 Binary files /dev/null and b/lexer/fuzzer_corpus/e26caaba5987ef0e4c8e8960a65d163eb739b2a2 differ diff --git a/lexer/fuzzer_corpus/e27d3a4fd0455fcd2766c39bf0c71431549a65cb b/lexer/fuzzer_corpus/e27d3a4fd0455fcd2766c39bf0c71431549a65cb new file mode 100644 index 0000000000000..d25fd48b9fcfc Binary files /dev/null and b/lexer/fuzzer_corpus/e27d3a4fd0455fcd2766c39bf0c71431549a65cb differ diff --git a/lexer/fuzzer_corpus/e2ca6c74ac95f6b766e427545f297fa1df3e7ae8 b/lexer/fuzzer_corpus/e2ca6c74ac95f6b766e427545f297fa1df3e7ae8 new file mode 100644 index 0000000000000..580257bba3434 Binary files /dev/null and b/lexer/fuzzer_corpus/e2ca6c74ac95f6b766e427545f297fa1df3e7ae8 differ diff --git a/lexer/fuzzer_corpus/e358ad9b805d394bdc48b7258e0c3b8becb2513b b/lexer/fuzzer_corpus/e358ad9b805d394bdc48b7258e0c3b8becb2513b new file mode 100644 index 0000000000000..46817d9566881 Binary files /dev/null and b/lexer/fuzzer_corpus/e358ad9b805d394bdc48b7258e0c3b8becb2513b differ diff --git a/lexer/fuzzer_corpus/e36e20af158c5a94704e98e0ee695eca5c85d557 b/lexer/fuzzer_corpus/e36e20af158c5a94704e98e0ee695eca5c85d557 new file mode 100644 index 0000000000000..d0a3745d42e3a Binary files /dev/null and b/lexer/fuzzer_corpus/e36e20af158c5a94704e98e0ee695eca5c85d557 differ diff --git a/lexer/fuzzer_corpus/e4faefa4502eaf582fb184a2300122f40de097af b/lexer/fuzzer_corpus/e4faefa4502eaf582fb184a2300122f40de097af new file mode 100644 index 0000000000000..bd474b2519cc1 Binary files /dev/null and b/lexer/fuzzer_corpus/e4faefa4502eaf582fb184a2300122f40de097af differ diff --git a/lexer/fuzzer_corpus/e5f2b058f54b45dc07f3b9d6e4637e8b7af2c5a7 b/lexer/fuzzer_corpus/e5f2b058f54b45dc07f3b9d6e4637e8b7af2c5a7 new file mode 100644 index 0000000000000..a365d7fdb7ca6 Binary files /dev/null and b/lexer/fuzzer_corpus/e5f2b058f54b45dc07f3b9d6e4637e8b7af2c5a7 differ diff --git a/lexer/fuzzer_corpus/e72cc6d41d020b5c579ab185b34447ec9520f86c b/lexer/fuzzer_corpus/e72cc6d41d020b5c579ab185b34447ec9520f86c new file mode 100644 index 0000000000000..c63664d67c8cd Binary files /dev/null and b/lexer/fuzzer_corpus/e72cc6d41d020b5c579ab185b34447ec9520f86c differ diff --git a/lexer/fuzzer_corpus/e82759d6621dc86de4cf5033868a3673e74053dd b/lexer/fuzzer_corpus/e82759d6621dc86de4cf5033868a3673e74053dd new file mode 100644 index 0000000000000..b678df50e02fb Binary files /dev/null and b/lexer/fuzzer_corpus/e82759d6621dc86de4cf5033868a3673e74053dd differ diff --git a/lexer/fuzzer_corpus/e8546e69f53ac69e138b8a739d321f629be57164 b/lexer/fuzzer_corpus/e8546e69f53ac69e138b8a739d321f629be57164 new file mode 100644 index 0000000000000..deb715648485e Binary files /dev/null and b/lexer/fuzzer_corpus/e8546e69f53ac69e138b8a739d321f629be57164 differ diff --git a/lexer/fuzzer_corpus/e8d8ab82dea469e3e3c7a536df5a77e3ad899129 b/lexer/fuzzer_corpus/e8d8ab82dea469e3e3c7a536df5a77e3ad899129 new file mode 100644 index 0000000000000..2c54e3c4a387b Binary files /dev/null and b/lexer/fuzzer_corpus/e8d8ab82dea469e3e3c7a536df5a77e3ad899129 differ diff --git a/lexer/fuzzer_corpus/e8e793189ab4cce6a062fa09c0740693b6ce1e19 b/lexer/fuzzer_corpus/e8e793189ab4cce6a062fa09c0740693b6ce1e19 new file mode 100644 index 0000000000000..17eaf77249699 Binary files /dev/null and b/lexer/fuzzer_corpus/e8e793189ab4cce6a062fa09c0740693b6ce1e19 differ diff --git a/lexer/fuzzer_corpus/e8ec7cc1ca98820ae6e98c1354ec2d6bece46ef2 b/lexer/fuzzer_corpus/e8ec7cc1ca98820ae6e98c1354ec2d6bece46ef2 new file mode 100644 index 0000000000000..8dc2caf3d1fad Binary files /dev/null and b/lexer/fuzzer_corpus/e8ec7cc1ca98820ae6e98c1354ec2d6bece46ef2 differ diff --git a/lexer/fuzzer_corpus/e91c47ea25efbbfb60ba17d786caa9659bbab6d2 b/lexer/fuzzer_corpus/e91c47ea25efbbfb60ba17d786caa9659bbab6d2 new file mode 100644 index 0000000000000..a3d4a44e9a72c Binary files /dev/null and b/lexer/fuzzer_corpus/e91c47ea25efbbfb60ba17d786caa9659bbab6d2 differ diff --git a/lexer/fuzzer_corpus/ea17e12d2e985a026980a016b72acacb30244a76 b/lexer/fuzzer_corpus/ea17e12d2e985a026980a016b72acacb30244a76 new file mode 100644 index 0000000000000..fea2cd1e94727 Binary files /dev/null and b/lexer/fuzzer_corpus/ea17e12d2e985a026980a016b72acacb30244a76 differ diff --git a/lexer/fuzzer_corpus/ea69aad1620b5025cc85f415080cd799b66ebc68 b/lexer/fuzzer_corpus/ea69aad1620b5025cc85f415080cd799b66ebc68 new file mode 100644 index 0000000000000..a25e1affc6876 Binary files /dev/null and b/lexer/fuzzer_corpus/ea69aad1620b5025cc85f415080cd799b66ebc68 differ diff --git a/lexer/fuzzer_corpus/ea6da9d77a360c4826d1a29c2bf40e9209bc1c0b b/lexer/fuzzer_corpus/ea6da9d77a360c4826d1a29c2bf40e9209bc1c0b new file mode 100644 index 0000000000000..ae5fa74178501 Binary files /dev/null and b/lexer/fuzzer_corpus/ea6da9d77a360c4826d1a29c2bf40e9209bc1c0b differ diff --git a/lexer/fuzzer_corpus/eb2f5a2dd2fee2ef20a54406427e10043b345485 b/lexer/fuzzer_corpus/eb2f5a2dd2fee2ef20a54406427e10043b345485 new file mode 100644 index 0000000000000..3e3df4f89e5fe Binary files /dev/null and b/lexer/fuzzer_corpus/eb2f5a2dd2fee2ef20a54406427e10043b345485 differ diff --git a/lexer/fuzzer_corpus/eb6be60b4aa6b17f16953f6095b6ea7efa92f6d4 b/lexer/fuzzer_corpus/eb6be60b4aa6b17f16953f6095b6ea7efa92f6d4 new file mode 100644 index 0000000000000..0b61ba76c5b8e Binary files /dev/null and b/lexer/fuzzer_corpus/eb6be60b4aa6b17f16953f6095b6ea7efa92f6d4 differ diff --git a/lexer/fuzzer_corpus/ec34b45a65e9bad07d332490983b305e6f1d358b b/lexer/fuzzer_corpus/ec34b45a65e9bad07d332490983b305e6f1d358b new file mode 100644 index 0000000000000..7e8807a84dad0 Binary files /dev/null and b/lexer/fuzzer_corpus/ec34b45a65e9bad07d332490983b305e6f1d358b differ diff --git a/lexer/fuzzer_corpus/ec6905048c1682c644f01e41531ab3d87e0a2247 b/lexer/fuzzer_corpus/ec6905048c1682c644f01e41531ab3d87e0a2247 new file mode 100644 index 0000000000000..7eb644f4062a2 Binary files /dev/null and b/lexer/fuzzer_corpus/ec6905048c1682c644f01e41531ab3d87e0a2247 differ diff --git a/lexer/fuzzer_corpus/ed2df06c65bbfdf156d4afc88bdc66d9d3290613 b/lexer/fuzzer_corpus/ed2df06c65bbfdf156d4afc88bdc66d9d3290613 new file mode 100644 index 0000000000000..d8685a076f82f Binary files /dev/null and b/lexer/fuzzer_corpus/ed2df06c65bbfdf156d4afc88bdc66d9d3290613 differ diff --git a/lexer/fuzzer_corpus/ed93b3caa649c0dc8d2e7bc2fa885ecd7ad0ba36 b/lexer/fuzzer_corpus/ed93b3caa649c0dc8d2e7bc2fa885ecd7ad0ba36 new file mode 100644 index 0000000000000..e25b51e534a1f Binary files /dev/null and b/lexer/fuzzer_corpus/ed93b3caa649c0dc8d2e7bc2fa885ecd7ad0ba36 differ diff --git a/lexer/fuzzer_corpus/edd491a75e971973bd4cd9dc84a0023cebd62f92 b/lexer/fuzzer_corpus/edd491a75e971973bd4cd9dc84a0023cebd62f92 new file mode 100644 index 0000000000000..f9f7103313c28 Binary files /dev/null and b/lexer/fuzzer_corpus/edd491a75e971973bd4cd9dc84a0023cebd62f92 differ diff --git a/lexer/fuzzer_corpus/edf2f1a4740661b72760a302a261518fa842bde0 b/lexer/fuzzer_corpus/edf2f1a4740661b72760a302a261518fa842bde0 new file mode 100644 index 0000000000000..fb4c62b18cb25 Binary files /dev/null and b/lexer/fuzzer_corpus/edf2f1a4740661b72760a302a261518fa842bde0 differ diff --git a/lexer/fuzzer_corpus/eee58a61a1959843405fb465464ec698f6181a6f b/lexer/fuzzer_corpus/eee58a61a1959843405fb465464ec698f6181a6f new file mode 100644 index 0000000000000..4256e29819dd3 Binary files /dev/null and b/lexer/fuzzer_corpus/eee58a61a1959843405fb465464ec698f6181a6f differ diff --git a/lexer/fuzzer_corpus/ef6721b59164581f5525e2e2f16bbdb626de6ba7 b/lexer/fuzzer_corpus/ef6721b59164581f5525e2e2f16bbdb626de6ba7 new file mode 100644 index 0000000000000..4723ed3591848 Binary files /dev/null and b/lexer/fuzzer_corpus/ef6721b59164581f5525e2e2f16bbdb626de6ba7 differ diff --git a/lexer/fuzzer_corpus/f05b2bd236751d7863172a380f5ab4315344968e b/lexer/fuzzer_corpus/f05b2bd236751d7863172a380f5ab4315344968e new file mode 100644 index 0000000000000..02c0f1cbc4046 Binary files /dev/null and b/lexer/fuzzer_corpus/f05b2bd236751d7863172a380f5ab4315344968e differ diff --git a/lexer/fuzzer_corpus/f0c16a24aba99a02eb9924dd4a4725ab8e92ffad b/lexer/fuzzer_corpus/f0c16a24aba99a02eb9924dd4a4725ab8e92ffad new file mode 100644 index 0000000000000..9648d0406caa0 Binary files /dev/null and b/lexer/fuzzer_corpus/f0c16a24aba99a02eb9924dd4a4725ab8e92ffad differ diff --git a/lexer/fuzzer_corpus/f10576d1e2fb48314b8dfbfbeb5d708defe6097b b/lexer/fuzzer_corpus/f10576d1e2fb48314b8dfbfbeb5d708defe6097b new file mode 100644 index 0000000000000..aa081d4ae33ae Binary files /dev/null and b/lexer/fuzzer_corpus/f10576d1e2fb48314b8dfbfbeb5d708defe6097b differ diff --git a/lexer/fuzzer_corpus/f17479189c9591f3c73906ed84b33b87ac6e8e1f b/lexer/fuzzer_corpus/f17479189c9591f3c73906ed84b33b87ac6e8e1f new file mode 100644 index 0000000000000..e28054e5624bd Binary files /dev/null and b/lexer/fuzzer_corpus/f17479189c9591f3c73906ed84b33b87ac6e8e1f differ diff --git a/lexer/fuzzer_corpus/f1778e9e69871a38d2ce4d8b7c4a3e0b35ece46a b/lexer/fuzzer_corpus/f1778e9e69871a38d2ce4d8b7c4a3e0b35ece46a new file mode 100644 index 0000000000000..4afddd260071e Binary files /dev/null and b/lexer/fuzzer_corpus/f1778e9e69871a38d2ce4d8b7c4a3e0b35ece46a differ diff --git a/lexer/fuzzer_corpus/f2e3dc7f73104b527baf917360356e7994267cdd b/lexer/fuzzer_corpus/f2e3dc7f73104b527baf917360356e7994267cdd new file mode 100644 index 0000000000000..7b5bb94341d15 Binary files /dev/null and b/lexer/fuzzer_corpus/f2e3dc7f73104b527baf917360356e7994267cdd differ diff --git a/lexer/fuzzer_corpus/f3b6007923c6e42984caf9c5fb239cf38815e841 b/lexer/fuzzer_corpus/f3b6007923c6e42984caf9c5fb239cf38815e841 new file mode 100644 index 0000000000000..76b6937b7028c Binary files /dev/null and b/lexer/fuzzer_corpus/f3b6007923c6e42984caf9c5fb239cf38815e841 differ diff --git a/lexer/fuzzer_corpus/f3da8335de7de651461ad28357d506e940d18bdf b/lexer/fuzzer_corpus/f3da8335de7de651461ad28357d506e940d18bdf new file mode 100644 index 0000000000000..cb5a33698db0e Binary files /dev/null and b/lexer/fuzzer_corpus/f3da8335de7de651461ad28357d506e940d18bdf differ diff --git a/lexer/fuzzer_corpus/f40831624c06f31927bd2af861887ad578246010 b/lexer/fuzzer_corpus/f40831624c06f31927bd2af861887ad578246010 new file mode 100644 index 0000000000000..d6331f72e067c Binary files /dev/null and b/lexer/fuzzer_corpus/f40831624c06f31927bd2af861887ad578246010 differ diff --git a/lexer/fuzzer_corpus/f4ddda0ae8524f4f632d5d03291b59acc1952a74 b/lexer/fuzzer_corpus/f4ddda0ae8524f4f632d5d03291b59acc1952a74 new file mode 100644 index 0000000000000..64c06eef4c52a Binary files /dev/null and b/lexer/fuzzer_corpus/f4ddda0ae8524f4f632d5d03291b59acc1952a74 differ diff --git a/lexer/fuzzer_corpus/f53e9d74f34d349f856d8aba3f40712bcfe8d916 b/lexer/fuzzer_corpus/f53e9d74f34d349f856d8aba3f40712bcfe8d916 new file mode 100644 index 0000000000000..a56654412bd14 Binary files /dev/null and b/lexer/fuzzer_corpus/f53e9d74f34d349f856d8aba3f40712bcfe8d916 differ diff --git a/lexer/fuzzer_corpus/f5581adcf52c7acfd5fa26cbb317231aff1d9ed7 b/lexer/fuzzer_corpus/f5581adcf52c7acfd5fa26cbb317231aff1d9ed7 new file mode 100644 index 0000000000000..4c2860cb90b0f Binary files /dev/null and b/lexer/fuzzer_corpus/f5581adcf52c7acfd5fa26cbb317231aff1d9ed7 differ diff --git a/lexer/fuzzer_corpus/f57bb62ad6d29b8f4d1fb3baa4474f3af4e0cbab b/lexer/fuzzer_corpus/f57bb62ad6d29b8f4d1fb3baa4474f3af4e0cbab new file mode 100644 index 0000000000000..0a561b0e2f2e6 Binary files /dev/null and b/lexer/fuzzer_corpus/f57bb62ad6d29b8f4d1fb3baa4474f3af4e0cbab differ diff --git a/lexer/fuzzer_corpus/f6117176339050e85b72129ff118f2713b118931 b/lexer/fuzzer_corpus/f6117176339050e85b72129ff118f2713b118931 new file mode 100644 index 0000000000000..5713182d7f510 Binary files /dev/null and b/lexer/fuzzer_corpus/f6117176339050e85b72129ff118f2713b118931 differ diff --git a/lexer/fuzzer_corpus/f66e211bfde320bf50efe03fde1cbbbf5df440c6 b/lexer/fuzzer_corpus/f66e211bfde320bf50efe03fde1cbbbf5df440c6 new file mode 100644 index 0000000000000..60419111f276d Binary files /dev/null and b/lexer/fuzzer_corpus/f66e211bfde320bf50efe03fde1cbbbf5df440c6 differ diff --git a/lexer/fuzzer_corpus/f6a16230ec874c52b6c1cd73ccc0b9dee2dd5574 b/lexer/fuzzer_corpus/f6a16230ec874c52b6c1cd73ccc0b9dee2dd5574 new file mode 100644 index 0000000000000..98d9305e7a606 Binary files /dev/null and b/lexer/fuzzer_corpus/f6a16230ec874c52b6c1cd73ccc0b9dee2dd5574 differ diff --git a/lexer/fuzzer_corpus/f7014162c0e7c8229146243ca875cec5757f6b39 b/lexer/fuzzer_corpus/f7014162c0e7c8229146243ca875cec5757f6b39 new file mode 100644 index 0000000000000..62b991c65f92c Binary files /dev/null and b/lexer/fuzzer_corpus/f7014162c0e7c8229146243ca875cec5757f6b39 differ diff --git a/lexer/fuzzer_corpus/f7e31e99e09ddb661dc4a5f639f48baa48e233fb b/lexer/fuzzer_corpus/f7e31e99e09ddb661dc4a5f639f48baa48e233fb new file mode 100644 index 0000000000000..158c8f5a349e4 Binary files /dev/null and b/lexer/fuzzer_corpus/f7e31e99e09ddb661dc4a5f639f48baa48e233fb differ diff --git a/lexer/fuzzer_corpus/f9373536de828c70789357df4be4069df64d5dc2 b/lexer/fuzzer_corpus/f9373536de828c70789357df4be4069df64d5dc2 new file mode 100644 index 0000000000000..32636ee6306fc Binary files /dev/null and b/lexer/fuzzer_corpus/f9373536de828c70789357df4be4069df64d5dc2 differ diff --git a/lexer/fuzzer_corpus/f9a0b5a03a08526b1622f57bf3cefae54eb49abd b/lexer/fuzzer_corpus/f9a0b5a03a08526b1622f57bf3cefae54eb49abd new file mode 100644 index 0000000000000..002186f3d796e Binary files /dev/null and b/lexer/fuzzer_corpus/f9a0b5a03a08526b1622f57bf3cefae54eb49abd differ diff --git a/lexer/fuzzer_corpus/f9a6502095aa770e7213f96511a3d63f33757674 b/lexer/fuzzer_corpus/f9a6502095aa770e7213f96511a3d63f33757674 new file mode 100644 index 0000000000000..d53ce0629ff13 Binary files /dev/null and b/lexer/fuzzer_corpus/f9a6502095aa770e7213f96511a3d63f33757674 differ diff --git a/lexer/fuzzer_corpus/f9b7f4ad29450f6bdf1baf2663c7b86a7ec39cfc b/lexer/fuzzer_corpus/f9b7f4ad29450f6bdf1baf2663c7b86a7ec39cfc new file mode 100644 index 0000000000000..431f131a7469b Binary files /dev/null and b/lexer/fuzzer_corpus/f9b7f4ad29450f6bdf1baf2663c7b86a7ec39cfc differ diff --git a/lexer/fuzzer_corpus/faa689e2f7b3c3877c1581ae943cffa5c037ffbf b/lexer/fuzzer_corpus/faa689e2f7b3c3877c1581ae943cffa5c037ffbf new file mode 100644 index 0000000000000..d2c6264467cde Binary files /dev/null and b/lexer/fuzzer_corpus/faa689e2f7b3c3877c1581ae943cffa5c037ffbf differ diff --git a/lexer/fuzzer_corpus/facca3ae362b2fab961eb9e724b3790b1977d0a8 b/lexer/fuzzer_corpus/facca3ae362b2fab961eb9e724b3790b1977d0a8 new file mode 100644 index 0000000000000..0eb2647757571 Binary files /dev/null and b/lexer/fuzzer_corpus/facca3ae362b2fab961eb9e724b3790b1977d0a8 differ diff --git a/lexer/fuzzer_corpus/fbe65ad556a0f0fe9b8215385e70da5555a3fba1 b/lexer/fuzzer_corpus/fbe65ad556a0f0fe9b8215385e70da5555a3fba1 new file mode 100644 index 0000000000000..9bb31f58c9a2a Binary files /dev/null and b/lexer/fuzzer_corpus/fbe65ad556a0f0fe9b8215385e70da5555a3fba1 differ diff --git a/lexer/fuzzer_corpus/fc880c8c473a9d0b14bb5827260c1170e8ca9535 b/lexer/fuzzer_corpus/fc880c8c473a9d0b14bb5827260c1170e8ca9535 new file mode 100644 index 0000000000000..961ecc2e21274 Binary files /dev/null and b/lexer/fuzzer_corpus/fc880c8c473a9d0b14bb5827260c1170e8ca9535 differ diff --git a/lexer/fuzzer_corpus/fc96ab8937b4d16598d2831e4d1f89df4b720d72 b/lexer/fuzzer_corpus/fc96ab8937b4d16598d2831e4d1f89df4b720d72 new file mode 100644 index 0000000000000..f578dc9df90fd Binary files /dev/null and b/lexer/fuzzer_corpus/fc96ab8937b4d16598d2831e4d1f89df4b720d72 differ diff --git a/lexer/fuzzer_corpus/fcc91d6415ebf086ccc8b84ef729d5099cc19a84 b/lexer/fuzzer_corpus/fcc91d6415ebf086ccc8b84ef729d5099cc19a84 new file mode 100644 index 0000000000000..c2428eb03bef1 Binary files /dev/null and b/lexer/fuzzer_corpus/fcc91d6415ebf086ccc8b84ef729d5099cc19a84 differ diff --git a/lexer/fuzzer_corpus/fd958cb8a73d5d70d0be71a387c7973c3b1df2e9 b/lexer/fuzzer_corpus/fd958cb8a73d5d70d0be71a387c7973c3b1df2e9 new file mode 100644 index 0000000000000..898f0737218f4 Binary files /dev/null and b/lexer/fuzzer_corpus/fd958cb8a73d5d70d0be71a387c7973c3b1df2e9 differ diff --git a/lexer/fuzzer_corpus/fdce145b982b47c9598cfe55883672e86727d5b8 b/lexer/fuzzer_corpus/fdce145b982b47c9598cfe55883672e86727d5b8 new file mode 100644 index 0000000000000..615ce7d56136e Binary files /dev/null and b/lexer/fuzzer_corpus/fdce145b982b47c9598cfe55883672e86727d5b8 differ diff --git a/lexer/fuzzer_corpus/fe9a584f5c7c5d9eb5e4b15f85960ca7f1079a2e b/lexer/fuzzer_corpus/fe9a584f5c7c5d9eb5e4b15f85960ca7f1079a2e new file mode 100644 index 0000000000000..678903a0403e1 Binary files /dev/null and b/lexer/fuzzer_corpus/fe9a584f5c7c5d9eb5e4b15f85960ca7f1079a2e differ diff --git a/lexer/fuzzer_corpus/fecd4b2d80be6971216cd554f44793b3b8adaba6 b/lexer/fuzzer_corpus/fecd4b2d80be6971216cd554f44793b3b8adaba6 new file mode 100644 index 0000000000000..dc9be270d23d3 Binary files /dev/null and b/lexer/fuzzer_corpus/fecd4b2d80be6971216cd554f44793b3b8adaba6 differ diff --git a/lexer/fuzzer_corpus/fef279c42c269b2a38ccfbf781f73897284ec2b2 b/lexer/fuzzer_corpus/fef279c42c269b2a38ccfbf781f73897284ec2b2 new file mode 100644 index 0000000000000..12cb389136e18 Binary files /dev/null and b/lexer/fuzzer_corpus/fef279c42c269b2a38ccfbf781f73897284ec2b2 differ diff --git a/lexer/fuzzer_corpus/ff5f77350f4850c12dc9576ea268de27bb55ade1 b/lexer/fuzzer_corpus/ff5f77350f4850c12dc9576ea268de27bb55ade1 new file mode 100644 index 0000000000000..f856ab875de2b Binary files /dev/null and b/lexer/fuzzer_corpus/ff5f77350f4850c12dc9576ea268de27bb55ade1 differ diff --git a/lexer/fuzzer_corpus/ffcfd70e30e22415a17a3152c0ba1cc0f9f756a3 b/lexer/fuzzer_corpus/ffcfd70e30e22415a17a3152c0ba1cc0f9f756a3 new file mode 100644 index 0000000000000..43e972674483f Binary files /dev/null and b/lexer/fuzzer_corpus/ffcfd70e30e22415a17a3152c0ba1cc0f9f756a3 differ diff --git a/lexer/token_kind.cpp b/lexer/token_kind.cpp new file mode 100644 index 0000000000000..7aba5c216382f --- /dev/null +++ b/lexer/token_kind.cpp @@ -0,0 +1,103 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include "lexer/token_kind.h" + +#include "llvm/ADT/StringRef.h" + +namespace Carbon { + +auto TokenKind::Name() const -> llvm::StringRef { + static constexpr llvm::StringLiteral Names[] = { +#define CARBON_TOKEN(TokenName) #TokenName, +#include "lexer/token_registry.def" + }; + return Names[static_cast(kind_value)]; +} + +auto TokenKind::IsSymbol() const -> bool { + static constexpr bool Table[] = { +#define CARBON_TOKEN(TokenName) false, +#define CARBON_SYMBOL_TOKEN(TokenName, Spelling) true, +#include "lexer/token_registry.def" + }; + return Table[static_cast(kind_value)]; +} + +auto TokenKind::IsGroupingSymbol() const -> bool { + static constexpr bool Table[] = { +#define CARBON_TOKEN(TokenName) false, +#define CARBON_OPENING_GROUP_SYMBOL_TOKEN(TokenName, Spelling, ClosingName) \ + true, +#define CARBON_CLOSING_GROUP_SYMBOL_TOKEN(TokenName, Spelling, OpeningName) \ + true, +#include "lexer/token_registry.def" + }; + return Table[static_cast(kind_value)]; +} + +auto TokenKind::IsOpeningSymbol() const -> bool { + static constexpr bool Table[] = { +#define CARBON_TOKEN(TokenName) false, +#define CARBON_OPENING_GROUP_SYMBOL_TOKEN(TokenName, Spelling, ClosingName) \ + true, +#include "lexer/token_registry.def" + }; + return Table[static_cast(kind_value)]; +} + +auto TokenKind::GetClosingSymbol() const -> TokenKind { + static constexpr TokenKind Table[] = { +#define CARBON_TOKEN(TokenName) Error(), +#define CARBON_OPENING_GROUP_SYMBOL_TOKEN(TokenName, Spelling, ClosingName) \ + ClosingName(), +#include "lexer/token_registry.def" + }; + auto result = Table[static_cast(kind_value)]; + assert(result != Error() && "Only opening symbols are valid!"); + return result; +} + +auto TokenKind::IsClosingSymbol() const -> bool { + static constexpr bool Table[] = { +#define CARBON_TOKEN(TokenName) false, +#define CARBON_CLOSING_GROUP_SYMBOL_TOKEN(TokenName, Spelling, OpeningName) \ + true, +#include "lexer/token_registry.def" + }; + return Table[static_cast(kind_value)]; +} + +auto TokenKind::GetOpeningSymbol() const -> TokenKind { + static constexpr TokenKind Table[] = { +#define CARBON_TOKEN(TokenName) Error(), +#define CARBON_CLOSING_GROUP_SYMBOL_TOKEN(TokenName, Spelling, OpeningName) \ + OpeningName(), +#include "lexer/token_registry.def" + }; + auto result = Table[static_cast(kind_value)]; + assert(result != Error() && "Only closing symbols are valid!"); + return result; +} + +auto TokenKind::IsKeyword() const -> bool { + static constexpr bool Table[] = { +#define CARBON_TOKEN(TokenName) false, +#define CARBON_KEYWORD_TOKEN(TokenName, Spelling) true, +#include "lexer/token_registry.def" + }; + return Table[static_cast(kind_value)]; +} + +auto TokenKind::GetFixedSpelling() const -> llvm::StringRef { + static constexpr llvm::StringLiteral Table[] = { +#define CARBON_TOKEN(TokenName) "", +#define CARBON_SYMBOL_TOKEN(TokenName, Spelling) Spelling, +#define CARBON_KEYWORD_TOKEN(TokenName, Spelling) Spelling, +#include "lexer/token_registry.def" + }; + return Table[static_cast(kind_value)]; +} + +} // namespace Carbon diff --git a/lexer/token_kind.h b/lexer/token_kind.h new file mode 100644 index 0000000000000..69dfbe1af1c52 --- /dev/null +++ b/lexer/token_kind.h @@ -0,0 +1,88 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#ifndef LEXER_TOKEN_KIND_H_ +#define LEXER_TOKEN_KIND_H_ + +#include + +#include + +#include "llvm/ADT/StringRef.h" + +namespace Carbon { + +class TokenKind { + enum class KindEnum : int8_t { +#define CARBON_TOKEN(TokenName) TokenName, +#include "lexer/token_registry.def" + }; + + public: +#define CARBON_TOKEN(TokenName) \ + static constexpr auto TokenName()->TokenKind { return KindEnum::TokenName; } +#include "lexer/token_registry.def" + + // The default constructor is deleted as objects of this type should always be + // constructed using the above factory functions for each unique kind. + TokenKind() = delete; + + auto operator==(const TokenKind& rhs) const -> bool { + return kind_value == rhs.kind_value; + } + auto operator!=(const TokenKind& rhs) const -> bool { + return kind_value != rhs.kind_value; + } + + // Get a friendly name for the token for logging or debugging. + auto Name() const -> llvm::StringRef; + + // Test whether this kind of token is a simple symbol sequence (punctuation, + // not letters) that appears directly in the source text and can be + // unambiguously lexed with `starts_with` logic. While these may appear + // inside of other tokens, outside of the contents of other tokens they + // don't require any specific characters before or after to distinguish them + // in the source. Returns false otherwise. + auto IsSymbol() const -> bool; + + // Test whether this kind of token is a grouping symbol (part of an opening + // and closing pair that must always be matched in the token stream). + auto IsGroupingSymbol() const -> bool; + + // Test whether this kind of token is an opening symbol for a group. + auto IsOpeningSymbol() const -> bool; + + // Returns the associated closing symbol for an opening symbol. + // + // The token kind must be an opening symbol. + auto GetClosingSymbol() const -> TokenKind; + + // Test whether this kind of token is an closing symbol for a group. + auto IsClosingSymbol() const -> bool; + + // Returns the associated opening symbol for a closing symbol. + // + // The token kind must be an closing symbol. + auto GetOpeningSymbol() const -> TokenKind; + + // Test whether this kind of token is a keyword. + auto IsKeyword() const -> bool; + + // If this token kind has a fixed spelling when in source code, returns it. + // Otherwise returns an empty string. + auto GetFixedSpelling() const -> llvm::StringRef; + + // Enable implicit conversion to an int, including in a `constexpr` context, + // to enable usage in `switch` and `case`. + constexpr operator int() const { return static_cast(kind_value); } + + private: + constexpr TokenKind(KindEnum kind_value) : kind_value(kind_value) {} + + KindEnum kind_value; +}; + +} // namespace Carbon + +#endif // LEXER_TOKEN_KIND_H_ diff --git a/lexer/token_kind_test.cpp b/lexer/token_kind_test.cpp new file mode 100644 index 0000000000000..c803a0441326f --- /dev/null +++ b/lexer/token_kind_test.cpp @@ -0,0 +1,93 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include "lexer/token_kind.h" + +#include + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "llvm/ADT/StringRef.h" + +namespace Carbon { +namespace { + +using ::testing::MatchesRegex; + +// We restrict symbols to punctuation characters that are expected to be widely +// available on modern keyboards used for programming. +constexpr llvm::StringLiteral SymbolRegex = "[][{}!@#%^&*()/?\\|;:.,<>=+~-]+"; + +// We restrict keywords to be lowercase ASCII letters and underscores. +constexpr llvm::StringLiteral KeywordRegex = "[a-z_]+"; + +#define CARBON_TOKEN(TokenName) \ + TEST(TokenKindTest, TokenName) { \ + EXPECT_EQ(#TokenName, TokenKind::TokenName().Name()); \ + EXPECT_FALSE(TokenKind::TokenName().IsSymbol()); \ + EXPECT_FALSE(TokenKind::TokenName().IsKeyword()); \ + EXPECT_EQ("", TokenKind::TokenName().GetFixedSpelling()); \ + } +#define CARBON_SYMBOL_TOKEN(TokenName, Spelling) \ + TEST(TokenKindTest, TokenName) { \ + EXPECT_EQ(#TokenName, TokenKind::TokenName().Name()); \ + EXPECT_TRUE(TokenKind::TokenName().IsSymbol()); \ + EXPECT_FALSE(TokenKind::TokenName().IsGroupingSymbol()); \ + EXPECT_FALSE(TokenKind::TokenName().IsOpeningSymbol()); \ + EXPECT_FALSE(TokenKind::TokenName().IsClosingSymbol()); \ + EXPECT_FALSE(TokenKind::TokenName().IsKeyword()); \ + EXPECT_EQ(Spelling, TokenKind::TokenName().GetFixedSpelling()); \ + EXPECT_THAT(Spelling, MatchesRegex(SymbolRegex.str())); \ + } +#define CARBON_OPENING_GROUP_SYMBOL_TOKEN(TokenName, Spelling, ClosingName) \ + TEST(TokenKindTest, TokenName) { \ + EXPECT_EQ(#TokenName, TokenKind::TokenName().Name()); \ + EXPECT_TRUE(TokenKind::TokenName().IsSymbol()); \ + EXPECT_TRUE(TokenKind::TokenName().IsGroupingSymbol()); \ + EXPECT_TRUE(TokenKind::TokenName().IsOpeningSymbol()); \ + EXPECT_EQ(TokenKind::ClosingName(), \ + TokenKind::TokenName().GetClosingSymbol()); \ + EXPECT_FALSE(TokenKind::TokenName().IsClosingSymbol()); \ + EXPECT_FALSE(TokenKind::TokenName().IsKeyword()); \ + EXPECT_EQ(Spelling, TokenKind::TokenName().GetFixedSpelling()); \ + EXPECT_THAT(Spelling, MatchesRegex(SymbolRegex.str())); \ + } +#define CARBON_CLOSING_GROUP_SYMBOL_TOKEN(TokenName, Spelling, OpeningName) \ + TEST(TokenKindTest, TokenName) { \ + EXPECT_EQ(#TokenName, TokenKind::TokenName().Name()); \ + EXPECT_TRUE(TokenKind::TokenName().IsSymbol()); \ + EXPECT_TRUE(TokenKind::TokenName().IsGroupingSymbol()); \ + EXPECT_FALSE(TokenKind::TokenName().IsOpeningSymbol()); \ + EXPECT_TRUE(TokenKind::TokenName().IsClosingSymbol()); \ + EXPECT_EQ(TokenKind::OpeningName(), \ + TokenKind::TokenName().GetOpeningSymbol()); \ + EXPECT_FALSE(TokenKind::TokenName().IsKeyword()); \ + EXPECT_EQ(Spelling, TokenKind::TokenName().GetFixedSpelling()); \ + EXPECT_THAT(Spelling, MatchesRegex(SymbolRegex.str())); \ + } +#define CARBON_KEYWORD_TOKEN(TokenName, Spelling) \ + TEST(TokenKindTest, TokenName) { \ + EXPECT_EQ(#TokenName, TokenKind::TokenName().Name()); \ + EXPECT_FALSE(TokenKind::TokenName().IsSymbol()); \ + EXPECT_TRUE(TokenKind::TokenName().IsKeyword()); \ + EXPECT_EQ(Spelling, TokenKind::TokenName().GetFixedSpelling()); \ + EXPECT_THAT(Spelling, MatchesRegex(KeywordRegex.str())); \ + } +#include "lexer/token_registry.def" + +// Verify that the symbol tokens are sorted from longest to shortest. This is +// important to ensure that simply in-order testing will identify tokens +// following the max-munch rule. +TEST(TokenKindTest, SymbolsInDescendingLength) { + int previous_length = INT_MAX; +#define CARBON_SYMBOL_TOKEN(TokenName, Spelling) \ + EXPECT_LE(llvm::StringRef(Spelling).size(), previous_length) \ + << "Symbol token not in descending length order: " << #TokenName; \ + previous_length = llvm::StringRef(Spelling).size(); +#include "lexer/token_registry.def" + EXPECT_GT(previous_length, 0); +} + +} // namespace +} // namespace Carbon diff --git a/lexer/token_registry.def b/lexer/token_registry.def new file mode 100644 index 0000000000000..48c34b58a3563 --- /dev/null +++ b/lexer/token_registry.def @@ -0,0 +1,162 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +// Note that this is an X-macro header. +// +// It does not use `#include` guards, and instead is designed to be `#include`ed +// after some set of x-macros are defined in order for its inclusion to expand +// to the desired output. +// +// The viable X-macros to define prior to including the header are: +// +// - `CARBON_TOKEN` +// - `CARBON_SYMBOL_TOKEN` +// - `CARBON_OPENING_GROUP_SYMBOL_TOKEN` +// - `CARBON_CLOSING_GROUP_SYMBOL_TOKEN` +// - `CARBON_KEYWORD_TOKEN` +// +// This tree represents the subset relationship between these macros. +// +// Regardless of which of these macros is defined prior to inclusion, all of the +// definitions will be removed at the end of including this file to clean up the +// set of defined macros. + +#ifndef CARBON_TOKEN +#define CARBON_TOKEN(Name) +#endif + +#ifndef CARBON_SYMBOL_TOKEN +#define CARBON_SYMBOL_TOKEN(Name, Spelling) CARBON_TOKEN(Name) +#endif + +// Note that symbols need to be ordered from longest to shortest to effectively +// provide max-munch lexing. + +// clang-format off + +CARBON_SYMBOL_TOKEN(GreaterGreaterEqual, ">>=") +CARBON_SYMBOL_TOKEN(LessEqualGreater, "<=>") +CARBON_SYMBOL_TOKEN(LessLessEqual, "<<=") + +CARBON_SYMBOL_TOKEN(AmpEqual, "&=") +CARBON_SYMBOL_TOKEN(CaretEqual, "^=") +CARBON_SYMBOL_TOKEN(ColonEqual, ":=") +CARBON_SYMBOL_TOKEN(EqualEqual, "==") +CARBON_SYMBOL_TOKEN(EqualGreater, "=>") +CARBON_SYMBOL_TOKEN(ExclaimEqual, "!=") +CARBON_SYMBOL_TOKEN(GreaterEqual, ">=") +CARBON_SYMBOL_TOKEN(GreaterGreater, ">>") +CARBON_SYMBOL_TOKEN(LessEqual, "<=") +CARBON_SYMBOL_TOKEN(LessGreater, "<>") +CARBON_SYMBOL_TOKEN(LessLess, "<<") +CARBON_SYMBOL_TOKEN(LessMinus, "<-") +CARBON_SYMBOL_TOKEN(MinusEqual, "-=") +CARBON_SYMBOL_TOKEN(MinusGreater, "->") +CARBON_SYMBOL_TOKEN(MinusMinus, "--") +CARBON_SYMBOL_TOKEN(PercentEqual, "%=") +CARBON_SYMBOL_TOKEN(PipeEqual, "|=") +CARBON_SYMBOL_TOKEN(PlusEqual, "+=") +CARBON_SYMBOL_TOKEN(PlusPlus, "++") +CARBON_SYMBOL_TOKEN(SlashEqual, "/=") +CARBON_SYMBOL_TOKEN(StarEqual, "*=") +CARBON_SYMBOL_TOKEN(TildeEqual, "~=") + +CARBON_SYMBOL_TOKEN(Amp, "&") +CARBON_SYMBOL_TOKEN(At, "@") +CARBON_SYMBOL_TOKEN(Backslash, "\\") +CARBON_SYMBOL_TOKEN(Caret, "^") +CARBON_SYMBOL_TOKEN(Colon, ":") +CARBON_SYMBOL_TOKEN(Comma, ",") +CARBON_SYMBOL_TOKEN(Equal, "=") +CARBON_SYMBOL_TOKEN(Exclaim, "!") +CARBON_SYMBOL_TOKEN(Greater, ">") +CARBON_SYMBOL_TOKEN(Hash, "#") +CARBON_SYMBOL_TOKEN(Less, "<") +CARBON_SYMBOL_TOKEN(Minus, "-") +CARBON_SYMBOL_TOKEN(Percent, "%") +CARBON_SYMBOL_TOKEN(Period, ".") +CARBON_SYMBOL_TOKEN(Pipe, "|") +CARBON_SYMBOL_TOKEN(Plus, "+") +CARBON_SYMBOL_TOKEN(Question, "?") +CARBON_SYMBOL_TOKEN(Semi, ";") +CARBON_SYMBOL_TOKEN(Slash, "/") +CARBON_SYMBOL_TOKEN(Star, "*") +CARBON_SYMBOL_TOKEN(Tilde, "~") + +// clang-format on + +#ifndef CARBON_OPENING_GROUP_SYMBOL_TOKEN +#define CARBON_OPENING_GROUP_SYMBOL_TOKEN(Name, Spelling, ClosingName) \ + CARBON_SYMBOL_TOKEN(Name, Spelling) +#endif +// clang-format on +CARBON_OPENING_GROUP_SYMBOL_TOKEN(OpenParen, "(", CloseParen) +CARBON_OPENING_GROUP_SYMBOL_TOKEN(OpenCurlyBrace, "{", CloseCurlyBrace) +// clang-format off +#undef CARBON_OPENING_GROUP_SYMBOL_TOKEN + +#ifndef CARBON_CLOSING_GROUP_SYMBOL_TOKEN +#define CARBON_CLOSING_GROUP_SYMBOL_TOKEN(Name, Spelling, OpeningName) \ + CARBON_SYMBOL_TOKEN(Name, Spelling) +#endif +// clang-format on +CARBON_CLOSING_GROUP_SYMBOL_TOKEN(CloseParen, ")", OpenParen) +CARBON_CLOSING_GROUP_SYMBOL_TOKEN(CloseCurlyBrace, "}", OpenCurlyBrace) +// clang-format off +#undef CARBON_CLOSING_GROUP_SYMBOL_TOKEN + +#undef CARBON_SYMBOL_TOKEN + +#ifndef CARBON_KEYWORD_TOKEN +#define CARBON_KEYWORD_TOKEN(Name, Spelling) CARBON_TOKEN(Name) +#endif +// clang-format off +CARBON_KEYWORD_TOKEN(AndKeyword, "and") +CARBON_KEYWORD_TOKEN(AsmKeyword, "asm") +CARBON_KEYWORD_TOKEN(BreakKeyword, "break") +CARBON_KEYWORD_TOKEN(CatchKeyword, "catch") +CARBON_KEYWORD_TOKEN(ClassKeyword, "class") +CARBON_KEYWORD_TOKEN(ContinueKeyword, "continue") +CARBON_KEYWORD_TOKEN(ElseKeyword, "else") +CARBON_KEYWORD_TOKEN(EnumKeyword, "enum") +CARBON_KEYWORD_TOKEN(ExternKeyword, "extern") +CARBON_KEYWORD_TOKEN(FinalKeyword, "final") +CARBON_KEYWORD_TOKEN(FixKeyword, "fix") +CARBON_KEYWORD_TOKEN(FnKeyword, "fn") +CARBON_KEYWORD_TOKEN(ForKeyword, "for") +CARBON_KEYWORD_TOKEN(GuardKeyword, "guard") +CARBON_KEYWORD_TOKEN(IfKeyword, "if") +CARBON_KEYWORD_TOKEN(InlineKeyword, "inline") +CARBON_KEYWORD_TOKEN(InoutKeyword, "inout") +CARBON_KEYWORD_TOKEN(InterfaceKeyword, "interface") +CARBON_KEYWORD_TOKEN(InternalKeyword, "internal") +CARBON_KEYWORD_TOKEN(LetKeyword, "let") +CARBON_KEYWORD_TOKEN(LoopKeyword, "loop") +CARBON_KEYWORD_TOKEN(MatchKeyword, "match") +CARBON_KEYWORD_TOKEN(NotKeyword, "not") +CARBON_KEYWORD_TOKEN(OrKeyword, "or") +CARBON_KEYWORD_TOKEN(OutKeyword, "out") +CARBON_KEYWORD_TOKEN(PrivateKeyword, "private") +CARBON_KEYWORD_TOKEN(ProtectedKeyword, "protected") +CARBON_KEYWORD_TOKEN(PublicKeyword, "public") +CARBON_KEYWORD_TOKEN(RefKeyword, "ref") +CARBON_KEYWORD_TOKEN(ReturnKeyword, "return") +CARBON_KEYWORD_TOKEN(StaticKeyword, "static") +CARBON_KEYWORD_TOKEN(StructKeyword, "struct") +CARBON_KEYWORD_TOKEN(ThroKeyword, "throw") +CARBON_KEYWORD_TOKEN(TryKeyword, "try") +CARBON_KEYWORD_TOKEN(UnderscoreKeyword, "_") +CARBON_KEYWORD_TOKEN(VarKeyword, "var") +CARBON_KEYWORD_TOKEN(VirtualKeyword, "virtual") +CARBON_KEYWORD_TOKEN(WhileKeyword, "while") +CARBON_KEYWORD_TOKEN(XorKeyword, "xor") +// clang-format on +#undef CARBON_KEYWORD_TOKEN + +CARBON_TOKEN(Identifier) +CARBON_TOKEN(IntegerLiteral) +CARBON_TOKEN(DocComment) +CARBON_TOKEN(Error) + +#undef CARBON_TOKEN diff --git a/lexer/tokenized_buffer.cpp b/lexer/tokenized_buffer.cpp new file mode 100644 index 0000000000000..c4f9915a81bf6 --- /dev/null +++ b/lexer/tokenized_buffer.cpp @@ -0,0 +1,570 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include "lexer/tokenized_buffer.h" + +#include +#include +#include + +#include "llvm/ADT/StringExtras.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/ADT/StringSwitch.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/Format.h" +#include "llvm/Support/FormatVariadic.h" +#include "llvm/Support/raw_ostream.h" + +namespace Carbon { + +static auto TakeLeadingIntegerLiteral(llvm::StringRef source_text) + -> llvm::StringRef { + return source_text.take_while([](char c) { return llvm::isDigit(c); }); +} + +struct UnmatchedClosing { + static constexpr llvm::StringLiteral ShortName = "syntax-balanced-delimiters"; + static constexpr llvm::StringLiteral Message = + "Closing symbol without a corresponding opening symbol."; + + struct Substitutions {}; + static auto Format(const Substitutions&) -> std::string { + return Message.str(); + } +}; + +struct MismatchedClosing { + static constexpr llvm::StringLiteral ShortName = "syntax-balanced-delimiters"; + static constexpr llvm::StringLiteral Message = + "Closing symbol does not match most recent opening symbol."; + + struct Substitutions {}; + static auto Format(const Substitutions&) -> std::string { + return Message.str(); + } +}; + +struct UnrecognizedCharacters { + static constexpr llvm::StringLiteral ShortName = + "syntax-unrecognized-characters"; + static constexpr llvm::StringLiteral Message = + "Encountered unrecognized characters while parsing."; + + struct Substitutions {}; + static auto Format(const Substitutions&) -> std::string { + return Message.str(); + } +}; + +// Implementation of the lexer logic itself. +// +// The design is that lexing can loop over the source buffer, consuming it into +// tokens by calling into this API. This class handles the state and breaks down +// the different lexing steps that may be used. It directly updates the provided +// tokenized buffer with the lexed tokens. +class TokenizedBuffer::Lexer { + TokenizedBuffer& buffer; + DiagnosticEmitter& emitter; + + Line current_line; + LineInfo* current_line_info; + + int current_column = 0; + bool set_indent = false; + + llvm::SmallVector open_groups; + + public: + Lexer(TokenizedBuffer& buffer, DiagnosticEmitter& emitter) + : buffer(buffer), + emitter(emitter), + current_line(buffer.AddLine({0, 0, 0})), + current_line_info(&buffer.GetLineInfo(current_line)) {} + + auto SkipWhitespace(llvm::StringRef& source_text) -> bool { + while (!source_text.empty()) { + // We only support line-oriented commenting and lex comments as-if they + // were whitespace. Any comment must be the only non-whitespace on the + // line. + if (source_text.startswith("//") && !set_indent) { + // Check if the comment has a special starting sequence of three slashes + // followed by a space. This represents a documentation comment that is + // preserved as a token in the buffer. When parsing, these comments will + // only be accepted in specific parts of the grammar and will be + // associated with the parsed constructs as structure documentation. All + // other comments are simply treated as whitespace. + if (source_text.startswith("///")) { + current_line_info->indent = current_column; + set_indent = true; + buffer.AddToken({.kind = TokenKind::DocComment(), + .token_line = current_line, + .column = current_column}); + } + while (!source_text.empty() && source_text.front() != '\n') { + ++current_column; + source_text = source_text.drop_front(); + } + if (source_text.empty()) + break; + } + + switch (source_text.front()) { + default: + // If we find a non-whitespace character without exhausting the + // buffer, return true to continue lexing. + return true; + + case '\n': + // New lines are special in order to track line structure. + current_line_info->length = current_column; + // If this is the last character in the source, directly return here + // to avoid creating an empty line. + source_text = source_text.drop_front(); + if (source_text.empty()) + return false; + + // Otherwise, add a line and set up to continue lexing. + current_line = buffer.AddLine( + {current_line_info->start + current_column + 1, 0, 0}); + current_line_info = &buffer.GetLineInfo(current_line); + current_column = 0; + set_indent = false; + continue; + + case ' ': + case '\t': + // Skip other forms of whitespace while tracking column. + // FIXME: This obviously needs looooots more work to handle unicode + // whitespace as well as special handling to allow better tokenization + // of operators. This is just a stub to check that our column + // management works. + ++current_column; + source_text = source_text.drop_front(); + continue; + } + } + + assert(source_text.empty() && "Cannot reach here w/o finishing the text!"); + // Update the line length as this is also the end of a line. + current_line_info->length = current_column; + return false; + } + + auto LexIntegerLiteral(llvm::StringRef& source_text) -> bool { + llvm::StringRef int_text = TakeLeadingIntegerLiteral(source_text); + if (int_text.empty()) + return false; + llvm::APInt int_value; + if (int_text.getAsInteger(/*Radix=*/0, int_value)) + return false; + + int int_column = current_column; + current_column += int_text.size(); + source_text = source_text.drop_front(int_text.size()); + + if (!set_indent) { + current_line_info->indent = int_column; + set_indent = true; + } + auto token = buffer.AddToken({.kind = TokenKind::IntegerLiteral(), + .token_line = current_line, + .column = int_column}); + buffer.GetTokenInfo(token).literal_index = buffer.int_literals.size(); + buffer.int_literals.push_back(std::move(int_value)); + return true; + } + + auto LexSymbolToken(llvm::StringRef& source_text) -> bool { + TokenKind kind = llvm::StringSwitch(source_text) +#define CARBON_SYMBOL_TOKEN(Name, Spelling) \ + .StartsWith(Spelling, TokenKind::Name()) +#include "lexer/token_registry.def" + .Default(TokenKind::Error()); + if (kind == TokenKind::Error()) + return false; + + if (!set_indent) { + current_line_info->indent = current_column; + set_indent = true; + } + + CloseInvalidOpenGroups(kind); + + Token token = buffer.AddToken( + {.kind = kind, .token_line = current_line, .column = current_column}); + current_column += kind.GetFixedSpelling().size(); + source_text = source_text.drop_front(kind.GetFixedSpelling().size()); + + // Opening symbols just need to be pushed onto our queue of opening groups. + if (kind.IsOpeningSymbol()) { + open_groups.push_back(token); + return true; + } + + // Only closing symbols need further special handling. + if (!kind.IsClosingSymbol()) + return true; + + TokenInfo& closing_token_info = buffer.GetTokenInfo(token); + + // Check that there is a matching opening symbol before we consume this as + // a closing symbol. + if (open_groups.empty()) { + closing_token_info.kind = TokenKind::Error(); + closing_token_info.error_length = kind.GetFixedSpelling().size(); + buffer.has_errors = true; + + emitter.EmitError( + [](UnmatchedClosing::Substitutions&) {}); + // Note that this still returns true as we do consume a symbol. + return true; + } + + // Finally can handle a normal closing symbol. + Token opening_token = open_groups.pop_back_val(); + TokenInfo& opening_token_info = buffer.GetTokenInfo(opening_token); + opening_token_info.closing_token = token; + closing_token_info.opening_token = opening_token; + return true; + } + + // Closes all open groups that cannot remain open across the symbol `K`. + // Users may pass `Error` to close all open groups. + auto CloseInvalidOpenGroups(TokenKind kind) -> void { + if (!kind.IsClosingSymbol() && kind != TokenKind::Error()) + return; + + while (!open_groups.empty()) { + Token opening_token = open_groups.back(); + TokenKind opening_kind = buffer.GetTokenInfo(opening_token).kind; + if (kind == opening_kind.GetClosingSymbol()) + return; + + open_groups.pop_back(); + buffer.has_errors = true; + emitter.EmitError( + [](MismatchedClosing::Substitutions&) {}); + + // TODO: do a smarter backwards scan for where to put the closing + // token. + Token closing_token = + buffer.AddToken({.kind = opening_kind.GetClosingSymbol(), + .is_recovery = true, + .token_line = current_line, + .column = current_column}); + TokenInfo& opening_token_info = buffer.GetTokenInfo(opening_token); + TokenInfo& closing_token_info = buffer.GetTokenInfo(closing_token); + opening_token_info.closing_token = closing_token; + closing_token_info.opening_token = opening_token; + } + } + + auto GetOrCreateIdentifier(llvm::StringRef text) -> Identifier { + auto insert_result = buffer.identifier_map.insert( + {text, Identifier(buffer.identifier_infos.size())}); + if (insert_result.second) + buffer.identifier_infos.push_back({text}); + return insert_result.first->second; + } + + auto LexKeywordOrIdentifier(llvm::StringRef& source_text) -> bool { + if (!llvm::isAlpha(source_text.front()) && source_text.front() != '_') + return false; + + if (!set_indent) { + current_line_info->indent = current_column; + set_indent = true; + } + + // Take the valid characters off the front of the source buffer. + llvm::StringRef identifier_text = source_text.take_while( + [](char c) { return llvm::isAlnum(c) || c == '_'; }); + assert(!identifier_text.empty() && "Must have at least one character!"); + int identifier_column = current_column; + current_column += identifier_text.size(); + source_text = source_text.drop_front(identifier_text.size()); + + // Check if the text matches a keyword token, and if so use that. + TokenKind kind = llvm::StringSwitch(identifier_text) +#define CARBON_KEYWORD_TOKEN(Name, Spelling) .Case(Spelling, TokenKind::Name()) +#include "lexer/token_registry.def" + .Default(TokenKind::Error()); + if (kind != TokenKind::Error()) { + buffer.AddToken({.kind = kind, + .token_line = current_line, + .column = identifier_column}); + return true; + } + + // Otherwise we have a generic identifier. + buffer.AddToken({.kind = TokenKind::Identifier(), + .token_line = current_line, + .column = identifier_column, + .id = GetOrCreateIdentifier(identifier_text)}); + return true; + } + + auto LexError(llvm::StringRef& source_text) -> void { + llvm::StringRef error_text = source_text.take_while([](char c) { + if (llvm::isAlnum(c)) + return false; + switch (c) { + case '_': + return false; + case '\t': + return false; + case '\n': + return false; + } + return llvm::StringSwitch(llvm::StringRef(&c, 1)) +#define CARBON_SYMBOL_TOKEN(Name, Spelling) .StartsWith(Spelling, false) +#include "lexer/token_registry.def" + .Default(true); + }); + if (error_text.empty()) { + // TODO: Reimplement this to use the lexer properly. In the meantime, + // guarantee that we eat at least one byte. + error_text = source_text.take_front(1); + } + + // Longer errors get to be two tokens. + error_text = error_text.substr(0, std::numeric_limits::max()); + auto token = buffer.AddToken( + {.kind = TokenKind::Error(), + .token_line = current_line, + .column = current_column, + .error_length = static_cast(error_text.size())}); + // TODO: #19 - Need to convert to the diagnostics library. + llvm::errs() << "ERROR: Line " << buffer.GetLineNumber(token) << ", Column " + << buffer.GetColumnNumber(token) + << ": Unrecognized characters!\n"; + + current_column += error_text.size(); + source_text = source_text.drop_front(error_text.size()); + buffer.has_errors = true; + } +}; + +auto TokenizedBuffer::Lex(SourceBuffer& source, DiagnosticEmitter& emitter) + -> TokenizedBuffer { + TokenizedBuffer buffer(source); + Lexer lexer(buffer, emitter); + + llvm::StringRef source_text = source.Text(); + while (lexer.SkipWhitespace(source_text)) { + // Each time we find non-whitespace characters, try each kind of token we + // support lexing, from simplest to most complex. + if (lexer.LexSymbolToken(source_text)) + continue; + if (lexer.LexKeywordOrIdentifier(source_text)) + continue; + if (lexer.LexIntegerLiteral(source_text)) + continue; + lexer.LexError(source_text); + } + + lexer.CloseInvalidOpenGroups(TokenKind::Error()); + return buffer; +} + +auto TokenizedBuffer::GetKind(Token token) const -> TokenKind { + return GetTokenInfo(token).kind; +} + +auto TokenizedBuffer::GetLine(Token token) const -> Line { + return GetTokenInfo(token).token_line; +} + +auto TokenizedBuffer::GetLineNumber(Token token) const -> int { + return GetLineNumber(GetLine(token)); +} + +auto TokenizedBuffer::GetColumnNumber(Token token) const -> int { + return GetTokenInfo(token).column + 1; +} + +auto TokenizedBuffer::GetTokenText(Token token) const -> llvm::StringRef { + auto& token_info = GetTokenInfo(token); + llvm::StringRef fixed_spelling = token_info.kind.GetFixedSpelling(); + if (!fixed_spelling.empty()) + return fixed_spelling; + + if (token_info.kind == TokenKind::Error()) { + auto& line_info = GetLineInfo(token_info.token_line); + int64_t token_start = line_info.start + token_info.column; + return source->Text().substr(token_start, token_info.error_length); + } + + // Documentation comment tokens refer back to the source text. + if (token_info.kind == TokenKind::DocComment()) { + auto& line_info = GetLineInfo(token_info.token_line); + int64_t token_start = line_info.start + token_info.column; + int64_t token_stop = line_info.start + line_info.length; + return source->Text().slice(token_start, token_stop); + } + + // Refer back to the source text to preserve oddities like radix or leading + // 0's the author had. + if (token_info.kind == TokenKind::IntegerLiteral()) { + auto& line_info = GetLineInfo(token_info.token_line); + int64_t token_start = line_info.start + token_info.column; + return TakeLeadingIntegerLiteral(source->Text().substr(token_start)); + } + + assert(token_info.kind == TokenKind::Identifier() && + "Only identifiers have stored text!"); + return GetIdentifierText(token_info.id); +} + +auto TokenizedBuffer::GetIdentifier(Token token) const -> Identifier { + auto& token_info = GetTokenInfo(token); + assert(token_info.kind == TokenKind::Identifier() && + "The token must be an identifier!"); + return token_info.id; +} + +auto TokenizedBuffer::GetIntegerLiteral(Token token) const -> llvm::APInt { + auto& token_info = GetTokenInfo(token); + assert(token_info.kind == TokenKind::IntegerLiteral() && + "The token must be an integer literal!"); + return int_literals[token_info.literal_index]; +} + +auto TokenizedBuffer::GetMatchedClosingToken(Token opening_token) const + -> Token { + auto& opening_token_info = GetTokenInfo(opening_token); + assert(opening_token_info.kind.IsOpeningSymbol() && + "The token must be an opening group symbol!"); + return opening_token_info.closing_token; +} + +auto TokenizedBuffer::GetMatchedOpeningToken(Token closing_token) const + -> Token { + auto& closing_token_info = GetTokenInfo(closing_token); + assert(closing_token_info.kind.IsClosingSymbol() && + "The token must be an closing group symbol!"); + return closing_token_info.opening_token; +} + +auto TokenizedBuffer::IsRecoveryToken(Token token) const -> bool { + return GetTokenInfo(token).is_recovery; +} + +auto TokenizedBuffer::GetLineNumber(Line line) const -> int { + return line.index + 1; +} + +auto TokenizedBuffer::GetIndentColumnNumber(Line line) const -> int { + return GetLineInfo(line).indent + 1; +} + +auto TokenizedBuffer::GetIdentifierText(Identifier identifier) const + -> llvm::StringRef { + return identifier_infos[identifier.index].text; +} + +auto TokenizedBuffer::PrintWidths::Widen(const PrintWidths& widths) -> void { + index = std::max(widths.index, index); + kind = std::max(widths.kind, kind); + column = std::max(widths.column, column); + line = std::max(widths.line, line); + indent = std::max(widths.indent, indent); +} + +auto TokenizedBuffer::GetTokenPrintWidths(Token token) const -> PrintWidths { + PrintWidths widths = {}; + // Compute the printed width of the various token information. When numbers + // here are printed in decimal, the number of digits needed is is one more than + // the log-base-10 of the value. + widths.index = std::log10(token_infos.size()) + 1; + widths.kind = GetKind(token).Name().size(); + widths.line = std::log10(GetLineNumber(token)) + 1; + widths.column = std::log10(GetColumnNumber(token)) + 1; + widths.indent = std::log10(GetIndentColumnNumber(GetLine(token))) + 1; + return widths; +} + +auto TokenizedBuffer::Print(llvm::raw_ostream& output_stream) const -> void { + if (Tokens().begin() == Tokens().end()) + return; + + PrintWidths widths = {}; + widths.index = std::log10(token_infos.size()) + 1; + for (Token token : Tokens()) + widths.Widen(GetTokenPrintWidths(token)); + + for (Token token : Tokens()) { + PrintToken(output_stream, token, widths); + output_stream << "\n"; + } +} + +auto TokenizedBuffer::PrintToken(llvm::raw_ostream& output_stream, + Token token) const -> void { + PrintToken(output_stream, token, {}); +} + +auto TokenizedBuffer::PrintToken(llvm::raw_ostream& output_stream, Token token, + PrintWidths widths) const -> void { + widths.Widen(GetTokenPrintWidths(token)); + int token_index = token.index; + auto& token_info = GetTokenInfo(token); + llvm::StringRef token_text = GetTokenText(token); + + // Output the main chunk using one format string. We have to do the + // justification manually in order to use the dynamically computed widths + // and get the quotes included. + output_stream << llvm::formatv( + "token: { index: {0}, kind: {1}, line: {2}, column: {3}, indent: {4}, " + "spelling: '{5}'", + llvm::format_decimal(token_index, widths.index), + llvm::right_justify( + (llvm::Twine("'") + token_info.kind.Name() + "'").str(), + widths.kind + 2), + llvm::format_decimal(GetLineNumber(token_info.token_line), widths.line), + llvm::format_decimal(GetColumnNumber(token), widths.column), + llvm::format_decimal(GetIndentColumnNumber(token_info.token_line), + widths.indent), + token_text); + + if (token_info.kind == TokenKind::Identifier()) + output_stream << ", identifier: " << GetIdentifier(token).index; + else if (token_info.kind.IsOpeningSymbol()) + output_stream << ", closing_token: " << GetMatchedClosingToken(token).index; + else if (token_info.kind.IsClosingSymbol()) + output_stream << ", opening_token: " << GetMatchedOpeningToken(token).index; + + if (token_info.is_recovery) + output_stream << ", recovery: true"; + + output_stream << " }"; +} + +auto TokenizedBuffer::GetLineInfo(Line line) -> LineInfo& { + return line_infos[line.index]; +} + +auto TokenizedBuffer::GetLineInfo(Line line) const -> const LineInfo& { + return line_infos[line.index]; +} + +auto TokenizedBuffer::AddLine(LineInfo info) -> Line { + line_infos.push_back(info); + return Line(line_infos.size() - 1); +} + +auto TokenizedBuffer::GetTokenInfo(Token token) -> TokenInfo& { + return token_infos[token.index]; +} + +auto TokenizedBuffer::GetTokenInfo(Token token) const -> const TokenInfo& { + return token_infos[token.index]; +} + +auto TokenizedBuffer::AddToken(TokenInfo info) -> Token { + token_infos.push_back(info); + return Token(token_infos.size() - 1); +} + +} // namespace Carbon diff --git a/lexer/tokenized_buffer.h b/lexer/tokenized_buffer.h new file mode 100644 index 0000000000000..3a7e9e0f9690c --- /dev/null +++ b/lexer/tokenized_buffer.h @@ -0,0 +1,343 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#ifndef LEXER_TOKENIZED_BUFFER_H_ +#define LEXER_TOKENIZED_BUFFER_H_ + +#include + +#include + +#include "diagnostics/diagnostic_emitter.h" +#include "lexer/token_kind.h" +#include "llvm/ADT/APInt.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/Optional.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/ADT/iterator.h" +#include "llvm/ADT/iterator_range.h" +#include "source/source_buffer.h" + +namespace Carbon { + +// A buffer of tokenized Carbon source code. +// +// This is constructed by lexing the source code text into a series of tokens. +// The buffer provides lightweight handles to tokens and other lexed entities, +// as well as iterations to walk the sequence of tokens found in the buffer. +// +// Lexing errors result in a potentially incomplete sequence of tokens and +// `HasError` returning true. +class TokenizedBuffer { + public: + // A lightweight handle to a lexed token in a `TokenizedBuffer`. + // + // `Token` objects are designed to be passed by value, not reference or + // pointer. They are also designed to be small and efficient to store in data + // structures. + // + // `Token` objects from the same `TokenizedBuffer` can be compared with each + // other, both for being the same token within the buffer, and to establish + // relative position within the token stream that has been lexed out of the + // buffer. + // + // All other APIs to query a `Token` are on the `TokenizedBuffer`. + class Token { + public: + Token() = default; + + bool operator==(const Token& rhs) const { return index == rhs.index; } + bool operator!=(const Token& rhs) const { return index != rhs.index; } + bool operator<(const Token& rhs) const { return index < rhs.index; } + bool operator<=(const Token& rhs) const { return index <= rhs.index; } + bool operator>(const Token& rhs) const { return index > rhs.index; } + bool operator>=(const Token& rhs) const { return index >= rhs.index; } + + private: + friend class TokenizedBuffer; + + explicit Token(int index) : index(index) {} + + int32_t index; + }; + + // A lightweight handle to a lexed line in a `TokenizedBuffer`. + // + // `Line` objects are designed to be passed by value, not reference or + // pointer. They are also designed to be small and efficient to store in data + // structures. + // + // Each `Line` object refers to a specific line in the source code that was + // lexed. They can be compared directly to establish that they refer to the + // same line or the relative position of different lines within the source. + // + // All other APIs to query a `Line` are on the `TokenizedBuffer`. + class Line { + public: + Line() = default; + + bool operator==(const Line& rhs) const { return index == rhs.index; } + bool operator!=(const Line& rhs) const { return index != rhs.index; } + bool operator<(const Line& rhs) const { return index < rhs.index; } + bool operator<=(const Line& rhs) const { return index <= rhs.index; } + bool operator>(const Line& rhs) const { return index > rhs.index; } + bool operator>=(const Line& rhs) const { return index >= rhs.index; } + + private: + friend class TokenizedBuffer; + + explicit Line(int index) : index(index) {} + + int32_t index; + }; + + // A lightweight handle to a lexed identifier in a `TokenizedBuffer`. + // + // `Identifier` objects are designed to be passed by value, not reference or + // pointer. They are also designed to be small and efficient to store in data + // structures. + // + // Each identifier lexed is canonicalized to a single entry in the identifier + // table. `Identifier` objects will compare equal if they refer to the same + // identifier spelling. Where the identifier was written is not preserved. + // + // All other APIs to query a `Identifier` are on the `TokenizedBuffer`. + class Identifier { + public: + Identifier() = default; + + // Most normal APIs are provided by the `TokenizedBuffer`, we just support + // basic comparison operations. + bool operator==(const Identifier& rhs) const { return index == rhs.index; } + bool operator!=(const Identifier& rhs) const { return index != rhs.index; } + + private: + friend class TokenizedBuffer; + + explicit Identifier(int index) : index(index) {} + + int32_t index; + }; + + // Random-access iterator over tokens within the buffer. + class TokenIterator + : public llvm::iterator_facade_base< + TokenIterator, std::random_access_iterator_tag, Token, int> { + public: + TokenIterator() = default; + + explicit TokenIterator(Token token) : token(token) {} + + bool operator==(const TokenIterator& rhs) const { + return token == rhs.token; + } + bool operator<(const TokenIterator& rhs) const { return token < rhs.token; } + + const Token& operator*() const { return token; } + Token& operator*() { return token; } + + int operator-(const TokenIterator& rhs) const { + return token.index - rhs.token.index; + } + + TokenIterator& operator+=(int n) { + token.index += n; + return *this; + } + TokenIterator& operator-=(int n) { + token.index -= n; + return *this; + } + + private: + friend class TokenizedBuffer; + + Token token; + }; + + // Lexes a buffer of source code into a tokenized buffer. + // + // The provided source buffer must outlive any returned `TokenizedBuffer` + // which will refer into the source. + // + // FIXME: Need to pass in some diagnostic machinery to report the details of + // the error! Right now it prints to stderr. + static TokenizedBuffer Lex(SourceBuffer& source, DiagnosticEmitter& emitter); + + // Returns true if the buffer has errors that are detectable at lexing time. + auto HasErrors() const -> bool { return has_errors; } + + llvm::iterator_range Tokens() const { + return llvm::make_range(TokenIterator(Token(0)), + TokenIterator(Token(token_infos.size()))); + } + + auto Size() const -> int { return token_infos.size(); } + + auto GetKind(Token token) const -> TokenKind; + auto GetLine(Token token) const -> Line; + + // Returns the 1-based line number. + auto GetLineNumber(Token token) const -> int; + + // Returns the 1-based column number. + auto GetColumnNumber(Token token) const -> int; + + // Returns the source text lexed into this token. + auto GetTokenText(Token token) const -> llvm::StringRef; + + // Returns the identifier associated with this token. The token kind must be + // an `Identifier`. + auto GetIdentifier(Token token) const -> Identifier; + + // Returns the value of an `IntegerLiteral()` token. + auto GetIntegerLiteral(Token token) const -> llvm::APInt; + + // Returns the closing token matched with the given opening token. + // + // The given token must be an opening token kind. + auto GetMatchedClosingToken(Token opening_token) const -> Token; + + // Returns the opening token matched with the given closing token. + // + // The given token must be a closing token kind. + auto GetMatchedOpeningToken(Token closing_token) const -> Token; + + // Returns whether the token was created as part of an error recovery effort. + // + // For example, a closing paren inserted to match an unmatched paren. + auto IsRecoveryToken(Token token) const -> bool; + + // Returns the 1-based line number. + auto GetLineNumber(Line line) const -> int; + + // Returns the 1-based indentation column number. + auto GetIndentColumnNumber(Line line) const -> int; + + // Returns the text for an identifier. + auto GetIdentifierText(Identifier id) const -> llvm::StringRef; + + // Prints a description of the tokenized stream to the provided `raw_ostream`. + // + // It prints one line of information for each token in the buffer, including + // the kind of token, where it occurs within the source file, indentation for + // the associated line, the spelling of the token in source, and any + // additional information tracked such as which unique identifier it is or any + // matched grouping token. + // + // Each line is formatted as a YAML record: + // + // clang-format off + // ``` + // token: { index: 0, kind: 'Semi', line: 1, column: 1, indent: 1, spelling: ';' } + // ``` + // clang-format on + // + // This can be parsed as YAML using tools like `python-yq` combined with `jq` + // on the command line. The format is also reasonably amenable to other + // line-oriented shell tools from `grep` to `awk`. + auto Print(llvm::raw_ostream& output_stream) const -> void; + + // Prints a description of a single token. See `print` for details on the + // format. + auto PrintToken(llvm::raw_ostream& output_stream, Token token) const -> void; + + private: + // Implementation detail struct implementing the actual lexer logic. + class Lexer; + friend Lexer; + + // Specifies minimum widths to use when printing a token's fields via + // `printToken`. + struct PrintWidths { + int index; + int kind; + int column; + int line; + int indent; + + // Widens `this` to the maximum of `this` and `new_width` for each + // dimension. + void Widen(const PrintWidths& new_width); + }; + + struct TokenInfo { + TokenKind kind; + + // Whether the token was injected artificially during error recovery. + bool is_recovery = false; + + // Line on which the Token starts. + Line token_line; + + // Zero-based byte offset of the token within its line. + int32_t column; + + // We may have up to 32 bits of payload, based on the kind of token. + union { + static_assert( + sizeof(Token) <= sizeof(int32_t), + "Unable to pack token and identifier index into the same space!"); + + Identifier id; + int32_t literal_index; + Token closing_token; + Token opening_token; + int32_t error_length; + }; + }; + + struct LineInfo { + // Zero-based byte offset of the start of the line within the source buffer + // provided. + int64_t start; + + // The byte length of the line. Does not include the newline character (or a + // null terminator or EOF). + int32_t length; + + // The byte offset from the start of the line of the first non-whitespace + // character. + int32_t indent; + }; + + struct IdentifierInfo { + llvm::StringRef text; + }; + + // The constructor is merely responsible for trivial initialization of + // members. A working object of this type is built with the `lex` function + // above so that its return can indicate if an error was encountered while + // lexing. + explicit TokenizedBuffer(SourceBuffer& source) : source(&source) {} + + auto GetLineInfo(Line line) -> LineInfo&; + auto GetLineInfo(Line line) const -> const LineInfo&; + auto AddLine(LineInfo info) -> Line; + auto GetTokenInfo(Token token) -> TokenInfo&; + auto GetTokenInfo(Token token) const -> const TokenInfo&; + auto AddToken(TokenInfo info) -> Token; + auto GetTokenPrintWidths(Token token) const -> PrintWidths; + auto PrintToken(llvm::raw_ostream& output_stream, Token token, + PrintWidths widths) const -> void; + + SourceBuffer* source; + + llvm::SmallVector token_infos; + + llvm::SmallVector line_infos; + + llvm::SmallVector identifier_infos; + + llvm::SmallVector int_literals; + + llvm::DenseMap identifier_map; + + bool has_errors = false; +}; + +} // namespace Carbon + +#endif // LEXER_TOKENIZED_BUFFER_H_ diff --git a/lexer/tokenized_buffer_fuzzer.cpp b/lexer/tokenized_buffer_fuzzer.cpp new file mode 100644 index 0000000000000..9953b278af62e --- /dev/null +++ b/lexer/tokenized_buffer_fuzzer.cpp @@ -0,0 +1,60 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include + +#include "diagnostics/diagnostic_emitter.h" +#include "lexer/tokenized_buffer.h" +#include "llvm/ADT/StringRef.h" + +namespace Carbon { + +extern "C" int LLVMFuzzerTestOneInput(const unsigned char* data, size_t size) { + // We need two bytes of data to compute a file name length. + if (size < 2) + return 0; + unsigned short raw_filename_length; + std::memcpy(&raw_filename_length, data, 2); + data += 2; + size -= 2; + size_t filename_length = raw_filename_length; + + // We need enough data to populate this filename length. + if (size < filename_length) + return 0; + llvm::StringRef filename(reinterpret_cast(data), + filename_length); + data += filename_length; + size -= filename_length; + + // The rest of the data is the source text. + auto source = SourceBuffer::CreateFromText( + llvm::StringRef(reinterpret_cast(data), size), filename); + + // Use a real diagnostic emitter to get lazy codepaths to execute. + DiagnosticEmitter emitter = NullDiagnosticEmitter(); + + auto buffer = TokenizedBuffer::Lex(source, emitter); + if (buffer.HasErrors()) + return 0; + + // Walk the lexed and tokenized buffer to ensure it isn't corrupt in some way. + // + // TODO: We should enhance this to do more sanity checks on the resulting + // token stream. + for (TokenizedBuffer::Token token : buffer.Tokens()) { + int line_number = buffer.GetLineNumber(token); + (void)line_number; + assert(line_number > 0 && "Invalid line number!"); + assert(line_number < INT_MAX && "Invalid line number!"); + int column_number = buffer.GetColumnNumber(token); + (void)column_number; + assert(column_number > 0 && "Invalid line number!"); + assert(column_number < INT_MAX && "Invalid line number!"); + } + + return 0; +} + +} // namespace Carbon diff --git a/lexer/tokenized_buffer_test.cpp b/lexer/tokenized_buffer_test.cpp new file mode 100644 index 0000000000000..4ff520a1162c4 --- /dev/null +++ b/lexer/tokenized_buffer_test.cpp @@ -0,0 +1,681 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include "lexer/tokenized_buffer.h" + +#include + +#include "diagnostics/diagnostic_emitter.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "lexer/tokenized_buffer_test_helpers.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/None.h" +#include "llvm/ADT/Sequence.h" +#include "llvm/ADT/SmallString.h" +#include "llvm/ADT/Twine.h" +#include "llvm/Support/SourceMgr.h" +#include "llvm/Support/YAMLParser.h" +#include "llvm/Support/raw_ostream.h" + +namespace Carbon { +namespace { + +using ::Carbon::Testing::ExpectedToken; +using ::Carbon::Testing::HasTokens; +using ::Carbon::Testing::IsKeyValueScalars; +using ::testing::Eq; +using ::testing::NotNull; +using ::testing::StrEq; + +struct LexerTest : ::testing::Test { + llvm::SmallVector source_storage; + + auto GetSourceBuffer(llvm::Twine text) -> SourceBuffer& { + source_storage.push_back(SourceBuffer::CreateFromText(text.str())); + return source_storage.back(); + } + + auto Lex(llvm::Twine text) -> TokenizedBuffer { + // TODO: build a full mock for this. + return TokenizedBuffer::Lex(GetSourceBuffer(text), NullDiagnosticEmitter()); + } +}; + +TEST_F(LexerTest, HandlesEmptyBuffer) { + auto buffer = Lex(""); + EXPECT_FALSE(buffer.HasErrors()); + EXPECT_EQ(buffer.Tokens().begin(), buffer.Tokens().end()); +} + +TEST_F(LexerTest, TracksLinesAndColumns) { + auto buffer = Lex("\n ;;\n ;;;\n"); + EXPECT_FALSE(buffer.HasErrors()); + EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ + {.kind = TokenKind::Semi(), + .line = 2, + .column = 3, + .indent_column = 3}, + {.kind = TokenKind::Semi(), + .line = 2, + .column = 4, + .indent_column = 3}, + {.kind = TokenKind::Semi(), + .line = 3, + .column = 4, + .indent_column = 4}, + {.kind = TokenKind::Semi(), + .line = 3, + .column = 5, + .indent_column = 4}, + {.kind = TokenKind::Semi(), + .line = 3, + .column = 6, + .indent_column = 4}, + })); +} + +TEST_F(LexerTest, HandlesIntegerLiteral) { + auto buffer = Lex("12-578\n 1 2"); + EXPECT_FALSE(buffer.HasErrors()); + ASSERT_THAT(buffer, HasTokens(llvm::ArrayRef{ + {.kind = TokenKind::IntegerLiteral(), + .line = 1, + .column = 1, + .indent_column = 1, + .text = "12"}, + {.kind = TokenKind::Minus(), + .line = 1, + .column = 3, + .indent_column = 1}, + {.kind = TokenKind::IntegerLiteral(), + .line = 1, + .column = 4, + .indent_column = 1, + .text = "578"}, + {.kind = TokenKind::IntegerLiteral(), + .line = 2, + .column = 3, + .indent_column = 3, + .text = "1"}, + {.kind = TokenKind::IntegerLiteral(), + .line = 2, + .column = 6, + .indent_column = 3, + .text = "2"}, + })); + auto token_12 = buffer.Tokens().begin(); + EXPECT_EQ(buffer.GetIntegerLiteral(*token_12), 12); + auto token_578 = buffer.Tokens().begin() + 2; + EXPECT_EQ(buffer.GetIntegerLiteral(*token_578), 578); + auto token_1 = buffer.Tokens().begin() + 3; + EXPECT_EQ(buffer.GetIntegerLiteral(*token_1), 1); + auto token_2 = buffer.Tokens().begin() + 4; + EXPECT_EQ(buffer.GetIntegerLiteral(*token_2), 2); +} + +TEST_F(LexerTest, HandlesGarbageCharacters) { + const char GarbageText[] = "$$💩-$\n$\0$12$"; + auto buffer = Lex(llvm::StringRef(GarbageText, sizeof(GarbageText) - 1)); + EXPECT_TRUE(buffer.HasErrors()); + EXPECT_THAT( + buffer, + HasTokens(llvm::ArrayRef{ + {.kind = TokenKind::Error(), + .line = 1, + .column = 1, + .text = llvm::StringRef("$$💩", 6)}, + // 💩 takes 4 bytes, and we count column as bytes offset. + {.kind = TokenKind::Minus(), .line = 1, .column = 7}, + {.kind = TokenKind::Error(), .line = 1, .column = 8, .text = "$"}, + // newline + {.kind = TokenKind::Error(), + .line = 2, + .column = 1, + .text = llvm::StringRef("$\0$", 3)}, + {.kind = TokenKind::IntegerLiteral(), + .line = 2, + .column = 4, + .text = "12"}, + {.kind = TokenKind::Error(), .line = 2, .column = 6, .text = "$"}, + })); +} + +TEST_F(LexerTest, Symbols) { + // We don't need to exhaustively test symbols here as they're handled with + // common code, but we want to check specific patterns to verify things like + // max-munch rule and handling of interesting symbols. + auto buffer = Lex("<<<"); + EXPECT_FALSE(buffer.HasErrors()); + EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ + {TokenKind::LessLess()}, + {TokenKind::Less()}, + })); + + buffer = Lex("<<=>>"); + EXPECT_FALSE(buffer.HasErrors()); + EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ + {TokenKind::LessLessEqual()}, + {TokenKind::GreaterGreater()}, + })); + + buffer = Lex("< <=> >"); + EXPECT_FALSE(buffer.HasErrors()); + EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ + {TokenKind::Less()}, + {TokenKind::LessEqualGreater()}, + {TokenKind::Greater()}, + })); + + buffer = Lex("\\/?#@&^!"); + EXPECT_FALSE(buffer.HasErrors()); + EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ + {TokenKind::Backslash()}, + {TokenKind::Slash()}, + {TokenKind::Question()}, + {TokenKind::Hash()}, + {TokenKind::At()}, + {TokenKind::Amp()}, + {TokenKind::Caret()}, + {TokenKind::Exclaim()}, + })); +} + +TEST_F(LexerTest, Parens) { + auto buffer = Lex("()"); + EXPECT_FALSE(buffer.HasErrors()); + EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ + {TokenKind::OpenParen()}, + {TokenKind::CloseParen()}, + })); + + buffer = Lex("((()()))"); + EXPECT_FALSE(buffer.HasErrors()); + EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ + {TokenKind::OpenParen()}, + {TokenKind::OpenParen()}, + {TokenKind::OpenParen()}, + {TokenKind::CloseParen()}, + {TokenKind::OpenParen()}, + {TokenKind::CloseParen()}, + {TokenKind::CloseParen()}, + {TokenKind::CloseParen()}, + })); +} + +TEST_F(LexerTest, CurlyBraces) { + auto buffer = Lex("{}"); + EXPECT_FALSE(buffer.HasErrors()); + EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ + {TokenKind::OpenCurlyBrace()}, + {TokenKind::CloseCurlyBrace()}, + })); + + buffer = Lex("{{{}{}}}"); + EXPECT_FALSE(buffer.HasErrors()); + EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ + {TokenKind::OpenCurlyBrace()}, + {TokenKind::OpenCurlyBrace()}, + {TokenKind::OpenCurlyBrace()}, + {TokenKind::CloseCurlyBrace()}, + {TokenKind::OpenCurlyBrace()}, + {TokenKind::CloseCurlyBrace()}, + {TokenKind::CloseCurlyBrace()}, + {TokenKind::CloseCurlyBrace()}, + })); +} + +TEST_F(LexerTest, MatchingGroups) { + { + TokenizedBuffer buffer = Lex("(){}"); + ASSERT_FALSE(buffer.HasErrors()); + auto it = buffer.Tokens().begin(); + auto open_paren_token = *it++; + auto close_paren_token = *it++; + EXPECT_EQ(close_paren_token, + buffer.GetMatchedClosingToken(open_paren_token)); + EXPECT_EQ(open_paren_token, + buffer.GetMatchedOpeningToken(close_paren_token)); + auto open_curly_token = *it++; + auto close_curly_token = *it++; + EXPECT_EQ(close_curly_token, + buffer.GetMatchedClosingToken(open_curly_token)); + EXPECT_EQ(open_curly_token, + buffer.GetMatchedOpeningToken(close_curly_token)); + EXPECT_EQ(buffer.Tokens().end(), it); + } + + { + TokenizedBuffer buffer = Lex("({x}){(y)} {{((z))}}"); + ASSERT_FALSE(buffer.HasErrors()); + auto it = buffer.Tokens().begin(); + auto open_paren_token = *it++; + auto open_curly_token = *it++; + ASSERT_EQ("x", buffer.GetIdentifierText(buffer.GetIdentifier(*it++))); + auto close_curly_token = *it++; + auto close_paren_token = *it++; + EXPECT_EQ(close_paren_token, + buffer.GetMatchedClosingToken(open_paren_token)); + EXPECT_EQ(open_paren_token, + buffer.GetMatchedOpeningToken(close_paren_token)); + EXPECT_EQ(close_curly_token, + buffer.GetMatchedClosingToken(open_curly_token)); + EXPECT_EQ(open_curly_token, + buffer.GetMatchedOpeningToken(close_curly_token)); + + open_curly_token = *it++; + open_paren_token = *it++; + ASSERT_EQ("y", buffer.GetIdentifierText(buffer.GetIdentifier(*it++))); + close_paren_token = *it++; + close_curly_token = *it++; + EXPECT_EQ(close_curly_token, + buffer.GetMatchedClosingToken(open_curly_token)); + EXPECT_EQ(open_curly_token, + buffer.GetMatchedOpeningToken(close_curly_token)); + EXPECT_EQ(close_paren_token, + buffer.GetMatchedClosingToken(open_paren_token)); + EXPECT_EQ(open_paren_token, + buffer.GetMatchedOpeningToken(close_paren_token)); + + open_curly_token = *it++; + auto inner_open_curly_token = *it++; + open_paren_token = *it++; + auto inner_open_paren_token = *it++; + ASSERT_EQ("z", buffer.GetIdentifierText(buffer.GetIdentifier(*it++))); + auto inner_close_paren_token = *it++; + close_paren_token = *it++; + auto inner_close_curly_token = *it++; + close_curly_token = *it++; + EXPECT_EQ(close_curly_token, + buffer.GetMatchedClosingToken(open_curly_token)); + EXPECT_EQ(open_curly_token, + buffer.GetMatchedOpeningToken(close_curly_token)); + EXPECT_EQ(inner_close_curly_token, + buffer.GetMatchedClosingToken(inner_open_curly_token)); + EXPECT_EQ(inner_open_curly_token, + buffer.GetMatchedOpeningToken(inner_close_curly_token)); + EXPECT_EQ(close_paren_token, + buffer.GetMatchedClosingToken(open_paren_token)); + EXPECT_EQ(open_paren_token, + buffer.GetMatchedOpeningToken(close_paren_token)); + EXPECT_EQ(inner_close_paren_token, + buffer.GetMatchedClosingToken(inner_open_paren_token)); + EXPECT_EQ(inner_open_paren_token, + buffer.GetMatchedOpeningToken(inner_close_paren_token)); + + EXPECT_EQ(buffer.Tokens().end(), it); + } +} + +TEST_F(LexerTest, MismatchedGroups) { + auto buffer = Lex("{"); + EXPECT_TRUE(buffer.HasErrors()); + EXPECT_THAT(buffer, + HasTokens(llvm::ArrayRef{ + {TokenKind::OpenCurlyBrace()}, + {.kind = TokenKind::CloseCurlyBrace(), .recovery = true}, + })); + + buffer = Lex("}"); + EXPECT_TRUE(buffer.HasErrors()); + EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ + {.kind = TokenKind::Error(), .text = "}"}, + })); + + buffer = Lex("{(}"); + EXPECT_TRUE(buffer.HasErrors()); + EXPECT_THAT( + buffer, + HasTokens(llvm::ArrayRef{ + {.kind = TokenKind::OpenCurlyBrace(), .column = 1}, + {.kind = TokenKind::OpenParen(), .column = 2}, + {.kind = TokenKind::CloseParen(), .column = 3, .recovery = true}, + {.kind = TokenKind::CloseCurlyBrace(), .column = 3}, + })); + + buffer = Lex(")({)"); + EXPECT_TRUE(buffer.HasErrors()); + EXPECT_THAT( + buffer, + HasTokens(llvm::ArrayRef{ + {.kind = TokenKind::Error(), .column = 1, .text = ")"}, + {.kind = TokenKind::OpenParen(), .column = 2}, + {.kind = TokenKind::OpenCurlyBrace(), .column = 3}, + {.kind = TokenKind::CloseCurlyBrace(), .column = 4, .recovery = true}, + {.kind = TokenKind::CloseParen(), .column = 4}, + })); +} + +TEST_F(LexerTest, Keywords) { + auto buffer = Lex(" fn"); + EXPECT_FALSE(buffer.HasErrors()); + EXPECT_THAT( + buffer, + HasTokens(llvm::ArrayRef{ + {.kind = TokenKind::FnKeyword(), .column = 4, .indent_column = 4}, + })); + + buffer = Lex("and or not if else for loop return var break continue _"); + EXPECT_FALSE(buffer.HasErrors()); + EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ + {TokenKind::AndKeyword()}, + {TokenKind::OrKeyword()}, + {TokenKind::NotKeyword()}, + {TokenKind::IfKeyword()}, + {TokenKind::ElseKeyword()}, + {TokenKind::ForKeyword()}, + {TokenKind::LoopKeyword()}, + {TokenKind::ReturnKeyword()}, + {TokenKind::VarKeyword()}, + {TokenKind::BreakKeyword()}, + {TokenKind::ContinueKeyword()}, + {TokenKind::UnderscoreKeyword()}, + })); +} + +TEST_F(LexerTest, Comments) { + auto buffer = Lex(" ;\n // foo\n ;"); + EXPECT_FALSE(buffer.HasErrors()); + EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ + {.kind = TokenKind::Semi(), + .line = 1, + .column = 2, + .indent_column = 2}, + {.kind = TokenKind::Semi(), + .line = 3, + .column = 3, + .indent_column = 3}, + })); + + buffer = Lex("// foo\n//\n// bar"); + EXPECT_FALSE(buffer.HasErrors()); + EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{})); + + // Make sure weird characters aren't a problem. + buffer = Lex(" //foo#$!^?@-_💩🍫⃠ [̲̅$̲̅(̲̅ ͡° ͜ʖ ͡°̲̅)̲̅$̲̅]"); + EXPECT_FALSE(buffer.HasErrors()); + EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{})); +} + +TEST_F(LexerTest, DocComments) { + auto buffer = Lex(" /// foo"); + EXPECT_FALSE(buffer.HasErrors()); + EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ + {.kind = TokenKind::DocComment(), + .line = 1, + .column = 3, + .indent_column = 3, + .text = "/// foo"}, + })); + + buffer = Lex("/// foo\n//\n/// bar"); + EXPECT_FALSE(buffer.HasErrors()); + EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ + {.kind = TokenKind::DocComment(), + .line = 1, + .column = 1, + .indent_column = 1, + .text = "/// foo"}, + {.kind = TokenKind::DocComment(), + .line = 3, + .column = 1, + .indent_column = 1, + .text = "/// bar"}, + })); + + buffer = Lex("/// foo\n///\n/// bar"); + EXPECT_FALSE(buffer.HasErrors()); + EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ + {.kind = TokenKind::DocComment(), + .line = 1, + .column = 1, + .indent_column = 1, + .text = "/// foo"}, + {.kind = TokenKind::DocComment(), + .line = 2, + .column = 1, + .indent_column = 1, + .text = "///"}, + {.kind = TokenKind::DocComment(), + .line = 3, + .column = 1, + .indent_column = 1, + .text = "/// bar"}, + })); + + // Make sure weird characters aren't a problem. + buffer = Lex(" ///foo#$!^?@-_💩🍫⃠ [̲̅$̲̅(̲̅ ͡° ͜ʖ ͡°̲̅)̲̅$̲̅]"); + EXPECT_FALSE(buffer.HasErrors()); + EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ + {.kind = TokenKind::DocComment(), + .line = 1, + .column = 3, + .indent_column = 3, + .text = "///foo#$!^?@-_💩🍫⃠ [̲̅$̲̅(̲̅ ͡° ͜ʖ ͡°̲̅)̲̅$̲̅]"}, + })); +} + +TEST_F(LexerTest, Identifiers) { + auto buffer = Lex(" foobar"); + EXPECT_FALSE(buffer.HasErrors()); + EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ + {.kind = TokenKind::Identifier(), + .column = 4, + .indent_column = 4, + .text = "foobar"}, + })); + + // Check different kinds of identifier character sequences. + buffer = Lex("_foo_bar"); + EXPECT_FALSE(buffer.HasErrors()); + EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ + {.kind = TokenKind::Identifier(), .text = "_foo_bar"}, + })); + + buffer = Lex("foo2bar00"); + EXPECT_FALSE(buffer.HasErrors()); + EXPECT_THAT(buffer, + HasTokens(llvm::ArrayRef{ + {.kind = TokenKind::Identifier(), .text = "foo2bar00"}, + })); + + // Check that we can parse identifiers that start with a keyword. + buffer = Lex("fnord"); + EXPECT_FALSE(buffer.HasErrors()); + EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ + {.kind = TokenKind::Identifier(), .text = "fnord"}, + })); + + // Check multiple identifiers with indent and interning. + buffer = Lex(" foo;bar\nbar \n foo\tfoo"); + EXPECT_FALSE(buffer.HasErrors()); + EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ + {.kind = TokenKind::Identifier(), + .line = 1, + .column = 4, + .indent_column = 4, + .text = "foo"}, + {.kind = TokenKind::Semi()}, + {.kind = TokenKind::Identifier(), + .line = 1, + .column = 8, + .indent_column = 4, + .text = "bar"}, + {.kind = TokenKind::Identifier(), + .line = 2, + .column = 1, + .indent_column = 1, + .text = "bar"}, + {.kind = TokenKind::Identifier(), + .line = 3, + .column = 3, + .indent_column = 3, + .text = "foo"}, + {.kind = TokenKind::Identifier(), + .line = 3, + .column = 7, + .indent_column = 3, + .text = "foo"}, + })); +} + +auto GetAndDropLine(llvm::StringRef& text) -> std::string { + auto newline_offset = text.find_first_of('\n'); + llvm::StringRef line = text.slice(0, newline_offset); + + if (newline_offset != llvm::StringRef::npos) + text = text.substr(newline_offset + 1); + else + text = ""; + + return line.str(); +} + +TEST_F(LexerTest, Printing) { + auto buffer = Lex(";"); + ASSERT_FALSE(buffer.HasErrors()); + std::string print_storage; + llvm::raw_string_ostream print_stream(print_storage); + buffer.Print(print_stream); + llvm::StringRef print = print_stream.str(); + EXPECT_THAT(GetAndDropLine(print), + StrEq("token: { index: 0, kind: 'Semi', line: 1, column: 1, " + "indent: 1, spelling: ';' }")); + EXPECT_TRUE(print.empty()) << print; + + // Test kind padding. + buffer = Lex("(;foo;)"); + ASSERT_FALSE(buffer.HasErrors()); + print_storage.clear(); + buffer.Print(print_stream); + print = print_stream.str(); + EXPECT_THAT(GetAndDropLine(print), + StrEq("token: { index: 0, kind: 'OpenParen', line: 1, column: " + "1, indent: 1, spelling: '(', closing_token: 4 }")); + EXPECT_THAT(GetAndDropLine(print), + StrEq("token: { index: 1, kind: 'Semi', line: 1, column: " + "2, indent: 1, spelling: ';' }")); + EXPECT_THAT(GetAndDropLine(print), + StrEq("token: { index: 2, kind: 'Identifier', line: 1, column: " + "3, indent: 1, spelling: 'foo', identifier: 0 }")); + EXPECT_THAT(GetAndDropLine(print), + StrEq("token: { index: 3, kind: 'Semi', line: 1, column: " + "6, indent: 1, spelling: ';' }")); + EXPECT_THAT(GetAndDropLine(print), + StrEq("token: { index: 4, kind: 'CloseParen', line: 1, column: " + "7, indent: 1, spelling: ')', opening_token: 0 }")); + EXPECT_TRUE(print.empty()) << print; + + // Test digit padding with max values of 9, 10, and 11. + buffer = Lex(";\n\n\n\n\n\n\n\n\n\n ;;"); + ASSERT_FALSE(buffer.HasErrors()); + print_storage.clear(); + buffer.Print(print_stream); + print = print_stream.str(); + EXPECT_THAT(GetAndDropLine(print), + StrEq("token: { index: 0, kind: 'Semi', line: 1, column: 1, " + "indent: 1, spelling: ';' }")); + EXPECT_THAT(GetAndDropLine(print), + StrEq("token: { index: 1, kind: 'Semi', line: 11, column: 9, " + "indent: 9, spelling: ';' }")); + EXPECT_THAT(GetAndDropLine(print), + StrEq("token: { index: 2, kind: 'Semi', line: 11, column: 10, " + "indent: 9, spelling: ';' }")); + EXPECT_TRUE(print.empty()) << print; +} + +TEST_F(LexerTest, PrintingAsYaml) { + // Test that we can parse this into YAML and verify line and indent data. + auto buffer = Lex("\n ;\n\n\n; ;\n\n\n\n\n\n\n\n\n\n\n"); + ASSERT_FALSE(buffer.HasErrors()); + std::string print_output; + llvm::raw_string_ostream print_stream(print_output); + buffer.Print(print_stream); + print_stream.flush(); + + // Parse the output into a YAML stream. This will print errors to stderr. + llvm::SourceMgr source_manager; + llvm::yaml::Stream yaml_stream(print_output, source_manager); + auto yaml_it = yaml_stream.begin(); + auto* root_node = llvm::dyn_cast(yaml_it->getRoot()); + ASSERT_THAT(root_node, NotNull()); + + // Walk the top-level mapping of tokens, dig out the sub-mapping of data for + // each taken, and then verify those entries. + auto mapping_it = llvm::cast(root_node)->begin(); + auto* token_node = llvm::dyn_cast(&*mapping_it); + ASSERT_THAT(token_node, NotNull()); + auto* token_key_node = + llvm::dyn_cast(token_node->getKey()); + ASSERT_THAT(token_key_node, NotNull()); + EXPECT_THAT(token_key_node->getRawValue(), StrEq("token")); + auto* token_value_node = + llvm::dyn_cast(token_node->getValue()); + ASSERT_THAT(token_value_node, NotNull()); + auto token_it = token_value_node->begin(); + EXPECT_THAT(&*token_it, IsKeyValueScalars("index", "0")); + ++token_it; + EXPECT_THAT(&*token_it, IsKeyValueScalars("kind", "Semi")); + ++token_it; + EXPECT_THAT(&*token_it, IsKeyValueScalars("line", "2")); + ++token_it; + EXPECT_THAT(&*token_it, IsKeyValueScalars("column", "2")); + ++token_it; + EXPECT_THAT(&*token_it, IsKeyValueScalars("indent", "2")); + ++token_it; + EXPECT_THAT(&*token_it, IsKeyValueScalars("spelling", ";")); + EXPECT_THAT(++token_it, Eq(token_value_node->end())); + + ++mapping_it; + token_node = llvm::dyn_cast(&*mapping_it); + ASSERT_THAT(token_node, NotNull()); + token_key_node = llvm::dyn_cast(token_node->getKey()); + ASSERT_THAT(token_key_node, NotNull()); + EXPECT_THAT(token_key_node->getRawValue(), StrEq("token")); + token_value_node = + llvm::dyn_cast(token_node->getValue()); + ASSERT_THAT(token_value_node, NotNull()); + token_it = token_value_node->begin(); + EXPECT_THAT(&*token_it, IsKeyValueScalars("index", "1")); + ++token_it; + EXPECT_THAT(&*token_it, IsKeyValueScalars("kind", "Semi")); + ++token_it; + EXPECT_THAT(&*token_it, IsKeyValueScalars("line", "5")); + ++token_it; + EXPECT_THAT(&*token_it, IsKeyValueScalars("column", "1")); + ++token_it; + EXPECT_THAT(&*token_it, IsKeyValueScalars("indent", "1")); + ++token_it; + EXPECT_THAT(&*token_it, IsKeyValueScalars("spelling", ";")); + EXPECT_THAT(++token_it, Eq(token_value_node->end())); + + ++mapping_it; + token_node = llvm::dyn_cast(&*mapping_it); + ASSERT_THAT(token_node, NotNull()); + token_key_node = llvm::dyn_cast(token_node->getKey()); + ASSERT_THAT(token_key_node, NotNull()); + EXPECT_THAT(token_key_node->getRawValue(), StrEq("token")); + token_value_node = + llvm::dyn_cast(token_node->getValue()); + ASSERT_THAT(token_value_node, NotNull()); + token_it = token_value_node->begin(); + EXPECT_THAT(&*token_it, IsKeyValueScalars("index", "2")); + ++token_it; + EXPECT_THAT(&*token_it, IsKeyValueScalars("kind", "Semi")); + ++token_it; + EXPECT_THAT(&*token_it, IsKeyValueScalars("line", "5")); + ++token_it; + EXPECT_THAT(&*token_it, IsKeyValueScalars("column", "3")); + ++token_it; + EXPECT_THAT(&*token_it, IsKeyValueScalars("indent", "1")); + ++token_it; + EXPECT_THAT(&*token_it, IsKeyValueScalars("spelling", ";")); + EXPECT_THAT(++token_it, Eq(token_value_node->end())); + + ASSERT_THAT(++mapping_it, Eq(root_node->end())); + ASSERT_THAT(++yaml_it, Eq(yaml_stream.end())); +} + +} // namespace +} // namespace Carbon diff --git a/lexer/tokenized_buffer_test_helpers.h b/lexer/tokenized_buffer_test_helpers.h new file mode 100644 index 0000000000000..807ad9f8f5141 --- /dev/null +++ b/lexer/tokenized_buffer_test_helpers.h @@ -0,0 +1,172 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#ifndef LEXER_TOKENIZED_BUFFER_TEST_HELPERS_H_ +#define LEXER_TOKENIZED_BUFFER_TEST_HELPERS_H_ + +#include "gmock/gmock.h" +#include "lexer/tokenized_buffer.h" +#include "llvm/ADT/SmallString.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/YAMLParser.h" + +namespace Carbon { + +inline void PrintTo(const TokenizedBuffer& buffer, std::ostream* output) { + std::string message; + llvm::raw_string_ostream message_stream(message); + message_stream << "\n"; + buffer.Print(message_stream); + *output << message_stream.str(); +} + +namespace Testing { + +struct ExpectedToken { + TokenKind kind; + int line = -1; + int column = -1; + int indent_column = -1; + bool recovery = false; + llvm::StringRef text = ""; + + friend std::ostream& operator<<(std::ostream& output, + const ExpectedToken& expected) { + output << "\ntoken: { kind: '" << expected.kind.Name().str(); + if (expected.line != -1) + output << "', line: " << expected.line; + if (expected.column != -1) + output << ", column " << expected.column; + if (expected.indent_column != -1) + output << ", indent: " << expected.indent_column; + if (!expected.text.empty()) + output << ", spelling: '" << expected.text.str() << "'"; + if (expected.recovery) + output << ", recovery: true"; + output << " }"; + return output; + } +}; + +// TODO: Consider rewriting this into a `TokenEq` matcher which is used inside +// `ElementsAre`. If that isn't easily done, potentially worth checking for size +// mismatches first. +MATCHER_P(HasTokens, raw_all_expected, "") { + const TokenizedBuffer& buffer = arg; + llvm::ArrayRef all_expected = raw_all_expected; + + bool matches = true; + auto buffer_it = buffer.Tokens().begin(); + for (const ExpectedToken& expected : all_expected) { + if (buffer_it == buffer.Tokens().end()) { + // The size check outside the loop will fail and print useful info. + break; + } + + int index = buffer_it - buffer.Tokens().begin(); + auto token = *buffer_it++; + + TokenKind actual_kind = buffer.GetKind(token); + if (actual_kind != expected.kind) { + *result_listener << "\nToken " << index << " is a " + << actual_kind.Name().str() << ", expected a " + << expected.kind.Name().str() << "."; + matches = false; + } + + int actual_line = buffer.GetLineNumber(token); + if (expected.line != -1 && actual_line != expected.line) { + *result_listener << "\nToken " << index << " is at line " << actual_line + << ", expected " << expected.line << "."; + matches = false; + } + + int actual_column = buffer.GetColumnNumber(token); + if (expected.column != -1 && actual_column != expected.column) { + *result_listener << "\nToken " << index << " is at column " + << actual_column << ", expected " << expected.column + << "."; + matches = false; + } + + int actual_indent_column = + buffer.GetIndentColumnNumber(buffer.GetLine(token)); + if (expected.indent_column != -1 && + actual_indent_column != expected.indent_column) { + *result_listener << "\nToken " << index << " has column indent " + << actual_indent_column << ", expected " + << expected.indent_column << "."; + matches = false; + } + + int actual_recovery = buffer.IsRecoveryToken(token); + if (expected.recovery != actual_recovery) { + *result_listener << "\nToken " << index << " is " + << (actual_recovery ? "recovery" : "non-recovery") + << ", expected " + << (expected.recovery ? "recovery" : "non-recovery") + << "."; + matches = false; + } + + llvm::StringRef actual_text = buffer.GetTokenText(token); + if (!expected.text.empty() && actual_text != expected.text) { + *result_listener << "\nToken " << index << " has spelling `" + << actual_text.str() << "`, expected `" + << expected.text.str() << "`."; + matches = false; + } + } + + int actual_size = buffer.Tokens().end() - buffer.Tokens().begin(); + if (static_cast(all_expected.size()) != actual_size) { + *result_listener << "\nExpected " << all_expected.size() + << " tokens but found " << actual_size << "."; + matches = false; + } + return matches; +} + +MATCHER_P2(IsKeyValueScalars, key, value, "") { + auto* kv_node = llvm::dyn_cast(arg); + if (!kv_node) { + *result_listener << "this is a `" << arg->getType() + << "` node, not a `KeyValueNode`."; + return false; + } + + llvm::SmallString<128> storage; + + auto* key_node = llvm::dyn_cast(kv_node->getKey()); + if (!key_node) { + *result_listener << "the key is a `" << arg->getType() + << "` node, not a `ScalarNode`."; + return false; + } + if (key != key_node->getValue(storage)) { + *result_listener << "the key is `" << key_node->getValue(storage).str() + << "`, expected `" << key << "`."; + return false; + } + + auto* value_node = + llvm::dyn_cast(kv_node->getValue()); + if (!value_node) { + *result_listener << "the value is a `" << arg->getType() + << "` node, not a `ScalarNode`."; + return false; + } + if (value != value_node->getValue(storage)) { + *result_listener << "the value is `" << value_node->getValue(storage).str() + << "`, expected `" << value << "`."; + return false; + } + + return true; +} + +} // namespace testing +} // namespace Carbon + +#endif // LEXER_TOKENIZED_BUFFER_TEST_HELPERS_H_