feat: token prediction (speculative decoding) #768
giladgdbuild.yml
on: pull_request
Build
1m 29s
Matrix: build-binaries
Update documentation website - no version release
0s
Update documentation website
0s
Auto-approve documentation website deployment
0s
Matrix: build-electron-example
Annotations
5 errors and 1 warning
|
test/modelDependent/llama3.1/controlledEvaluate.test.ts > llama 3.1 > controlled evaluate > get probabilities for 3 tokens:
test/modelDependent/llama3.1/controlledEvaluate.test.ts#L83
Error: Snapshot `llama 3.1 > controlled evaluate > get probabilities for 3 tokens 1` mismatched
- Expected
+ Received
@@ -7,77 +7,77 @@
},
},
{
"next": {
"probabilities": Map {
- 35308 => 0.5214946,
+ 35308 => 0.4790331,
- 27096 => 0.2432059,
+ 27096 => 0.2675457,
- 11 => 0.0221824,
+ 11 => 0.0219745,
- 198 => 0.0119446,
+ 198 => 0.0139698,
- 374 => 0.0083614,
+ 374 => 0.0089695,
- 863 => 0.0083608,
+ 863 => 0.0086552,
- 1131 => 0.0068347,
+ 1131 => 0.0067397,
- 25 => 0.0062433,
+ 25 => 0.0059004,
- 7940 => 0.0054039,
+ 320 => 0.0057462,
- 1 => 0.0051688,
+ 7940 => 0.0056277,
},
},
},
{
"next": {
"probabilities": Map {
- 927 => 0.9811904,
+ 927 => 0.9799656,
- 198 => 0.0033849,
+ 198 => 0.0035156,
- 6288 => 0.0032705,
+ 6288 => 0.002895,
- 279 => 0.0006553,
+ 279 => 0.0007419,
- 1633 => 0.0003184,
+ 264 => 0.0003396,
- 1035 => 0.0003114,
+ 297 => 0.0003235,
- 13 => 0.0002917,
+ 1633 => 0.000317,
- 264 => 0.0002895,
+ 1579 => 0.0003098,
- 297 => 0.0002833,
+ 13 => 0.0003071,
- 720 => 0.000249,
+ 1035 => 0.0003007,
},
"token": 927,
},
},
{
"next": {
- "confidence": 0.9306729,
+ "confidence": 0.9299947,
"token": 279,
},
},
{
"next": {
- "confidence": 0.9597685,
+ "confidence": 0.9565306,
"probabilities": Map {
- 16053 => 0.9597685,
+ 16053 => 0.9565306,
- 1208 => 0.0047506,
+ 1208 => 0.0052157,
- 198 => 0.0031827,
+ 5679 => 0.0031882,
- 5679 => 0.0029162,
+ 198 => 0.0030333,
- 65536 => 0.0019724,
+ 65536 => 0.002155,
- 6435 => 0.0009124,
+ 6435 => 0.0010304,
- 2697 => 0.0006706,
+ 21811 => 0.0006608,
- 720 => 0.0005979,
+ 32681 => 0.0005958,
- 21811 => 0.0005517,
+ 2697 => 0.0005905,
- 45363 => 0.0005495,
+ 45363 => 0.0005836,
},
},
},
{
"next": {
- "confidence": 0.987146,
+ "confidence": 0.9837725,
"probabilities": Map {
- 5679 => 0.987146,
+ 5679 => 0.9837725,
- 21811 => 0.0014387,
+ 21811 => 0.0018095,
- 198 => 0.0009368,
+ 198 => 0.0010854,
- 8415 => 0.0007225,
+ 8415 => 0.0008999,
- 12875 => 0.0003803,
+ 4194 => 0.0004496,
- 4194 => 0.000347,
+ 12875 => 0.0004042,
- 720 => 0.0002815,
+ 720 => 0.0003245,
- 14588 => 0.0002761,
+ 14588 => 0.0003239,
- 9522 => 0.0002417,
+ 9522 => 0.0003201,
- 627 => 0.0002042,
+ 627 => 0.0002572,
},
"token": 5679,
},
},
]
❯ test/modelDependent/llama3.1/controlledEvaluate.test.ts:83:35
|
|
test/modelDependent/llama3.1/evaluateWithMetadata.test.ts > llama 3.1 > evaluate with metadata > with probabilities:
test/modelDependent/llama3.1/evaluateWithMetadata.test.ts#L93
Error: Snapshot `llama 3.1 > evaluate with metadata > with probabilities 1` mismatched
- Expected
+ Received
[
{
"probabilities": Map {
- 578 => 0.4307296,
+ 578 => 0.4234016,
- 1115 => 0.1304376,
+ 1115 => 0.1350679,
- 1102 => 0.0516853,
+ 1102 => 0.0514064,
- 763 => 0.042889,
+ 763 => 0.0404493,
- 1283 => 0.029397,
+ 1283 => 0.0289963,
- 2100 => 0.0293787,
+ 2100 => 0.0283249,
- 15636 => 0.0262684,
+ 15636 => 0.0264627,
- 2030 => 0.021849,
+ 2030 => 0.0217779,
- 320 => 0.016903,
+ 320 => 0.0198824,
- 1628 => 0.0118695,
+ 1628 => 0.0125392,
},
"token": 578,
},
{
"probabilities": Map {
- 16053 => 0.4223004,
+ 16053 => 0.4444298,
- 4062 => 0.303636,
+ 4062 => 0.282037,
- 39935 => 0.0603957,
+ 39935 => 0.0518339,
- 2944 => 0.0373029,
+ 2944 => 0.039056,
- 5679 => 0.0238112,
+ 5679 => 0.0239568,
- 11914 => 0.0162982,
+ 11914 => 0.0161418,
- 2144 => 0.0146834,
+ 2144 => 0.0160438,
- 1121 => 0.0069844,
+ 1121 => 0.0065072,
- 17571 => 0.0057943,
+ 17571 => 0.0058377,
- 3446 => 0.0049344,
+ 3446 => 0.004879,
},
"token": 16053,
},
{
"probabilities": Map {
- 5679 => 0.9981183,
+ 5679 => 0.9980332,
- 12875 => 0.0001592,
+ 12875 => 0.0001527,
- 18964 => 0.0001154,
+ 39935 => 0.000116,
- 39935 => 0.000115,
+ 18964 => 0.0001074,
- 13 => 0.0001049,
+ 13 => 0.0001033,
- 627 => 0.0000928,
+ 627 => 0.0000947,
- 656 => 0.0000626,
+ 893 => 0.0000697,
- 893 => 0.0000563,
+ 656 => 0.0000611,
- 198 => 0.0000522,
+ 374 => 0.0000547,
- 374 => 0.0000519,
+ 198 => 0.0000541,
},
"token": 5679,
},
{
"probabilities": Map {
- 374 => 0.8126624,
+ 374 => 0.8100054,
- 1587 => 0.0481527,
+ 1587 => 0.0497223,
- 596 => 0.0247333,
+ 596 => 0.027551,
- 1120 => 0.022303,
+ 3250 => 0.0219455,
- 3250 => 0.0215451,
+ 1120 => 0.0206675,
- 706 => 0.0161837,
+ 706 => 0.0160834,
- 15849 => 0.0086941,
+ 15849 => 0.0083946,
- 1053 => 0.0059118,
+ 1053 => 0.0057803,
- 55064 => 0.0037806,
+ 11 => 0.0039905,
- 11 => 0.0036656,
+ 55064 => 0.0035692,
},
"token": 374,
},
{
"probabilities": Map {
- 2288 => 0.275759,
+ 2288 => 0.2930796,
- 1120 => 0.1666484,
+ 539 => 0.1523461,
- 539 => 0.1577515,
+ 1120 => 0.1495192,
- 779 => 0.1334379,
+ 779 => 0.143051,
- 264 => 0.055854,
+ 264 => 0.0561702,
- 1101 => 0.0292162,
+ 1101 => 0.0285375,
- 16053 => 0.0176804,
+ 16053 => 0.0172203,
- 5042 => 0.0158628,
+ 5042 => 0.0158594,
- 1193 => 0.014583,
+ 2744 => 0.012851,
- 2744 => 0.0140902,
+ 1193 => 0.0126306,
},
"token": 2288,
},
{
"probabilities": Map {
- 16053 => 0.9066131,
+ 16053 => 0.9030056,
- 13326 => 0.0636239,
+ 13326 => 0.0659881,
- 19781 => 0.0071587,
+ 19781 => 0.0069219,
- 17551 => 0.0020244,
+ 17551 => 0.0020206,
- 10968 => 0.0012676,
+ 10968 => 0.0014594,
- 11920 => 0.0011004,
+ 6435 => 0.0010619,
- 6435 => 0.0010087,
+ 11920 => 0.0010104,
- 34386 => 0.0007757,
+ 34386 => 0.0007334,
- 1208 => 0.0006099,
+ 1208 => 0.0006973,
- 25366 => 0.0005671,
+ 25366 => 0.0005825,
},
"token": 16053,
},
{
"probabilities": Map {
- 311 => 0.9882798,
+ 311 => 0.9878785,
- 1524 => 0.0061862,
+ 1524 => 0.0062293,
- 11 => 0.0025764,
+ 11 => 0.0027594,
- 323 => 0.0005241,
+ 323 => 0.0006127,
- 13 => 0.0003535,
+ 13 => 0.0003716,
- 627 => 0.000321,
+
|
|
test/modelDependent/llama3.1/evaluateWithMetadata.test.ts > llama 3.1 > evaluate with metadata > with confidence:
test/modelDependent/llama3.1/evaluateWithMetadata.test.ts#L272
Error: Snapshot `llama 3.1 > evaluate with metadata > with confidence 1` mismatched
- Expected
+ Received
[
{
- "confidence": 0.4307296,
+ "confidence": 0.4234016,
"token": 578,
},
{
- "confidence": 0.4223004,
+ "confidence": 0.4444298,
"token": 16053,
},
{
- "confidence": 0.9981183,
+ "confidence": 0.9980332,
"token": 5679,
},
{
- "confidence": 0.8126624,
+ "confidence": 0.8100054,
"token": 374,
},
{
- "confidence": 0.275759,
+ "confidence": 0.2930796,
"token": 2288,
},
{
- "confidence": 0.9066131,
+ "confidence": 0.9030056,
"token": 16053,
},
{
- "confidence": 0.9882798,
+ "confidence": 0.9878785,
"token": 311,
},
{
- "confidence": 0.7492305,
+ "confidence": 0.7602303,
"token": 2512,
},
{
- "confidence": 0.9521903,
+ "confidence": 0.9483847,
"token": 922,
},
{
- "confidence": 0.6508099,
+ "confidence": 0.6830861,
"token": 279,
},
]
❯ test/modelDependent/llama3.1/evaluateWithMetadata.test.ts:272:25
|
|
test/modelDependent/llama3.1/evaluateWithMetadata.test.ts > llama 3.1 > evaluate with metadata > with probabilities and confidence:
test/modelDependent/llama3.1/evaluateWithMetadata.test.ts#L341
Error: Snapshot `llama 3.1 > evaluate with metadata > with probabilities and confidence 1` mismatched
- Expected
+ Received
[
{
- "confidence": 0.4307296,
+ "confidence": 0.4234016,
"probabilities": Map {
- 578 => 0.4307296,
+ 578 => 0.4234016,
- 1115 => 0.1304376,
+ 1115 => 0.1350679,
- 1102 => 0.0516853,
+ 1102 => 0.0514064,
- 763 => 0.042889,
+ 763 => 0.0404493,
- 1283 => 0.029397,
+ 1283 => 0.0289963,
- 2100 => 0.0293787,
+ 2100 => 0.0283249,
- 15636 => 0.0262684,
+ 15636 => 0.0264627,
- 2030 => 0.021849,
+ 2030 => 0.0217779,
- 320 => 0.016903,
+ 320 => 0.0198824,
- 1628 => 0.0118695,
+ 1628 => 0.0125392,
},
"token": 578,
},
{
- "confidence": 0.4223004,
+ "confidence": 0.4444298,
"probabilities": Map {
- 16053 => 0.4223004,
+ 16053 => 0.4444298,
- 4062 => 0.303636,
+ 4062 => 0.282037,
- 39935 => 0.0603957,
+ 39935 => 0.0518339,
- 2944 => 0.0373029,
+ 2944 => 0.039056,
- 5679 => 0.0238112,
+ 5679 => 0.0239568,
- 11914 => 0.0162982,
+ 11914 => 0.0161418,
- 2144 => 0.0146834,
+ 2144 => 0.0160438,
- 1121 => 0.0069844,
+ 1121 => 0.0065072,
- 17571 => 0.0057943,
+ 17571 => 0.0058377,
- 3446 => 0.0049344,
+ 3446 => 0.004879,
},
"token": 16053,
},
{
- "confidence": 0.9981183,
+ "confidence": 0.9980332,
"probabilities": Map {
- 5679 => 0.9981183,
+ 5679 => 0.9980332,
- 12875 => 0.0001592,
+ 12875 => 0.0001527,
- 18964 => 0.0001154,
+ 39935 => 0.000116,
- 39935 => 0.000115,
+ 18964 => 0.0001074,
- 13 => 0.0001049,
+ 13 => 0.0001033,
- 627 => 0.0000928,
+ 627 => 0.0000947,
- 656 => 0.0000626,
+ 893 => 0.0000697,
- 893 => 0.0000563,
+ 656 => 0.0000611,
- 198 => 0.0000522,
+ 374 => 0.0000547,
- 374 => 0.0000519,
+ 198 => 0.0000541,
},
"token": 5679,
},
{
- "confidence": 0.8126624,
+ "confidence": 0.8100054,
"probabilities": Map {
- 374 => 0.8126624,
+ 374 => 0.8100054,
- 1587 => 0.0481527,
+ 1587 => 0.0497223,
- 596 => 0.0247333,
+ 596 => 0.027551,
- 1120 => 0.022303,
+ 3250 => 0.0219455,
- 3250 => 0.0215451,
+ 1120 => 0.0206675,
- 706 => 0.0161837,
+ 706 => 0.0160834,
- 15849 => 0.0086941,
+ 15849 => 0.0083946,
- 1053 => 0.0059118,
+ 1053 => 0.0057803,
- 55064 => 0.0037806,
+ 11 => 0.0039905,
- 11 => 0.0036656,
+ 55064 => 0.0035692,
},
"token": 374,
},
{
- "confidence": 0.275759,
+ "confidence": 0.2930796,
"probabilities": Map {
- 2288 => 0.275759,
+ 2288 => 0.2930796,
- 1120 => 0.1666484,
+ 539 => 0.1523461,
- 539 => 0.1577515,
+ 1120 => 0.1495192,
- 779 => 0.1334379,
+ 779 => 0.143051,
- 264 => 0.055854,
+ 264 => 0.0561702,
- 1101 => 0.0292162,
+ 1101 => 0.0285375,
- 16053 => 0.0176804,
+ 16053 => 0.0172203,
- 5042 => 0.0158628,
+ 5042 => 0.0158594,
- 1193 => 0.014583,
+ 2744 => 0.012851,
- 2744 => 0.0140902,
+ 1193 => 0.0126306,
},
"token": 2288,
},
{
- "confidence": 0.9066131,
+ "confidence": 0.9030056,
"probabilities": Map {
- 16053 => 0.9066131,
+ 16053 => 0.9030056,
- 13326 => 0.0636239,
+ 13326 => 0.0659881,
- 19781 => 0.0071587,
+ 19781 => 0.0069219,
- 17551 => 0.0020244,
+ 17551 => 0.0020206,
- 10968 => 0.0012676,
+ 10968 => 0.0014594,
- 11920 => 0.0011004,
+ 6435 => 0.0010619,
- 6435 => 0.0010087,
+ 11920 => 0.0010104,
- 34386 => 0.0007757,
+ 34386 => 0.0007334,
- 1208 => 0.0006099,
+ 1208 => 0.0006973,
- 25366 => 0.0005671,
+
|
|
Model dependent tests
Process completed with exit code 1.
|
|
Build
ubuntu-latest pipelines will use ubuntu-24.04 soon. For more details, see https://github.com/actions/runner-images/issues/10636
|
Artifacts
Produced during runtime
| Name | Size | |
|---|---|---|
|
bins-linux
|
154 MB |
|
|
bins-mac-arm64
|
1.16 MB |
|
|
bins-mac-x64
|
2.32 MB |
|
|
bins-win-arm
|
1.5 MB |
|
|
bins-win-x64
|
151 MB |
|
|
build
|
702 KB |
|
|
build-templates
|
26.3 KB |
|
|
llama.cpp
|
74 MB |
|