Evaluation Metrics

15-evaluation-metrics.ipynb

@@ -0,0 +1 @@
+{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"reference = \"the cat sat on the mat\"\nprediction = \"the cat sit on the\"","metadata":{"execution":{"iopub.status.busy":"2023-08-22T21:01:30.201600Z","iopub.execute_input":"2023-08-22T21:01:30.201982Z","iopub.status.idle":"2023-08-22T21:01:30.206020Z","shell.execute_reply.started":"2023-08-22T21:01:30.201950Z","shell.execute_reply":"2023-08-22T21:01:30.205392Z"},"trusted":true},"execution_count":7,"outputs":[]},{"cell_type":"markdown","source":"# Word Error Rate","metadata":{}},{"cell_type":"code","source":"from evaluate import load\n\nwer_metric = load(\"wer\")\n\nwer = wer_metric.compute(references=[reference], predictions=[prediction])\n\nprint(wer)","metadata":{"execution":{"iopub.status.busy":"2023-08-22T21:01:41.076529Z","iopub.execute_input":"2023-08-22T21:01:41.076869Z","iopub.status.idle":"2023-08-22T21:01:41.696999Z","shell.execute_reply.started":"2023-08-22T21:01:41.076844Z","shell.execute_reply":"2023-08-22T21:01:41.696100Z"},"trusted":true},"execution_count":8,"outputs":[{"name":"stdout","text":"0.3333333333333333\n","output_type":"stream"}]},{"cell_type":"code","source":"from transformers.models.whisper.english_normalizer import BasicTextNormalizer\n\nnormalizer = BasicTextNormalizer()\n\nprediction = \" He tells us that at this festive season of the year, with Christmas and roast beef looming before us, similarly is drawn from eating and its results occur most readily to the mind.\"\nnormalized_prediction = normalizer(prediction)\n\nnormalized_prediction","metadata":{"execution":{"iopub.status.busy":"2023-08-22T21:02:55.209601Z","iopub.execute_input":"2023-08-22T21:02:55.209910Z","iopub.status.idle":"2023-08-22T21:02:55.220099Z","shell.execute_reply.started":"2023-08-22T21:02:55.209886Z","shell.execute_reply":"2023-08-22T21:02:55.219078Z"},"trusted":true},"execution_count":9,"outputs":[{"execution_count":9,"output_type":"execute_result","data":{"text/plain":"' he tells us that at this festive season of the year with christmas and roast beef looming before us similarly is drawn from eating and its results occur most readily to the mind '"},"metadata":{}}]},{"cell_type":"code","source":"reference = \"HE TELLS US THAT AT THIS FESTIVE SEASON OF THE YEAR WITH CHRISTMAS AND ROAST BEEF LOOMING BEFORE US SIMILES DRAWN FROM EATING AND ITS RESULTS OCCUR MOST READILY TO THE MIND\"\nnormalized_referece = normalizer(reference)\n\nwer = wer_metric.compute(\n    references=[normalized_referece], predictions=[normalized_prediction]\n)\nwer","metadata":{"execution":{"iopub.status.busy":"2023-08-22T21:03:20.238596Z","iopub.execute_input":"2023-08-22T21:03:20.238921Z","iopub.status.idle":"2023-08-22T21:03:20.252047Z","shell.execute_reply.started":"2023-08-22T21:03:20.238898Z","shell.execute_reply":"2023-08-22T21:03:20.251010Z"},"trusted":true},"execution_count":10,"outputs":[{"execution_count":10,"output_type":"execute_result","data":{"text/plain":"0.0625"},"metadata":{}}]},{"cell_type":"code","source":"from transformers import pipeline\nimport torch\n\nif torch.cuda.is_available():\n    device = \"cuda:0\"\n    torch_dtype = torch.float16\nelse:\n    device = \"cpu\"\n    torch_dtype = torch.float32\n\npipe = pipeline(\n    \"automatic-speech-recognition\",\n    model=\"openai/whisper-small\",\n    torch_dtype=torch_dtype,\n    device=device,\n)","metadata":{"execution":{"iopub.status.busy":"2023-08-22T21:03:38.754647Z","iopub.execute_input":"2023-08-22T21:03:38.754993Z","iopub.status.idle":"2023-08-22T21:03:48.109904Z","shell.execute_reply.started":"2023-08-22T21:03:38.754969Z","shell.execute_reply":"2023-08-22T21:03:48.109053Z"},"trusted":true},"execution_count":11,"outputs":[{"output_type":"display_data","data":{"text/plain":"Downloading (…)lve/main/config.json:   0%|          | 0.00/1.97k [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"742f52c7f607469eb2c30d1f7116d844"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Downloading model.safetensors:   0%|          | 0.00/967M [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"4fcb9f7fa48340bc9a985961f512ad92"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Downloading (…)neration_config.json:   0%|          | 0.00/3.84k [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"3014a1613c974f8ab65ebe60f888cbb5"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Downloading (…)okenizer_config.json:   0%|          | 0.00/842 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"2c87852fae7847219e808ac093e8fb82"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Downloading (…)olve/main/vocab.json:   0%|          | 0.00/1.04M [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"72fd842075de45b28128371a615bd110"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Downloading (…)/main/tokenizer.json:   0%|          | 0.00/2.20M [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"5eb68cd59ec94cc58784ed01d53bfeb1"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Downloading (…)olve/main/merges.txt:   0%|          | 0.00/494k [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"30cbdd0452b248058bc7020c7b0b982a"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Downloading (…)main/normalizer.json:   0%|          | 0.00/52.7k [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"d0c2198b6bdd4a2591edad14b7a34bf6"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Downloading (…)in/added_tokens.json:   0%|          | 0.00/2.08k [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"b7cbfa45082c4705b93802ae1840ba26"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Downloading (…)cial_tokens_map.json:   0%|          | 0.00/2.08k [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"80daf4ba24b8474c881f12c2e18dd745"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Downloading (…)rocessor_config.json:   0%|          | 0.00/185k [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"af8a1d30ad424f65a1c568e54b636cea"}},"metadata":{}}]},{"cell_type":"code","source":"from huggingface_hub import notebook_login\n\nnotebook_login()","metadata":{"execution":{"iopub.status.busy":"2023-08-22T21:04:19.672434Z","iopub.execute_input":"2023-08-22T21:04:19.672765Z","iopub.status.idle":"2023-08-22T21:04:19.700910Z","shell.execute_reply.started":"2023-08-22T21:04:19.672741Z","shell.execute_reply":"2023-08-22T21:04:19.700244Z"},"trusted":true},"execution_count":12,"outputs":[{"output_type":"display_data","data":{"text/plain":"VBox(children=(HTML(value='<center> <img\\nsrc=https://huggingface.co/front/assets/huggingface_logo-noborder.sv…","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"2d5947454a8448c8a4d55e8e3625c753"}},"metadata":{}}]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}