diff --git a/404.html b/404.html index 6c089aa..4f6fae7 100644 --- a/404.html +++ b/404.html @@ -5,7 +5,7 @@ Page Not Found | Stable Code Docs - + diff --git a/assets/js/546e7911.a4d1327e.js b/assets/js/546e7911.a4d1327e.js deleted file mode 100644 index 486a722..0000000 --- a/assets/js/546e7911.a4d1327e.js +++ /dev/null @@ -1 +0,0 @@ -"use strict";(self.webpackChunkstable_code=self.webpackChunkstable_code||[]).push([[561],{6432:(e,n,t)=>{t.r(n),t.d(n,{assets:()=>r,contentTitle:()=>i,default:()=>u,frontMatter:()=>l,metadata:()=>s,toc:()=>c});var a=t(4848),o=t(8453);const l={sidebar_position:1,title:"Stable Code Finetuning with Axolotl"},i=void 0,s={id:"tutorial-finetuning/stable_code_axolotl",title:"Stable Code Finetuning with Axolotl",description:"",source:"@site/docs/tutorial-finetuning/stable_code_axolotl.mdx",sourceDirName:"tutorial-finetuning",slug:"/tutorial-finetuning/stable_code_axolotl",permalink:"/docs/tutorial-finetuning/stable_code_axolotl",draft:!1,unlisted:!1,editUrl:"https://github.com/facebook/docusaurus/tree/main/packages/create-docusaurus/templates/shared/docs/tutorial-finetuning/stable_code_axolotl.mdx",tags:[],version:"current",sidebarPosition:1,frontMatter:{sidebar_position:1,title:"Stable Code Finetuning with Axolotl"},sidebar:"tutorialSidebar",previous:{title:"Tutorial - Finetuning Stable Code",permalink:"/docs/category/tutorial---finetuning-stable-code"}},r={},c=[];function d(e){const n={code:"code",pre:"pre",...(0,o.R)(),...e.components};return(0,a.jsxs)(a.Fragment,{children:[(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-python",children:"import torch\n# Check so there is a gpu available, a T4(free tier) is enough to run this notebook\nassert (torch.cuda.is_available()==True)\n"})}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-python",children:'!pip install torch=="2.1.2"\n!pip install -e git+https://github.com/OpenAccess-AI-Collective/axolotl#egg=axolotl\n!pip install flash-attn=="2.5.0"\n!pip install deepspeed=="0.13.1"\n'})}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-python",children:'import yaml\n\n# Your YAML string\nyaml_string = """\nbase_model: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T\nmodel_type: LlamaForCausalLM\ntokenizer_type: LlamaTokenizer\nis_llama_derived_model: true\n\nload_in_8bit: false\nload_in_4bit: true\nstrict: false\n\ndatasets:\n - path: mhenrichsen/alpaca_2k_test\n type: alpaca\ndataset_prepared_path:\nval_set_size: 0.05\noutput_dir: ./qlora-out\n\nadapter: qlora\nlora_model_dir:\n\nsequence_len: 1096\nsample_packing: true\npad_to_sequence_len: true\n\nlora_r: 32\nlora_alpha: 16\nlora_dropout: 0.05\nlora_target_modules:\nlora_target_linear: true\nlora_fan_in_fan_out:\n\nwandb_project:\nwandb_entity:\nwandb_watch:\nwandb_name:\nwandb_log_model:\n\nmlflow_experiment_name: colab-example\n\ngradient_accumulation_steps: 1\nmicro_batch_size: 1\nnum_epochs: 4\nmax_steps: 20\noptimizer: paged_adamw_32bit\nlr_scheduler: cosine\nlearning_rate: 0.0002\n\ntrain_on_inputs: false\ngroup_by_length: false\nbf16: false\nfp16: true\ntf32: false\n\ngradient_checkpointing: true\nearly_stopping_patience:\nresume_from_checkpoint:\nlocal_rank:\nlogging_steps: 1\nxformers_attention:\nflash_attention: false\n\nwarmup_steps: 10\nevals_per_epoch:\nsaves_per_epoch:\ndebug:\ndeepspeed:\nweight_decay: 0.0\nfsdp:\nfsdp_config:\nspecial_tokens:\n\n"""\n\n# Convert the YAML string to a Python dictionary\nyaml_dict = yaml.safe_load(yaml_string)\n\n# Specify your file path\nfile_path = \'test_axolotl.yaml\'\n\n# Write the YAML file\nwith open(file_path, \'w\') as file:\n yaml.dump(yaml_dict, file)\n'})}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-python",children:"# Buy using the ! the comand will be executed as a bash command\n!accelerate launch -m axolotl.cli.train /content/test_axolotl.yaml\n"})}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-python",children:'# Buy using the ! the comand will be executed as a bash command\n!accelerate launch -m axolotl.cli.inference /content/test_axolotl.yaml \\\n --qlora_model_dir="./qlora-out" --gradio\n'})})]})}function u(e={}){const{wrapper:n}={...(0,o.R)(),...e.components};return n?(0,a.jsx)(n,{...e,children:(0,a.jsx)(d,{...e})}):d(e)}},8453:(e,n,t)=>{t.d(n,{R:()=>i,x:()=>s});var a=t(6540);const o={},l=a.createContext(o);function i(e){const n=a.useContext(l);return a.useMemo((function(){return"function"==typeof e?e(n):{...n,...e}}),[n,e])}function s(e){let n;return n=e.disableParentContext?"function"==typeof e.components?e.components(o):e.components||o:i(e.components),a.createElement(l.Provider,{value:n},e.children)}}}]); \ No newline at end of file diff --git a/assets/js/546e7911.b94e85be.js b/assets/js/546e7911.b94e85be.js new file mode 100644 index 0000000..30e2be6 --- /dev/null +++ b/assets/js/546e7911.b94e85be.js @@ -0,0 +1 @@ +"use strict";(self.webpackChunkstable_code=self.webpackChunkstable_code||[]).push([[561],{6432:(e,n,t)=>{t.r(n),t.d(n,{assets:()=>r,contentTitle:()=>i,default:()=>u,frontMatter:()=>l,metadata:()=>s,toc:()=>c});var o=t(4848),a=t(8453);const l={sidebar_position:1,title:"Stable Code Finetuning with Axolotl"},i="Stable Code Finetuning with Axolotl",s={id:"tutorial-finetuning/stable_code_axolotl",title:"Stable Code Finetuning with Axolotl",description:"Axolotl is a popular library for finetuning models. In this tutorial, we",source:"@site/docs/tutorial-finetuning/stable_code_axolotl.mdx",sourceDirName:"tutorial-finetuning",slug:"/tutorial-finetuning/stable_code_axolotl",permalink:"/docs/tutorial-finetuning/stable_code_axolotl",draft:!1,unlisted:!1,editUrl:"https://github.com/facebook/docusaurus/tree/main/packages/create-docusaurus/templates/shared/docs/tutorial-finetuning/stable_code_axolotl.mdx",tags:[],version:"current",sidebarPosition:1,frontMatter:{sidebar_position:1,title:"Stable Code Finetuning with Axolotl"},sidebar:"tutorialSidebar",previous:{title:"Tutorial - Finetuning Stable Code",permalink:"/docs/category/tutorial---finetuning-stable-code"}},r={},c=[];function d(e){const n={a:"a",code:"code",h1:"h1",p:"p",pre:"pre",...(0,a.R)(),...e.components};return(0,o.jsxs)(o.Fragment,{children:[(0,o.jsx)(n.h1,{id:"stable-code-finetuning-with-axolotl",children:"Stable Code Finetuning with Axolotl"}),"\n",(0,o.jsx)(n.p,{children:"Axolotl is a popular library for finetuning models. In this tutorial, we\nwill use Axolotl to finetune our Stable Code 3B model on FORTRAN code to\nshow you how to create a custom code completion model on a new\nprogramming language."}),"\n",(0,o.jsxs)(n.p,{children:["This tutorial is based on the wonderful notebooks from Maxime Labonne\u2019s\n",(0,o.jsx)(n.a,{href:"https://github.com/mlabonne/llm-course/tree/main",children:"llm course"})]}),"\n",(0,o.jsxs)("figure",{children:[(0,o.jsx)("a",{href:"https://colab.research.google.com/github/Stability-AI/stable-code-docs/blob/main/docs/tutorial-finetuning/stable_code_axolotl.ipynb",children:(0,o.jsx)("img",{src:"https://colab.research.google.com/assets/colab-badge.svg",alt:"Open In Colab"})}),(0,o.jsx)("figcaption",{children:"Open In Colab"})]}),"\n",(0,o.jsx)(n.pre,{children:(0,o.jsx)(n.code,{className:"language-python",children:"import torch\n# Check so there is a gpu available, a T4(free tier) is enough to run this notebook\nassert (torch.cuda.is_available()==True)\n"})}),"\n",(0,o.jsx)(n.p,{children:"Let\u2019s get the environment set up first. We will go ahead and install the\nnecessary packages."}),"\n",(0,o.jsx)(n.pre,{children:(0,o.jsx)(n.code,{className:"language-python",children:'!pip install torch=="2.1.2"\n!pip install -e git+https://github.com/OpenAccess-AI-Collective/axolotl#egg=axolotl\n!pip install flash-attn=="2.5.0"\n!pip install deepspeed=="0.13.1"\n'})}),"\n",(0,o.jsxs)(n.p,{children:["Axolotl relies on a config file written in YAML to specify the model and\nthe dataset and how to train the model. We will create a config file for\nour finetuning task to finetune the Stable Code 3B model on FORTRAN code\nusing QLoRA. The dataset we are picking is from\n",(0,o.jsx)(n.code,{children:"codeparrot/github-code-clean"})," dataset which is a collection of code\nscraped from GitHub. Specifically, we will be utilizing the FORTRAN MIT\nlicensed code."]}),"\n",(0,o.jsx)(n.pre,{children:(0,o.jsx)(n.code,{className:"language-python",children:'import yaml\n\n# Your YAML string\nyaml_string = """\nbase_model: stabilityai/stable-code-3b\nmodel_type: AutoModelForCausalLM\ntokenizer_type: AutoTokenizer\n\nload_in_8bit: false\nload_in_4bit: true\nstrict: false\n\ndatasets:\n - path: codeparrot/github-code-clean\n name: FORTRAN-mit\n type: completion\n field: code\ndataset_prepared_path:\nval_set_size: 0.05\noutput_dir: ./qlora-out\n\nadapter: qlora\nlora_model_dir:\n\nsequence_len: 1096\nsample_packing: true\npad_to_sequence_len: true\n\nlora_r: 32\nlora_alpha: 16\nlora_dropout: 0.05\nlora_target_modules:\nlora_target_linear: true\nlora_fan_in_fan_out:\n\nwandb_project:\nwandb_entity:\nwandb_watch:\nwandb_name:\nwandb_log_model:\n\nmlflow_experiment_name: colab-example\n\ngradient_accumulation_steps: 1\nmicro_batch_size: 1\nnum_epochs: 4\nmax_steps: 20\noptimizer: paged_adamw_32bit\nlr_scheduler: cosine\nlearning_rate: 0.0002\n\ntrain_on_inputs: false\ngroup_by_length: false\nbf16: false\nfp16: true\ntf32: false\n\ngradient_checkpointing: true\nearly_stopping_patience:\nresume_from_checkpoint:\nlocal_rank:\nlogging_steps: 1\nxformers_attention:\nflash_attention: false\n\nwarmup_steps: 10\nevals_per_epoch:\nsaves_per_epoch:\ndebug:\ndeepspeed:\nweight_decay: 0.0\nfsdp:\nfsdp_config:\nspecial_tokens:\n\n"""\n\n# Convert the YAML string to a Python dictionary\nyaml_dict = yaml.safe_load(yaml_string)\n\n# Specify your file path\nfile_path = \'test_axolotl.yaml\'\n\n# Write the YAML file\nwith open(file_path, \'w\') as file:\n yaml.dump(yaml_dict, file)\n'})}),"\n",(0,o.jsx)(n.p,{children:"Next, we can launch the run, which should take about XX if you are\nrunning this in a Colab environment with a T4 GPU:"}),"\n",(0,o.jsx)(n.pre,{children:(0,o.jsx)(n.code,{className:"language-python",children:"# Buy using the ! the comand will be executed as a bash command\n!accelerate launch -m axolotl.cli.train /content/test_axolotl.yaml\n"})}),"\n",(0,o.jsx)(n.p,{children:"Once the model has been finetuned, we can run it in a gradio\nenvironment:"}),"\n",(0,o.jsx)(n.pre,{children:(0,o.jsx)(n.code,{className:"language-python",children:'# Buy using the ! the comand will be executed as a bash command\n!accelerate launch -m axolotl.cli.inference /content/test_axolotl.yaml \\\n --qlora_model_dir="./qlora-out" --gradio\n'})})]})}function u(e={}){const{wrapper:n}={...(0,a.R)(),...e.components};return n?(0,o.jsx)(n,{...e,children:(0,o.jsx)(d,{...e})}):d(e)}},8453:(e,n,t)=>{t.d(n,{R:()=>i,x:()=>s});var o=t(6540);const a={},l=o.createContext(a);function i(e){const n=o.useContext(l);return o.useMemo((function(){return"function"==typeof e?e(n):{...n,...e}}),[n,e])}function s(e){let n;return n=e.disableParentContext?"function"==typeof e.components?e.components(a):e.components||a:i(e.components),o.createElement(l.Provider,{value:n},e.children)}}}]); \ No newline at end of file diff --git a/assets/js/935f2afb.17363a8a.js b/assets/js/935f2afb.7438d976.js similarity index 69% rename from assets/js/935f2afb.17363a8a.js rename to assets/js/935f2afb.7438d976.js index 7d08254..cb191ad 100644 --- a/assets/js/935f2afb.17363a8a.js +++ b/assets/js/935f2afb.7438d976.js @@ -1 +1 @@ -"use strict";(self.webpackChunkstable_code=self.webpackChunkstable_code||[]).push([[581],{5610:e=>{e.exports=JSON.parse('{"pluginId":"default","version":"current","label":"Next","banner":null,"badge":false,"noIndex":false,"className":"docs-version-current","isLast":true,"docsSidebars":{"tutorialSidebar":[{"type":"link","label":"Stable Code Overview","href":"/docs/overview","docId":"overview","unlisted":false},{"type":"category","label":"Tutorial - Using Stable Code","collapsible":true,"collapsed":true,"items":[{"type":"link","label":"Stable Code on Ollama","href":"/docs/tutorial-usage/ollama","docId":"tutorial-usage/ollama","unlisted":false},{"type":"link","label":"VSCode Extension","href":"/docs/tutorial-usage/vscode-extension","docId":"tutorial-usage/vscode-extension","unlisted":false},{"type":"link","label":"Repositories QA","href":"/docs/tutorial-usage/repository_qa","docId":"tutorial-usage/repository_qa","unlisted":false}],"href":"/docs/category/tutorial---using-stable-code"},{"type":"category","label":"Tutorial - Finetuning Stable Code","collapsible":true,"collapsed":true,"items":[{"type":"link","label":"Stable Code Finetuning with Axolotl","href":"/docs/tutorial-finetuning/stable_code_axolotl","docId":"tutorial-finetuning/stable_code_axolotl","unlisted":false}],"href":"/docs/category/tutorial---finetuning-stable-code"}]},"docs":{"overview":{"id":"overview","title":"Stable Code Overview","description":"Let\'s discover Docusaurus in less than 5 minutes.","sidebar":"tutorialSidebar"},"tutorial-finetuning/stable_code_axolotl":{"id":"tutorial-finetuning/stable_code_axolotl","title":"Stable Code Finetuning with Axolotl","description":"","sidebar":"tutorialSidebar"},"tutorial-usage/ollama":{"id":"tutorial-usage/ollama","title":"Stable Code on Ollama","description":"Stable Code is available on Ollama for running and serving. To do so, make sure to first download Ollama for your OS//ollama.com/download","sidebar":"tutorialSidebar"},"tutorial-usage/repository_qa":{"id":"tutorial-usage/repository_qa","title":"Repositories QA","description":"In this tutorial, we will show you how you can create a simple program","sidebar":"tutorialSidebar"},"tutorial-usage/vscode-extension":{"id":"tutorial-usage/vscode-extension","title":"VSCode Extension","description":"","sidebar":"tutorialSidebar"}}}')}}]); \ No newline at end of file +"use strict";(self.webpackChunkstable_code=self.webpackChunkstable_code||[]).push([[581],{5610:e=>{e.exports=JSON.parse('{"pluginId":"default","version":"current","label":"Next","banner":null,"badge":false,"noIndex":false,"className":"docs-version-current","isLast":true,"docsSidebars":{"tutorialSidebar":[{"type":"link","label":"Stable Code Overview","href":"/docs/overview","docId":"overview","unlisted":false},{"type":"category","label":"Tutorial - Using Stable Code","collapsible":true,"collapsed":true,"items":[{"type":"link","label":"Stable Code on Ollama","href":"/docs/tutorial-usage/ollama","docId":"tutorial-usage/ollama","unlisted":false},{"type":"link","label":"VSCode Extension","href":"/docs/tutorial-usage/vscode-extension","docId":"tutorial-usage/vscode-extension","unlisted":false},{"type":"link","label":"Repositories QA","href":"/docs/tutorial-usage/repository_qa","docId":"tutorial-usage/repository_qa","unlisted":false}],"href":"/docs/category/tutorial---using-stable-code"},{"type":"category","label":"Tutorial - Finetuning Stable Code","collapsible":true,"collapsed":true,"items":[{"type":"link","label":"Stable Code Finetuning with Axolotl","href":"/docs/tutorial-finetuning/stable_code_axolotl","docId":"tutorial-finetuning/stable_code_axolotl","unlisted":false}],"href":"/docs/category/tutorial---finetuning-stable-code"}]},"docs":{"overview":{"id":"overview","title":"Stable Code Overview","description":"Let\'s discover Docusaurus in less than 5 minutes.","sidebar":"tutorialSidebar"},"tutorial-finetuning/stable_code_axolotl":{"id":"tutorial-finetuning/stable_code_axolotl","title":"Stable Code Finetuning with Axolotl","description":"Axolotl is a popular library for finetuning models. In this tutorial, we","sidebar":"tutorialSidebar"},"tutorial-usage/ollama":{"id":"tutorial-usage/ollama","title":"Stable Code on Ollama","description":"Stable Code is available on Ollama for running and serving. To do so, make sure to first download Ollama for your OS//ollama.com/download","sidebar":"tutorialSidebar"},"tutorial-usage/repository_qa":{"id":"tutorial-usage/repository_qa","title":"Repositories QA","description":"In this tutorial, we will show you how you can create a simple program","sidebar":"tutorialSidebar"},"tutorial-usage/vscode-extension":{"id":"tutorial-usage/vscode-extension","title":"VSCode Extension","description":"","sidebar":"tutorialSidebar"}}}')}}]); \ No newline at end of file diff --git a/assets/js/runtime~main.2a226b91.js b/assets/js/runtime~main.9679b8b1.js similarity index 98% rename from assets/js/runtime~main.2a226b91.js rename to assets/js/runtime~main.9679b8b1.js index 1174217..e20b7c0 100644 --- a/assets/js/runtime~main.2a226b91.js +++ b/assets/js/runtime~main.9679b8b1.js @@ -1 +1 @@ -(()=>{"use strict";var e,a,t,r,c,o={},d={};function f(e){var a=d[e];if(void 0!==a)return a.exports;var t=d[e]={id:e,loaded:!1,exports:{}};return o[e].call(t.exports,t,t.exports,f),t.loaded=!0,t.exports}f.m=o,f.c=d,e=[],f.O=(a,t,r,c)=>{if(!t){var o=1/0;for(i=0;i=c)&&Object.keys(f.O).every((e=>f.O[e](t[n])))?t.splice(n--,1):(d=!1,c0&&e[i-1][2]>c;i--)e[i]=e[i-1];e[i]=[t,r,c]},f.n=e=>{var a=e&&e.__esModule?()=>e.default:()=>e;return f.d(a,{a:a}),a},t=Object.getPrototypeOf?e=>Object.getPrototypeOf(e):e=>e.__proto__,f.t=function(e,r){if(1&r&&(e=this(e)),8&r)return e;if("object"==typeof e&&e){if(4&r&&e.__esModule)return e;if(16&r&&"function"==typeof e.then)return e}var c=Object.create(null);f.r(c);var o={};a=a||[null,t({}),t([]),t(t)];for(var d=2&r&&e;"object"==typeof d&&!~a.indexOf(d);d=t(d))Object.getOwnPropertyNames(d).forEach((a=>o[a]=()=>e[a]));return o.default=()=>e,f.d(c,o),c},f.d=(e,a)=>{for(var t in a)f.o(a,t)&&!f.o(e,t)&&Object.defineProperty(e,t,{enumerable:!0,get:a[t]})},f.f={},f.e=e=>Promise.all(Object.keys(f.f).reduce(((a,t)=>(f.f[t](e,a),a)),[])),f.u=e=>"assets/js/"+({48:"a94703ab",61:"1f391b9e",98:"a7bd4aaa",134:"393be207",209:"01a85c17",249:"ccc49370",267:"a7023ddc",307:"962d33de",320:"9d646488",369:"9e421cea",374:"66406991",401:"17896441",413:"1db64337",472:"814f3328",497:"a80da1cf",561:"546e7911",581:"935f2afb",607:"62fdb048",634:"c4f5d8e4",643:"a6aa9e1f",647:"5e95c892",669:"30a24c52",694:"8717b14a",711:"9e4087bc",721:"46e868e2",722:"608ae6a4",727:"ff0a8395",806:"1bd909ac",813:"6875c492",882:"9335a94c",894:"b2f554cd",969:"14eb3368",972:"73664a40",991:"b2b675dd"}[e]||e)+"."+{48:"fb219bf3",61:"d2ed0784",98:"d0cef725",134:"8c62586e",209:"d1340b2b",237:"2d4647a1",249:"3c8d4bec",267:"0a916e73",307:"0a77a325",320:"dd4e595b",369:"14310c2e",374:"358b4fa4",401:"2729c66b",413:"adfff614",472:"72b158f8",497:"75696e1a",533:"ff8adb39",561:"a4d1327e",581:"17363a8a",607:"996c09e7",634:"b6ce02cd",643:"a6cc5230",647:"d676a9b1",669:"ce893299",694:"517ed12c",711:"7c7bc502",721:"7b3cf35b",722:"b2773118",727:"cdc01990",747:"704eae12",806:"a9e1fa1d",813:"5d8b32bc",882:"dde883a8",894:"47a4c664",969:"5772ec2b",972:"c615bff9",991:"f40c065b"}[e]+".js",f.miniCssF=e=>{},f.g=function(){if("object"==typeof globalThis)return globalThis;try{return this||new Function("return this")()}catch(e){if("object"==typeof window)return window}}(),f.o=(e,a)=>Object.prototype.hasOwnProperty.call(e,a),r={},c="stable-code:",f.l=(e,a,t,o)=>{if(r[e])r[e].push(a);else{var d,n;if(void 0!==t)for(var b=document.getElementsByTagName("script"),i=0;i{d.onerror=d.onload=null,clearTimeout(s);var c=r[e];if(delete r[e],d.parentNode&&d.parentNode.removeChild(d),c&&c.forEach((e=>e(t))),a)return a(t)},s=setTimeout(u.bind(null,void 0,{type:"timeout",target:d}),12e4);d.onerror=u.bind(null,d.onerror),d.onload=u.bind(null,d.onload),n&&document.head.appendChild(d)}},f.r=e=>{"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})},f.p="/",f.gca=function(e){return e={17896441:"401",66406991:"374",a94703ab:"48","1f391b9e":"61",a7bd4aaa:"98","393be207":"134","01a85c17":"209",ccc49370:"249",a7023ddc:"267","962d33de":"307","9d646488":"320","9e421cea":"369","1db64337":"413","814f3328":"472",a80da1cf:"497","546e7911":"561","935f2afb":"581","62fdb048":"607",c4f5d8e4:"634",a6aa9e1f:"643","5e95c892":"647","30a24c52":"669","8717b14a":"694","9e4087bc":"711","46e868e2":"721","608ae6a4":"722",ff0a8395:"727","1bd909ac":"806","6875c492":"813","9335a94c":"882",b2f554cd:"894","14eb3368":"969","73664a40":"972",b2b675dd:"991"}[e]||e,f.p+f.u(e)},(()=>{var e={354:0,869:0};f.f.j=(a,t)=>{var r=f.o(e,a)?e[a]:void 0;if(0!==r)if(r)t.push(r[2]);else if(/^(354|869)$/.test(a))e[a]=0;else{var c=new Promise(((t,c)=>r=e[a]=[t,c]));t.push(r[2]=c);var o=f.p+f.u(a),d=new Error;f.l(o,(t=>{if(f.o(e,a)&&(0!==(r=e[a])&&(e[a]=void 0),r)){var c=t&&("load"===t.type?"missing":t.type),o=t&&t.target&&t.target.src;d.message="Loading chunk "+a+" failed.\n("+c+": "+o+")",d.name="ChunkLoadError",d.type=c,d.request=o,r[1](d)}}),"chunk-"+a,a)}},f.O.j=a=>0===e[a];var a=(a,t)=>{var r,c,o=t[0],d=t[1],n=t[2],b=0;if(o.some((a=>0!==e[a]))){for(r in d)f.o(d,r)&&(f.m[r]=d[r]);if(n)var i=n(f)}for(a&&a(t);b{"use strict";var e,a,t,r,c,o={},d={};function f(e){var a=d[e];if(void 0!==a)return a.exports;var t=d[e]={id:e,loaded:!1,exports:{}};return o[e].call(t.exports,t,t.exports,f),t.loaded=!0,t.exports}f.m=o,f.c=d,e=[],f.O=(a,t,r,c)=>{if(!t){var o=1/0;for(i=0;i=c)&&Object.keys(f.O).every((e=>f.O[e](t[n])))?t.splice(n--,1):(d=!1,c0&&e[i-1][2]>c;i--)e[i]=e[i-1];e[i]=[t,r,c]},f.n=e=>{var a=e&&e.__esModule?()=>e.default:()=>e;return f.d(a,{a:a}),a},t=Object.getPrototypeOf?e=>Object.getPrototypeOf(e):e=>e.__proto__,f.t=function(e,r){if(1&r&&(e=this(e)),8&r)return e;if("object"==typeof e&&e){if(4&r&&e.__esModule)return e;if(16&r&&"function"==typeof e.then)return e}var c=Object.create(null);f.r(c);var o={};a=a||[null,t({}),t([]),t(t)];for(var d=2&r&&e;"object"==typeof d&&!~a.indexOf(d);d=t(d))Object.getOwnPropertyNames(d).forEach((a=>o[a]=()=>e[a]));return o.default=()=>e,f.d(c,o),c},f.d=(e,a)=>{for(var t in a)f.o(a,t)&&!f.o(e,t)&&Object.defineProperty(e,t,{enumerable:!0,get:a[t]})},f.f={},f.e=e=>Promise.all(Object.keys(f.f).reduce(((a,t)=>(f.f[t](e,a),a)),[])),f.u=e=>"assets/js/"+({48:"a94703ab",61:"1f391b9e",98:"a7bd4aaa",134:"393be207",209:"01a85c17",249:"ccc49370",267:"a7023ddc",307:"962d33de",320:"9d646488",369:"9e421cea",374:"66406991",401:"17896441",413:"1db64337",472:"814f3328",497:"a80da1cf",561:"546e7911",581:"935f2afb",607:"62fdb048",634:"c4f5d8e4",643:"a6aa9e1f",647:"5e95c892",669:"30a24c52",694:"8717b14a",711:"9e4087bc",721:"46e868e2",722:"608ae6a4",727:"ff0a8395",806:"1bd909ac",813:"6875c492",882:"9335a94c",894:"b2f554cd",969:"14eb3368",972:"73664a40",991:"b2b675dd"}[e]||e)+"."+{48:"fb219bf3",61:"d2ed0784",98:"d0cef725",134:"8c62586e",209:"d1340b2b",237:"2d4647a1",249:"3c8d4bec",267:"0a916e73",307:"0a77a325",320:"dd4e595b",369:"14310c2e",374:"358b4fa4",401:"2729c66b",413:"adfff614",472:"72b158f8",497:"75696e1a",533:"ff8adb39",561:"b94e85be",581:"7438d976",607:"996c09e7",634:"b6ce02cd",643:"a6cc5230",647:"d676a9b1",669:"ce893299",694:"517ed12c",711:"7c7bc502",721:"7b3cf35b",722:"b2773118",727:"cdc01990",747:"704eae12",806:"a9e1fa1d",813:"5d8b32bc",882:"dde883a8",894:"47a4c664",969:"5772ec2b",972:"c615bff9",991:"f40c065b"}[e]+".js",f.miniCssF=e=>{},f.g=function(){if("object"==typeof globalThis)return globalThis;try{return this||new Function("return this")()}catch(e){if("object"==typeof window)return window}}(),f.o=(e,a)=>Object.prototype.hasOwnProperty.call(e,a),r={},c="stable-code:",f.l=(e,a,t,o)=>{if(r[e])r[e].push(a);else{var d,n;if(void 0!==t)for(var b=document.getElementsByTagName("script"),i=0;i{d.onerror=d.onload=null,clearTimeout(s);var c=r[e];if(delete r[e],d.parentNode&&d.parentNode.removeChild(d),c&&c.forEach((e=>e(t))),a)return a(t)},s=setTimeout(u.bind(null,void 0,{type:"timeout",target:d}),12e4);d.onerror=u.bind(null,d.onerror),d.onload=u.bind(null,d.onload),n&&document.head.appendChild(d)}},f.r=e=>{"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})},f.p="/",f.gca=function(e){return e={17896441:"401",66406991:"374",a94703ab:"48","1f391b9e":"61",a7bd4aaa:"98","393be207":"134","01a85c17":"209",ccc49370:"249",a7023ddc:"267","962d33de":"307","9d646488":"320","9e421cea":"369","1db64337":"413","814f3328":"472",a80da1cf:"497","546e7911":"561","935f2afb":"581","62fdb048":"607",c4f5d8e4:"634",a6aa9e1f:"643","5e95c892":"647","30a24c52":"669","8717b14a":"694","9e4087bc":"711","46e868e2":"721","608ae6a4":"722",ff0a8395:"727","1bd909ac":"806","6875c492":"813","9335a94c":"882",b2f554cd:"894","14eb3368":"969","73664a40":"972",b2b675dd:"991"}[e]||e,f.p+f.u(e)},(()=>{var e={354:0,869:0};f.f.j=(a,t)=>{var r=f.o(e,a)?e[a]:void 0;if(0!==r)if(r)t.push(r[2]);else if(/^(354|869)$/.test(a))e[a]=0;else{var c=new Promise(((t,c)=>r=e[a]=[t,c]));t.push(r[2]=c);var o=f.p+f.u(a),d=new Error;f.l(o,(t=>{if(f.o(e,a)&&(0!==(r=e[a])&&(e[a]=void 0),r)){var c=t&&("load"===t.type?"missing":t.type),o=t&&t.target&&t.target.src;d.message="Loading chunk "+a+" failed.\n("+c+": "+o+")",d.name="ChunkLoadError",d.type=c,d.request=o,r[1](d)}}),"chunk-"+a,a)}},f.O.j=a=>0===e[a];var a=(a,t)=>{var r,c,o=t[0],d=t[1],n=t[2],b=0;if(o.some((a=>0!==e[a]))){for(r in d)f.o(d,r)&&(f.m[r]=d[r]);if(n)var i=n(f)}for(a&&a(t);b Archive | Stable Code Docs - + diff --git a/blog/index.html b/blog/index.html index e184e22..3791851 100644 --- a/blog/index.html +++ b/blog/index.html @@ -5,7 +5,7 @@ Blog | Stable Code Docs - + diff --git a/blog/long-blog-post/index.html b/blog/long-blog-post/index.html index 8c33d0b..279e914 100644 --- a/blog/long-blog-post/index.html +++ b/blog/long-blog-post/index.html @@ -5,7 +5,7 @@ Long Blog Post | Stable Code Docs - + diff --git a/blog/tags/docusaurus/index.html b/blog/tags/docusaurus/index.html index aef40fb..1e3e303 100644 --- a/blog/tags/docusaurus/index.html +++ b/blog/tags/docusaurus/index.html @@ -5,7 +5,7 @@ One post tagged with "docusaurus" | Stable Code Docs - + diff --git a/blog/tags/hello/index.html b/blog/tags/hello/index.html index 3d519f1..2c2479f 100644 --- a/blog/tags/hello/index.html +++ b/blog/tags/hello/index.html @@ -5,7 +5,7 @@ One post tagged with "hello" | Stable Code Docs - + diff --git a/blog/tags/index.html b/blog/tags/index.html index ce75266..333fe4e 100644 --- a/blog/tags/index.html +++ b/blog/tags/index.html @@ -5,7 +5,7 @@ Tags | Stable Code Docs - + diff --git a/docs/category/tutorial---finetuning-stable-code/index.html b/docs/category/tutorial---finetuning-stable-code/index.html index e78631b..d4732f4 100644 --- a/docs/category/tutorial---finetuning-stable-code/index.html +++ b/docs/category/tutorial---finetuning-stable-code/index.html @@ -5,10 +5,10 @@ Tutorial - Finetuning Stable Code | Stable Code Docs - + -

Tutorial - Finetuning Stable Code

+

Tutorial - Finetuning Stable Code

\ No newline at end of file diff --git a/docs/category/tutorial---using-stable-code/index.html b/docs/category/tutorial---using-stable-code/index.html index e9b8911..a6686bc 100644 --- a/docs/category/tutorial---using-stable-code/index.html +++ b/docs/category/tutorial---using-stable-code/index.html @@ -5,7 +5,7 @@ Tutorial - Using Stable Code | Stable Code Docs - + diff --git a/docs/overview/index.html b/docs/overview/index.html index 57e821e..b71a3b5 100644 --- a/docs/overview/index.html +++ b/docs/overview/index.html @@ -5,7 +5,7 @@ Stable Code Overview | Stable Code Docs - + diff --git a/docs/tutorial-finetuning/stable_code_axolotl/index.html b/docs/tutorial-finetuning/stable_code_axolotl/index.html index efa5a7e..993d5a1 100644 --- a/docs/tutorial-finetuning/stable_code_axolotl/index.html +++ b/docs/tutorial-finetuning/stable_code_axolotl/index.html @@ -3,16 +3,37 @@ -Stable Code Finetuning with Axolotl | Stable Code Docs +Stable Code Finetuning with Axolotl | Stable Code Docs - + -

Stable Code Finetuning with Axolotl

import torch
# Check so there is a gpu available, a T4(free tier) is enough to run this notebook
assert (torch.cuda.is_available()==True)
+

Stable Code Finetuning with Axolotl

+

Axolotl is a popular library for finetuning models. In this tutorial, we +will use Axolotl to finetune our Stable Code 3B model on FORTRAN code to +show you how to create a custom code completion model on a new +programming language.

+

This tutorial is based on the wonderful notebooks from Maxime Labonne’s +llm course

+
Open In Colab
Open In Colab
+
import torch
# Check so there is a gpu available, a T4(free tier) is enough to run this notebook
assert (torch.cuda.is_available()==True)
+

Let’s get the environment set up first. We will go ahead and install the +necessary packages.

!pip install torch=="2.1.2"
!pip install -e git+https://github.com/OpenAccess-AI-Collective/axolotl#egg=axolotl
!pip install flash-attn=="2.5.0"
!pip install deepspeed=="0.13.1"
-
import yaml

# Your YAML string
yaml_string = """
base_model: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
model_type: LlamaForCausalLM
tokenizer_type: LlamaTokenizer
is_llama_derived_model: true

load_in_8bit: false
load_in_4bit: true
strict: false

datasets:
  - path: mhenrichsen/alpaca_2k_test
    type: alpaca
dataset_prepared_path:
val_set_size: 0.05
output_dir: ./qlora-out

adapter: qlora
lora_model_dir:

sequence_len: 1096
sample_packing: true
pad_to_sequence_len: true

lora_r: 32
lora_alpha: 16
lora_dropout: 0.05
lora_target_modules:
lora_target_linear: true
lora_fan_in_fan_out:

wandb_project:
wandb_entity:
wandb_watch:
wandb_name:
wandb_log_model:

mlflow_experiment_name: colab-example

gradient_accumulation_steps: 1
micro_batch_size: 1
num_epochs: 4
max_steps: 20
optimizer: paged_adamw_32bit
lr_scheduler: cosine
learning_rate: 0.0002

train_on_inputs: false
group_by_length: false
bf16: false
fp16: true
tf32: false

gradient_checkpointing: true
early_stopping_patience:
resume_from_checkpoint:
local_rank:
logging_steps: 1
xformers_attention:
flash_attention: false

warmup_steps: 10
evals_per_epoch:
saves_per_epoch:
debug:
deepspeed:
weight_decay: 0.0
fsdp:
fsdp_config:
special_tokens:

"""

# Convert the YAML string to a Python dictionary
yaml_dict = yaml.safe_load(yaml_string)

# Specify your file path
file_path = 'test_axolotl.yaml'

# Write the YAML file
with open(file_path, 'w') as file:
    yaml.dump(yaml_dict, file)
+

Axolotl relies on a config file written in YAML to specify the model and +the dataset and how to train the model. We will create a config file for +our finetuning task to finetune the Stable Code 3B model on FORTRAN code +using QLoRA. The dataset we are picking is from +codeparrot/github-code-clean dataset which is a collection of code +scraped from GitHub. Specifically, we will be utilizing the FORTRAN MIT +licensed code.

+
import yaml

# Your YAML string
yaml_string = """
base_model: stabilityai/stable-code-3b
model_type: AutoModelForCausalLM
tokenizer_type: AutoTokenizer

load_in_8bit: false
load_in_4bit: true
strict: false

datasets:
  - path: codeparrot/github-code-clean
    name: FORTRAN-mit
    type: completion
    field: code
dataset_prepared_path:
val_set_size: 0.05
output_dir: ./qlora-out

adapter: qlora
lora_model_dir:

sequence_len: 1096
sample_packing: true
pad_to_sequence_len: true

lora_r: 32
lora_alpha: 16
lora_dropout: 0.05
lora_target_modules:
lora_target_linear: true
lora_fan_in_fan_out:

wandb_project:
wandb_entity:
wandb_watch:
wandb_name:
wandb_log_model:

mlflow_experiment_name: colab-example

gradient_accumulation_steps: 1
micro_batch_size: 1
num_epochs: 4
max_steps: 20
optimizer: paged_adamw_32bit
lr_scheduler: cosine
learning_rate: 0.0002

train_on_inputs: false
group_by_length: false
bf16: false
fp16: true
tf32: false

gradient_checkpointing: true
early_stopping_patience:
resume_from_checkpoint:
local_rank:
logging_steps: 1
xformers_attention:
flash_attention: false

warmup_steps: 10
evals_per_epoch:
saves_per_epoch:
debug:
deepspeed:
weight_decay: 0.0
fsdp:
fsdp_config:
special_tokens:

"""

# Convert the YAML string to a Python dictionary
yaml_dict = yaml.safe_load(yaml_string)

# Specify your file path
file_path = 'test_axolotl.yaml'

# Write the YAML file
with open(file_path, 'w') as file:
    yaml.dump(yaml_dict, file)
+

Next, we can launch the run, which should take about XX if you are +running this in a Colab environment with a T4 GPU:

# Buy using the ! the comand will be executed as a bash command
!accelerate launch -m axolotl.cli.train /content/test_axolotl.yaml
+

Once the model has been finetuned, we can run it in a gradio +environment:

# Buy using the ! the comand will be executed as a bash command
!accelerate launch -m axolotl.cli.inference /content/test_axolotl.yaml \
    --qlora_model_dir="./qlora-out" --gradio
\ No newline at end of file diff --git a/docs/tutorial-usage/ollama/index.html b/docs/tutorial-usage/ollama/index.html index 232925b..c3abcfa 100644 --- a/docs/tutorial-usage/ollama/index.html +++ b/docs/tutorial-usage/ollama/index.html @@ -5,7 +5,7 @@ Stable Code on Ollama | Stable Code Docs - + diff --git a/docs/tutorial-usage/repository_qa/index.html b/docs/tutorial-usage/repository_qa/index.html index 54cf29d..fbfe94c 100644 --- a/docs/tutorial-usage/repository_qa/index.html +++ b/docs/tutorial-usage/repository_qa/index.html @@ -5,7 +5,7 @@ Repositories QA | Stable Code Docs - + diff --git a/docs/tutorial-usage/vscode-extension/index.html b/docs/tutorial-usage/vscode-extension/index.html index c6ffcc1..4d1fdc8 100644 --- a/docs/tutorial-usage/vscode-extension/index.html +++ b/docs/tutorial-usage/vscode-extension/index.html @@ -5,7 +5,7 @@ VSCode Extension | Stable Code Docs - + diff --git a/index.html b/index.html index a64fe84..b7eb9cd 100644 --- a/index.html +++ b/index.html @@ -5,7 +5,7 @@ Hello from Stable Code Docs | Stable Code Docs - + diff --git a/markdown-page/index.html b/markdown-page/index.html index fc5700f..9d5c956 100644 --- a/markdown-page/index.html +++ b/markdown-page/index.html @@ -5,7 +5,7 @@ Markdown page example | Stable Code Docs - +