diff --git a/README.md b/README.md index 3b8f715..a72864d 100644 --- a/README.md +++ b/README.md @@ -379,6 +379,21 @@ python -W ignore main.py \ -n 10 ``` +### Bedrock + +```bash +python3 main.py \ + -db postgres \ + -q data/instruct_basic_postgres.csv data/instruct_advanced_postgres.csv data/questions_gen_postgres.csv \ + -o results/bedrock_llama_70b_basic.csv results/bedrock_llama_70b_advanced.csv results/bedrock_llama_70b_v1.csv \ + -g bedrock \ + -f prompts/prompt_cot_postgres.md \ + --cot_table_alias prealias \ + -m meta.llama3-70b-instruct-v1:0 \ + -c 0 \ + -p 10 +``` + ### Together Before running this, you must create an account with [Together.ai](https://together.ai/) and obtain an API key and store it with `export TOGETHER_API_KEY=`. Then, install `together` with `pip install together`. You can then run the following command: diff --git a/eval/together_runner.py b/eval/together_runner.py index 3782c69..0414e57 100644 --- a/eval/together_runner.py +++ b/eval/together_runner.py @@ -13,15 +13,17 @@ from utils.reporting import upload_results -client = Together(api_key=os.environ.get('TOGETHER_API_KEY')) +client = Together(api_key=os.environ.get("TOGETHER_API_KEY")) def process_row(row: Dict, model: str): start_time = time() if model.startswith("meta-llama"): - stop = ["<|eot_id|>","<|eom_id|>"] + stop = ["<|eot_id|>", "<|eom_id|>"] else: - print("Undefined stop token(s). Please specify the stop token(s) for the model.") + print( + "Undefined stop token(s). Please specify the stop token(s) for the model." + ) stop = [] messages = row["prompt"] response = client.chat.completions.create( @@ -30,13 +32,12 @@ def process_row(row: Dict, model: str): max_tokens=800, temperature=0.0, stop=stop, - stream=False + stream=False, ) content = response.choices[0].message.content generated_query = content.split("```", 1)[0].strip() end_time = time() - row["generated_query"] = generated_query row["latency_seconds"] = end_time - start_time row["tokens_used"] = None @@ -132,9 +133,7 @@ def run_together_eval(args): with ThreadPoolExecutor(max_workers=max_workers) as executor: futures = [] for row in df.to_dict("records"): - futures.append( - executor.submit(process_row, row, model) - ) + futures.append(executor.submit(process_row, row, model)) with tqdm(as_completed(futures), total=len(futures)) as pbar: for f in pbar: diff --git a/utils/gen_prompt.py b/utils/gen_prompt.py index 8079f4f..a2cd02e 100644 --- a/utils/gen_prompt.py +++ b/utils/gen_prompt.py @@ -131,6 +131,7 @@ def generate_prompt( Else, we will treat the file as a string template. """ from defog_data.metadata import dbs # to avoid CI error + is_json = prompt_file.endswith(".json") if is_json: with open(prompt_file, "r") as f: