diff --git a/chroma_db/12ba6ac1-25a3-4a76-a56e-25054de357b4/data_level0.bin b/chroma_db/12ba6ac1-25a3-4a76-a56e-25054de357b4/data_level0.bin deleted file mode 100644 index ea3192e..0000000 Binary files a/chroma_db/12ba6ac1-25a3-4a76-a56e-25054de357b4/data_level0.bin and /dev/null differ diff --git a/chroma_db/12ba6ac1-25a3-4a76-a56e-25054de357b4/header.bin b/chroma_db/12ba6ac1-25a3-4a76-a56e-25054de357b4/header.bin deleted file mode 100644 index 3e0932a..0000000 Binary files a/chroma_db/12ba6ac1-25a3-4a76-a56e-25054de357b4/header.bin and /dev/null differ diff --git a/chroma_db/12ba6ac1-25a3-4a76-a56e-25054de357b4/length.bin b/chroma_db/12ba6ac1-25a3-4a76-a56e-25054de357b4/length.bin deleted file mode 100644 index fe923b7..0000000 Binary files a/chroma_db/12ba6ac1-25a3-4a76-a56e-25054de357b4/length.bin and /dev/null differ diff --git a/chroma_db/12ba6ac1-25a3-4a76-a56e-25054de357b4/link_lists.bin b/chroma_db/12ba6ac1-25a3-4a76-a56e-25054de357b4/link_lists.bin deleted file mode 100644 index e69de29..0000000 diff --git a/chroma_db/chroma.sqlite3 b/chroma_db/chroma.sqlite3 deleted file mode 100644 index 8c1bfd8..0000000 Binary files a/chroma_db/chroma.sqlite3 and /dev/null differ diff --git a/git b/git deleted file mode 100644 index e69de29..0000000 diff --git a/src/backend/RAG/Astra_db/Astra_DB connection with LLM models.py b/src/backend/RAG/Astra_db/Astra_DB connection with LLM models.py deleted file mode 100644 index 86f870f..0000000 --- a/src/backend/RAG/Astra_db/Astra_DB connection with LLM models.py +++ /dev/null @@ -1,120 +0,0 @@ -import os - -from astrapy import DataAPIClient -from astrapy.db import AstraDB -from dotenv import load_dotenv -from langchain.schema import Document -from langchain_google_genai import ChatGoogleGenerativeAI -from langchain_openai import OpenAIEmbeddings - -load_dotenv() - -# Initialize AstraDB connection -ASTRA_DB_NAMESPACE = 'test' -ASTRA_DB_COLLECTION = 'test_collection_2' - -client = DataAPIClient(os.getenv('ASTRA_DB_TOKEN')) -database = client.get_database(os.getenv('ASTRA_DB_API_ENDPOINT')) - -db = AstraDB( - token=os.getenv('ASTRA_DB_TOKEN'), - api_endpoint=os.getenv('ASTRA_DB_API_ENDPOINT'), - namespace=os.getenv('ASTRA_DB_NAMESPACE'), -) -collection = db.collection(collection_name='test_collection_2') - -# Initializing OpenAI and Google Generative AI models -openai_api_key = os.environ.get('OPEN_API_KEY') -embedding_model = OpenAIEmbeddings(api_key=openai_api_key) -llm = ChatGoogleGenerativeAI(model='gemini-pro', google_api_key=os.environ.get('GOOGLE_GEMINI_API')) - -# similarity search in AstraDB -query = 'Which foods indicate protective or neutral effects' -query_embedding = embedding_model.embed_query(query) - -sort = {'$vector': query_embedding} -options = {'limit': 2} -projection = {'content': 1, 'metadata': 1, '$vector': 1} - -document_list = collection.find(sort=sort, options=options, projection=projection) - -# Prepare documents for LangChain processing -document_chunks = [] -for document in document_list['data']['documents']: - content = document.get('content', '') - metadata = document.get('metadata', {}) - document_chunks.append(Document(page_content=content, metadata=metadata)) - -# Process documents with LangChain and generate response -response = '' -if document_chunks: - relevant_info = '' - for doc in document_chunks: - relevant_info += f'Content: {doc.page_content}\nMetadata: {doc.metadata}\n\n' - print(f'Content: {doc.page_content}\nMetadata: {doc.metadata}\n\n') - context = ( - 'You are a health consultant specializing in answering' - 'health-related queries and providing comprehensive insights. ' - 'You have access to various documents containing' - 'information about the latest trends in the health industry, ' - 'summarize the key points, and draw conclusions' - 'based on the provided information. Consider the broader implications ' - 'while forming your conclusions.' - ) - - multi_step_reasoning = ( - '1. Read the provided statements carefully.\n' - '2. Identify and list the key points mentioned' - 'by the person asking the question.\n' - '3. Analyze these key points in the context' - '4. of the available information.\n' - '5. Draw well-reasoned conclusions based on your analysis.\n' - '6. Provide a summary of your findings,' - '7. highlighting the most critical insights.' - ) - - few_shot_examples = ( - 'Example 1:\n' - 'Q: What did the CEO of TechCorp say about their new product launch?\n' - 'A: The CEO emphasized the innovative features of the new product,' - 'its competitive pricing, and its potential to capture significant market share. ' - 'They also mentioned the strategic importance of this' - "launch for the company's growth plans.\n\n" - 'Example 2:\n' - "Q: What were the key takeaways from the CFO's statement at HealthInc?\n" - "A: The CFO highlighted the company's strong " - 'financial performance, cost-cutting measures, and plans' - 'for future investments in technology ' - 'to drive efficiency. They also discussed the ' - 'challenges posed by recent market volatility' - 'and their strategies to mitigate risks.' - ) - - hypothetical_scenario = ( - 'Imagine you are presenting your analysis to' - 'a group of doctors in a conference. ' - 'Your goal is to provide a clear, concise,' - 'and insightful analysis that helps them' - 'understand the latest trends in the health industry and ' - 'their potential impact on medical practices.' - ) - - modified_prompt = ( - f'{context}\n\n' - f'{multi_step_reasoning}\n\n' - f'{few_shot_examples}\n\n' - f'{hypothetical_scenario}\n\n' - f'Here is some relevant information:\n{relevant_info}\n\n' - f'Now, based on the provided information and your expertise' - ', answer the following questions:\n' - f'Q: What are the protective effects mentioned?\n' - f'Q: What can we conclude from this information?' - ) - - response = llm.predict(modified_prompt) - print(f'Response generated from {llm.__class__.__name__} is:', response) -else: - print('No documents found matching the query.') - -# Print the response -print(response) diff --git a/src/backend/RAG/FlaskServer/__init__.py b/src/backend/RAG/FlaskServer/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/backend/RAG/FlaskServer/server.py b/src/backend/RAG/FlaskServer/server.py deleted file mode 100644 index de50675..0000000 --- a/src/backend/RAG/FlaskServer/server.py +++ /dev/null @@ -1,36 +0,0 @@ -from threading import Thread - -from flask import Flask, jsonify, request -from kivy.app import App -from kivy.uix.button import Button - -from src.backend.RAG.LangChain_Implementation.retriever import Retriever - -flask_app = Flask(__name__) -retriever = Retriever() - - -@flask_app.route('/llm-response', methods=['POST']) -def get_llm_response(): - data = request.json - query = data.get('query') - if not query: - return jsonify({'error': 'No query provided'}), 400 - - response = retriever.simple_response(query) - return jsonify({'response': response}) - - -def run_flask(): - flask_app.run(host='localhost', port=5000) - - -class MyApp(App): - def build(self): - # Start Flask server in a separate thread - Thread(target=run_flask).start() - return Button(text='LLM Server Running') - - -if __name__ == '__main__': - MyApp().run() diff --git a/src/backend/RAG/LangChain_Implementation/basic_retrieval.py b/src/backend/RAG/LangChain_Implementation/basic_retrieval.py deleted file mode 100644 index cad0e0d..0000000 --- a/src/backend/RAG/LangChain_Implementation/basic_retrieval.py +++ /dev/null @@ -1,117 +0,0 @@ -import json -import os - -from langchain.embeddings import OpenAIEmbeddings -from langchain.schema import Document -from langchain.text_splitter import RecursiveCharacterTextSplitter -from langchain_community.embeddings import HuggingFaceEmbeddings -from langchain_community.llms.openai import OpenAI -from langchain_community.vectorstores.chroma import Chroma - - -class CustomJSONLoader: - def __init__( - self, file_path, jq_schema='.', json_lines=True, text_content=False, metadata_keys=None - ): - self.file_path = file_path - self.jq_schema = jq_schema - self.json_lines = json_lines - self.text_content = text_content - self.metadata_keys = metadata_keys if metadata_keys is not None else [] - - def load(self): - docs = [] - with open(self.file_path, 'r', encoding='utf-8') as file: - if self.json_lines: - for line in file: - stripped_line = line.strip() - if stripped_line: - try: - doc = json.loads(stripped_line) - docs.append(doc) - except json.JSONDecodeError: - continue - else: - docs = json.load(file) - return docs - - def extract_chunks_with_metadata(self, docs): - chunks_with_metadata = [] - for doc in docs: - if isinstance(doc, dict): - self._extract_from_dict(doc, chunks_with_metadata) - return chunks_with_metadata - - def _extract_from_dict(self, doc_dict, chunks_with_metadata): - if 'content' in doc_dict: - content = doc_dict['content'] - metadata = {key: doc_dict[key] for key in self.metadata_keys if key in doc_dict} - if isinstance(content, list): - content = ' '.join(content) - chunks_with_metadata.append({'content': content, 'metadata': metadata}) - - -base_path = r'C:\Users\manik\Desktop\Projects\AMOS Project\amos2024ss06-health-ai-framework' -sub_path = r'src\backend\RAG\LangChain_Implementation\blog_data.json' -file_path = os.path.join(base_path, sub_path) - -loader = CustomJSONLoader( - file_path=file_path, - jq_schema='.', - json_lines=False, - text_content='content', - metadata_keys=['author', 'title', 'url'], -) - -docs = loader.load() -chunks_with_metadata = loader.extract_chunks_with_metadata(docs) - -text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200) - -document_chunks = [] -for chunk_with_metadata in chunks_with_metadata: - content = chunk_with_metadata['content'] - metadata = chunk_with_metadata['metadata'] - - if isinstance(content, str): - split_chunks = text_splitter.split_text(content) - for chunk in split_chunks: - document_chunks.append(Document(page_content=chunk, metadata=metadata)) - -embedding_model = HuggingFaceEmbeddings(model_name='all-MiniLM-L6-v2') -embedding_model = OpenAIEmbeddings() - -embeddings = [embedding_model.embed_query(chunk) for chunk in split_chunks] - -chroma_db = Chroma(persist_directory='./chroma_db', embedding_function=OpenAIEmbeddings()) - -all_doc_ids = chroma_db.from_documents(documents=document_chunks, embedding=OpenAIEmbeddings()) -chroma_db.add_documents(documents=document_chunks) - -# ridiculous linting rules - thanks: -query_text1 = 'Can Blueberries Help with Diabetes and Repairing DNA?' -query_text2 = 'What can we conclude from this response?' -query_text = query_text1 + query_text2 - -query_embedding = embedding_model.embed_query(query_text) - -results = chroma_db._similarity_search_with_relevance_scores(query_text, k=2) - -relevant_info = '' -for result, score in results: - doc_content = result.page_content - metadata = result.metadata - metadata_str = ', '.join([f'{key}: {value}' for key, value in metadata.items()]) - relevant_info += f'Content: {doc_content}\nMetadata: {metadata_str}\n\n' - -print('Relevant information is:', relevant_info) - -modified_prompt = {'text': f'{query_text}\n\nHere is some relevant information:\n{relevant_info}'} - -prompt_text = modified_prompt['text'] - -llm = OpenAI(temperature=0.5) - -response = llm.predict(prompt_text) - -print('Response generated from LLM IS:', response) diff --git a/src/backend/RAG/LangChain_Implementation/blog_data.json b/src/backend/RAG/LangChain_Implementation/blog_data.json deleted file mode 100644 index e127c47..0000000 --- a/src/backend/RAG/LangChain_Implementation/blog_data.json +++ /dev/null @@ -1,500 +0,0 @@ -[ - { - "title": "Putting the Benefits of Fasting for Weight Loss to the Test", - "date": "2024-01-18T13:00:55+00:00", - "author": "Michael Greger M.D. FACLM", - "content": [ - "\n For more than a century, fasting has been used as a weight-loss treatment. \n I", - " ve talked about the benefits of caloric restriction. Well, the greatest calor", - "ic restriction is getting no calories at all. Fasting has been branded the ne", - "xt big weight loss fad, but it has a long history throughout various spiritual ", - "traditions, practiced by Moses, Jesus, Muhammed, and Buddha. In 1732, a noted ", - "physician wrote , He that eats till he is sick must fast till he is well. Abo", - "ut one in seven American adults today report taking that advice, using fastin", - "g as a means to control body weight, as I discuss in my video Benefits of Fast", - "ing for Weight Loss Put to the Test . \n \n Case reports of the treatment of obesi", - "ty through fasting date back more than a century in the medical literature. In", - " 1915, two Harvard doctors indelicately described two extraordinarily fat women", - ", one of whom was a veritable pork barrel. Their success led them to conclude", - " that successive moderate periods of starvation constitute a perfectly safe, ha", - "rmless, and effective method for reducing the weight of those suffering from obe", - "sity. \n \n The longest-recorded fast, published in 1973, made it into the Guinn", - "ess Book of World Records . To reach his ideal body weight, a 27-year-old man f", - "asted for 382 days straight, losing 276 pounds, and managed to keep nearly all ", - "of it off. He was given vitamin and mineral supplements so he wouldn t die, but ", - "no calories for more than a year. In the researchers acknowledgments, they than", - "ked him for his cheerful co-operation and steadfast application to the task of ", - "achieving a normal physique. \n \n In a U.S. Air Force study, more than 20 indivi", - "duals at least 100 pounds overweight and most unable to lose weight on previo", - "us diets were fasted for as long as 84 days. Nine dropped out of the study, but", - " the 16 who remained were unequivocally successful at losing 40 to 100 pounds.", - " In the first four days, the subjects were noted as losing as much as four pound", - "s a day, which probably represents mostly fluid, mostly water weight as the bo", - "dy starts to adapt. But, after a few weeks, they were steadily losing about a po", - "und a day of mostly straight fat. The investigator described the starvation prog", - "ram as a dramatic and exciting treatment for obesity. \n \n Of course, the singl", - "e most successful diet for weight loss namely no diet at all is also the single ", - "least sustainable. What other diet can cure morbid obesity in a matter of months", - " but practically be guaranteed to kill you within a year if you stick with it? T", - "he reason diets don t work, almost by definition, is that people go on them, the", - "n they go off of them. Permanent weight loss is only achieved through permanent ", - "lifestyle change. So, what s the point of fasting if you re just going to go bac", - "k to your regular diet and gain right back all of that lost weight? \n \n Fasting ", - "proponents cite the psychological benefit of realigning people s perceptions a", - "nd motivation. Some individuals have resigned themselves to the belief that weig", - "ht loss for them is somehow impossible. They may think that they are made di", - "fferently from those of normal weight in some way, and no matter what they do,", - " the pounds don t come off. But the rapid, unequivocal weight loss during fastin", - "g demonstrates to them that with a large enough change in eating habits, it s no", - "t just possible, but inevitable. This morale boost may then embolden them to m", - "ake better food choices once they resume eating. \n \n The break from food may al", - "low some an opportunity to pause and reflect on the role food is playing in t", - "heir lives not only the power it has over them but the power they have over it. ", - "In a fasting study entitled Correction and Control of Intractable Obesity, a", - " patient s personality was described as changing from one of desperation, with ", - "abandonment of hope, to that of an eager extravert full of plans for a promising", - " future. She realized that her weight was within her own power to control. The ", - "researchers concluded: This highly intellectual social worker has been returned", - " to a full degree of exceptional usefulness. \n \n After a fast, newfound commitm", - "ent to more healthful eating may be facilitated by a reduction in overall appe", - "tite reported post-fast, compared to pre-fast, at least temporarily. Even during", - " a fast, hunger may start to dissipate within the first 36 hours. So, challen", - "ging people s delusions about their exceptionality to the laws of physics think", - "ing they are made differently with short periods of total fasting may seem bar", - "baric. In reality, this method of weight reduction is remarkably well tolerated ", - "by obese patients. That seems to be a recurring theme in these published series", - " of cases. In the influential paper Treatment of Obesity by Total Fasting for u", - "p to 249 Days, the researchers remarked that the most surprising aspect of t", - "his study was the ease with which the prolonged fast was tolerated. All of thei", - "r patients spontaneously commented on their increased sense of well-being, and ", - "in some, this amounted to frank euphoria. They continued that, although treatm", - "ent by total fasting must only be prescribed under close medical supervision, t", - "hey are convinced that it is the treatment of choice, certainly in cases of gro", - "ss obesity. \n \n Fasting for a day can make people irritable and feel moody ", - "and distracted, but after a few days of fasting, many report feeling clear, el", - "ated, and alert even euphoric. This may be in part due to the significant rise i", - "n endorphins that accompanies fasting, as you can see in the graph below and at ", - "5:48 in my video . Mood enhancement during fasting is thought to perhaps repr", - "esent an adaptive survival mechanism to motivate the food search. This positive ", - "outlook towards the future may then facilitate the behavioral change necessary t", - "o lock in some of the weight-loss benefits. \n Is that what happens, though? Is f", - "asting actually effective over the long term? There are articles with titles l", - "ike Death During Therapeutic Starvation for Obesity. Is fasting even safe? We ", - "ll find out next. \n \n This is the sixth in a 14-part series on fasting for weigh", - "t loss. In case you missed any of the others, see the related videos below. \n My", - " book How Not to Diet is all about weight loss. You can learn more about it an", - "d order it here . \n " - ], - "key take away": [ - "\n Key Takeaways \n Although fasting has been called a fad, it has been practiced ", - "for centuries and has a long-standing place in different spiritual traditions.\n ", - "\n In the medical literature, fasting for the treatment of obesity dates back to", - " 1915 when it was said to be \n a perfectly safe, harmless, and effective meth", - "od for reducing the weight of those suffering from obesity. \n \n Today, about one", - " in seven American adults report fasting for weight control.\n \n In 1973, the Gu", - "inness Book of World Records published the longest recorded fast of 382 consec", - "utive days. The faster was given vitamin and mineral supplements so he wouldn ", - "t die, but no calories for more than a year , and lost 276 pounds and kept nea", - "rly all of it off .\n \n The reason diets don t work, almost by definition, is ", - "that people go on then off of them. Permanent weight loss is only achieved t", - "hrough permanent lifestyle change.\n \n Proponents of fasting point to the psychol", - "ogical benefit of realigning perceptions and motivation. The rapid weight loss", - " during fasting shows even the skeptical that making significant changes in eati", - "ng habits can result in losing weight, which may empower them to make better die", - "tary choices after the fast ends and recognize the power they have over food.\n \n", - " Hunger may start to wane within the first 36 hours of a fast.\n \n Fasting for ", - "just a day can cause feelings of irritability, moodiness, and distraction, but m", - "any report feeling clear, elated, alert, and even euphoric after a few days of n", - "ot eating, perhaps due in part to the significant rise in endorphins that acc", - "ompany fasting.\n \n Elevated mood during fasting may represent an adaptive su", - "rvival mechanism to motivate the food search.\n \n" - ], - "images": [ - "https://nutritionfacts.org/app/uploads/2024/01/5-48.png" - ], - "url": "https://nutritionfacts.org/blog/putting-the-benefits-of-fasting-for-weight-loss-to-the-test/" - }, - { - "title": "Can Blueberries Help with Diabetes and Repairing DNA?", - "date": "2023-05-18T12:00:57+00:00", - "author": "Michael Greger M.D. FACLM", - "content": [ - "\n Blueberries are put to the test against insulin resistance, oxidation, and DNA", - " damage.\n A famous pair of Harvard studies involved so many people over such lon", - "g a time that they ve accumulated millions of person-years of data. As I discu", - "ss in my video Flashback Friday: Blueberries for a Diabetic Diet and DNA Repair", - " ,\n the studies found the consumption of anthocyanin-rich foods, those containi", - "ng brightly colored plant pigments and particularly blueberries, was associate", - "d with a lower risk of developing type 2 diabetes. Just two or three servings of", - " blueberries a week has been associated with a 23 percent lower risk. In my Dail", - "y Dozen, I recommend at least one serving of berries every day a half-cup of fre", - "sh or frozen berries or a quarter-cup of dried.\n \n What do berries have to do wi", - "th diabetes? Well, type 2 diabetes is caused by insulin resistance, and interven", - "tional studies clearly showed that dietary BBE [berries] ameliorates insulin ", - "resistance. Sounds good, but that was in diabetic mice.\n What happens in peo", - "ple? As you can see in the graph below and at 1:05 in my video , those consumi", - "ng a lot of anthocyanin-rich foods like berries didn t just have less inflammat", - "ion,\n but significantly lower insulin resistance as well. By how much? By as mu", - "ch as you would get walking an hour or so a day, seven days a week.\n \n H ow ma", - "ny berries were they eating?\n They were getting 35 m illi g\nrams of anthocy", - "anins a day , which , as you can see below and at 1:28 in my video ,\n would b", - "e equivalent to a cup of strawberries or cherries, a half cup of raspberries", - ", a quarter cup of blackberries, or just a few spoon s\nful of blueberries. T ", - "hat was just a snapshot-in-time cross-sectional study , though . What we need ", - "are interventional trials, where you give people blueberries in a double-blind, ", - "randomized, and placebo-controlled clinical trial to truly put them to the test ", - ", and we got just that .\n \n In order to fake out people with a placebo ,\n ", - "the researchers used powdered blueberries equivalent to about two cups of fr", - "esh blueberries in a smoothie . As you can see in the graph below and at 2:09 i", - "n my video , t he results demonstrated a significant improvement in insulin ", - "sensitivity.\n \n There are a lot of blueberries in t wo cups !\n What benefits co", - "uld be expected for a typical half-cup serving? Another study demonstrated", - " a significant reduction in postprandial oxidation that is , all of the fr", - "ee radicals created when you eat some sugary breakfast like corn flakes. The a ", - "ntioxidant capacity of your bloodstream tak es a nosedive two hours afterwards ", - ",\n as your body tries to cope. But ,\n eat it with a half-cup of blueberries ,", - "\n and you r antioxidant levels start out higher and stay higher after the meal.", - " T he researchers also tried adding just a quarter cup of blueberries, bu", - "t that was not enough .\n S o, we should strive for a full serving.\n You can see", - " a graph depicting this below and at 2:29 in my video .\n \n The reason we care", - " about free radicals is because they can damage our DNA. Can a single portion", - " of blueberries really improve protection against DNA damage?\n Y es! Research", - "ers drew blood from people before and after they ate some frozen blueberries ", - ", then exposed their white blood cells to free radicals in the form of hydrogen", - " peroxide . They found that blueberr ies significantly reduced the DNA damage ", - "done within a single hour after berry consumption. \n However, the protective ef", - "fect was transient ,\n \n and our DNA vulnerability returned within two hours .\n ", - "So , we should eat super healthy foods like berries at every meal. In conclusi", - "on, one serving of blueberries can improve our cellular resistance to DNA dama", - "ge, \n thus supporting the importance of consuming vegetable foods regularly.\n", - " \n \n Berries pack such a nutritional punch that I recommend eating at least hal", - "f a cup a day in my Daily Dozen .\n \n( D ownload the free iPhone or Android", - " app, or print the Daily Dozen checklist here .\n)\n \n " - ], - "key take away": [ - "\n Key Takeaways \n A pair of Harvard studies that accumulated millions of person-", - "years of data found that intake of blueberries and other anthocyanin-rich foods ", - "was associated with lower risk of developing type 2 diabetes. My Daily Dozen rec", - "ommends at least one serving of berries (a half-cup fresh or frozen, or a quarte", - "r-cup dried) each day.\n In a snapshot-in-time cross-sectional study, researchers", - " found that individuals eating 35 milligrams of anthocyanins a day (equivalent t", - "o a few spoonsful of blueberries, a quarter-cup of blackberries, a half-cup of r", - "aspberries, or a cup of strawberries or cherries) had less inflammation and sign", - "ificantly lower insulin resistance as much as you d get walking every day for ab", - "out an hour.\n In an interventional trial, subjects were given powdered blueberri", - "es (equivalent to about two cups of fresh blueberries in a smoothie) or a placeb", - "o in a double-blind, randomized, and placebo-controlled clinical trial. Those gi", - "ven the berries had a significant improvement in insulin sensitivity.\n With a mo", - "re typical half-cup serving of berries, participants demonstrated a significant ", - "reduction in the creation of free radicals, which can damage our DNA, but a quar", - "ter-cup wasn t enough.\n Even a single portion of blueberries can improve protect", - "ion against DNA damage, but the effect is transient, so we should eat berries or", - " other healthy foods at every meal.\n" - ], - "images": [ - "https://nutritionfacts.org/app/uploads/2023/05/1-05.png", - "https://nutritionfacts.org/app/uploads/2023/05/1-28.png", - "https://nutritionfacts.org/app/uploads/2023/05/2-09.png", - "https://nutritionfacts.org/app/uploads/2023/05/2-29.png" - ], - "url": "https://nutritionfacts.org/blog/can-blueberries-help-with-diabetes-and-repairing-dna/" - }, - { - "title": "Circadian Rhythms and Our Blood Sugar Levels", - "date": "2024-04-30T12:00:57+00:00", - "author": "Michael Greger M.D. FACLM", - "content": [ - "\n The same meal eaten at the wrong time of day can double blood sugar. \n We ve k", - "nown for more than half a century that our glucose tolerance the ability of our ", - "body to keep our blood sugars under control declines as the day goes on. As yo", - "u can see in the graph below and at 0:25 in my video How Circadian Rhythms Affe", - "ct Blood Sugar Levels , if you hook yourself up to an IV and drip sugar water ", - "into your vein at a steady pace throughout the day, your blood sugars will start", - " to go up at about 8:00 pm, even though you haven t eaten anything and the infus", - "ion rate didn t change.\n The same amount of sugar is going into your system ever", - "y minute, but your ability to handle it deteriorates in the evening before bounc", - "ing right back in the morning. A meal eaten at 8:00 pm can cause twice the blo", - "od sugar response as an identical meal eaten at 8:00 am, as shown in the graph b", - "elow and at 0:51 in my video . It s as if you ate twice as much. Your body ju", - "st isn t expecting you to be eating when it s dark outside. Our species may have", - " only discovered how to use fire about a quarter million years ago. We just we", - "ren t built for 24-hour diners. \n One of the tests for diabetes is called the gl", - "ucose tolerance test, which sees how fast our body can clear sugar from our bloo", - "dstream. You swig down a cup of water with about four and a half tablespoons of ", - "regular corn syrup mixed in, then have your blood sugar measured two hours later", - ". By that point, your blood sugar should be under 140 mg/dL. Between 140 and 199", - " is considered to be a sign of prediabetes, and 200 and up is a sign of full-blo", - "wn diabetes, as you can see in the graph below and at 1:37 in my video . \n The ", - "circadian rhythm of glucose tolerance is so powerful that a person can test no", - "rmal in the morning but as a prediabetic later in the day. Prediabetics who ave", - "rage 163 mg/dL at 7:00 am may test out as frank diabetics at over 200 mg/dL at ", - "7:00 pm, as you can see in the graph below and at 1:53 in my video . \n Choosing", - " lower glycemic foods may help promote weight loss, but timing is critical. Due ", - "to this circadian pattern in glucose tolerance, a low-glycemic food at night can", - " cause a higher blood sugar spike than a high-glycemic food eaten in the morni", - "ng, as you can see below and at 2:05 in my video . \n We re so metabolically cr", - "ippled at night that researchers found that eating a bowl of All Bran cereal at", - " 8:00 pm caused as high a blood sugar spike as eating Rice Krispies at 8:00 am, ", - "as you can see in the graph below and at 2:23 in my video .\n High glycemic food", - "s at night would seem to represent the worst of both worlds. So, if you re goi", - "ng to eat refined grains and sugary junk, it might be less detrimental in the ", - "morning, as you can see in the graph below and at 2:32 in my video . \n \n The dr", - "op in glucose tolerance over the day could therefore help explain the weight-l", - "oss benefits of frontloading calories towards the beginning of the day. Even jus", - "t taking lunch earlier versus later may make a difference, as you can see in t", - "he graph below and at 2:48 in my video . \n People randomized to eat a large l", - "unch at 4:30 pm suffered a 46 percent greater blood sugar response compared to a", - "n identical meal eaten just a few hours earlier at 1:00 pm. A meal at 7:00 am ca", - "n cause 37 percent lower blood sugars than an identical meal at 1:00 pm, as yo", - "u can see below, and at 3:04 in my video . \n Now, there doesn t seem to be an", - "y difference between a meal at 8:00 pm and the same meal at midnight; they both ", - "seem to be too late, as you can see below, and at 3:15 in my video . \n But, ea", - "ting that late, at midnight or even 11:00 pm, can so disrupt your circadian rhy", - "thm that it can mess up your metabolism the next morning, resulting in significa", - "ntly higher blood sugars after breakfast, compared to eating the same dinner at ", - "6:00 pm the evening before, as shown in the graph below and at 3:32 in my video", - " .\n So, these revelations of chronobiology bring the breakfast debate full circl", - "e. Skipping breakfast not only generally fails to cause weight loss, but it wor", - "sens overall daily blood sugar control in both diabetic individuals and people ", - "who are not diabetic, as you can see in the graph below and at 3:44 in my vid", - "eo .\n Below and at 3:53, you can see a graph showing how the breakfast skippers ", - " have higher blood sugars even while they re sleeping 20 hours later. This may ", - "help explain why those who skip breakfast appear to be at higher risk of devel", - "oping type 2 diabetes in the first place. \n Breakfast skippers also tend to hav", - "e higher rates of heart disease, as well as having higher rates of atheroscle", - "rosis, in general. Is this just because skipping breakfast tends to cluster wit", - "h other unhealthy choices, including smoking and sicklier eating habits overall", - "? The link between skipping breakfast and heart disease even premature death in ", - "general seems to survive attempts to control for these confounding factors, bu", - "t you don t really know until you put it to the test.\n Does skipping breakfast l", - "ead to higher cholesterol, for example? Yes, researchers found a significant r", - "ise in LDL (bad) cholesterol in study participants randomized to skip breakfast;", - " they were about 10 points higher within just two weeks, as you can see below an", - "d at 4:45 in my video .\n The Israeli study with the caloric distribution of 700", - " calories for breakfast, 500 for lunch, and 200 for dinner that I ve discussed p", - "reviously found that the triglycerides of the king-prince-pauper group (those ", - "eating more at breakfast versus dinner) got significantly better a 60-point drop", - " while those of the pauper-prince-king group got significantly worse (a 26-point", - " rise). So, consuming more calories in the morning relative to the evening may a", - "ctually have a triple benefit: more weight loss, better blood sugar control, and", - " lower heart disease risk, as you can see below and at 5:18 in my video . \n If ", - "you re going to skip any meal, whether you re practicing intermittent fasting or", - " time-restricted feeding (where you try to fit all of your food intake into a ce", - "rtain time window each day), it may be safer and more effective to skip dinner r", - "ather than breakfast.\n I m back with the next installment of the chronobiology s", - "eries! I previously explored eating breakfast for weight loss ( Is Breakfast the", - " Most Important Meal for Weight Loss?\n and Is Skipping Breakfast Better for We", - "ight Loss?\n), introduced chronobiology ( How Circadian Rhythms Can Control Your ", - "Health and Weight ), and looked at the science on eating more in the mornings th", - "an the evenings ( Eat More Calories in the Morning to Lose Weight , Breakfast L", - "ike a King, Lunch Like a Prince, Dinner Like a Pauper , and Eat More Calories i", - "n the Morning Than the Evening ).\n Next, you ll see How to Sync Your Central Ci", - "rcadian Clock to Your Peripheral Clocks .\n The series will wrap up in the next c", - "ouple of weeks. See videos and blogs in related posts below.\n Note: The Israeli ", - "700/500/200 study that I mentioned is detailed in the Breakfast Like a King, Lu", - "nch Like a Prince, Dinner Like a Pauper video if you want to know more. Also, c", - "heck the corresponding blog in related posts. \n " - ], - "key take away": [ - "\n Key Takeaways \n Glucose tolerance, the ability of our body to regulate blood s", - "ugar levels, declines throughout the day, with higher blood sugar responses to m", - "eals consumed in the evening compared to the morning.\n Diabetes tests, such as t", - "he glucose tolerance test, may yield different results depending on the time of ", - "day, emphasizing the circadian influence on blood sugar control.\n Consuming high", - " glycemic foods at night can lead to higher blood sugar spikes than the same foo", - "ds eaten in the morning. This is due to the metabolic decline in glucose toleran", - "ce during the evening.\n Frontloading calories earlier in the day, including havi", - "ng a larger lunch, may contribute to better blood sugar control and potentially ", - "aid in weight loss by reducing overall blood sugar responses.\n Skipping breakfas", - "t may not only fail to promote weight loss but can also worsen daily blood sugar", - " control, increase the risk of type 2 diabetes, and lead to adverse cardiovascul", - "ar effects, such as higher LDL cholesterol levels. \n" - ], - "images": [ - "https://nutritionfacts.org/app/uploads/2024/04/0-25.png", - "https://nutritionfacts.org/app/uploads/2024/04/0-51.png", - "https://nutritionfacts.org/app/uploads/2024/04/1-37.png", - "https://nutritionfacts.org/app/uploads/2024/04/1-53.png", - "https://nutritionfacts.org/app/uploads/2024/04/2-05-1.png", - "https://nutritionfacts.org/app/uploads/2024/04/2-23.png", - "https://nutritionfacts.org/app/uploads/2024/04/2-32-1.png", - "https://nutritionfacts.org/app/uploads/2024/04/2-48.png", - "https://nutritionfacts.org/app/uploads/2024/04/3-04.png", - "https://nutritionfacts.org/app/uploads/2024/04/3-15.png", - "https://nutritionfacts.org/app/uploads/2024/04/3-32.png", - "https://nutritionfacts.org/app/uploads/2024/04/3-44.png", - "https://nutritionfacts.org/app/uploads/2024/04/3-53.png", - "https://nutritionfacts.org/app/uploads/2024/04/4-45.png", - "https://nutritionfacts.org/app/uploads/2024/04/5-18.png" - ], - "url": "https://nutritionfacts.org/blog/circadian-rhythms-and-our-blood-sugar-levels/" - }, - { - "title": "The Political Power of the Food Industry", - "date": "2023-10-03T12:00:28+00:00", - "author": "Michael Greger M.D. FACLM", - "content": [ - "\n What can millions of dollars in the hands of the lobbying industry do to shut ", - "down efforts to protect children? \n \n For nearly half a century, there have been", - " calls to ban the advertising of sugary cereals to children, a product that Ha", - "rvard nutrition professor Jean Mayer referred to as sugar-coated nothings. In ", - "a Senate hearing on nutrition education, he said, Properly speaking, they ought", - " to be called cereal-flavored candy, rather than sugar-covered cereals. \n \n As", - " I discuss in my video A Political Lesson on the Power of the Food Industry , t", - "he Senate committee invited the major manufacturers of children s cereals to t", - "estify, and they initially said yes until they heard what kinds of questions wer", - "e going to be asked. One cereal industry representative candidly admitted why th", - "e decision was made to boycott the hearing: They simply didn t have persuasive ", - "answers to why they were trying to sell kids breakfast candy.\n \n In the Mad Men", - " age before the consumer movement was in bloom, ad company executives were more", - " willing to talk frankly about the purpose of their ads and how they felt about ", - "aiming the ads at the child market. Said an executive of the Kellogg s ad firm", - ": Our primary goal is to sell products to children, not educate them. When you ", - "sell a woman on a product and she goes into the store and finds your brand isn t", - " in stock, she ll probably forget about it. But when you sell a kid on your prod", - "uct, if he can t get it, he will throw himself on the floor, stamp his feet and ", - "cry. You can t get a reaction like that out of an adult. \n \n Sugary cereals are", - " the number one food advertised to kids, but don t worry the industry will jus", - "t self-regulate. In response to public health concerns about the amount of mark", - "eting for nutritionally poor food directed to children, the Council of Better Bu", - "siness Bureaus launched the Children s Food and Beverage Advertising Initiative ", - " in which all the big cereal companies pledged to market only healthier dietary", - " choices in child-directed advertising. The candy industry signed on, too. De", - "spite pledging not to advertise to kids, after the initiative went into effect, ", - "kids actually saw more candy ads. Take Hershey, for example. It doubled its ad", - "vertising to children at the same time it pledged to not advertise to children.", - " \n \n The cereal companies got to decide for themselves their own definitions ", - "of healthier dietary choices. That should give us a sense of how serious they ", - "are at protecting children. For example, they classified Froot Loops and Reese ", - "s Peanut Butter Puffs consisting of up to 44% sugar by weight as healthier diet", - "ary choices. In that case, what are their unhealthy choices? It seems that th", - "e Children s Food and Beverage Advertising Initiative basically just based it", - "s maximal nutrient levels more on the current products marketed by its members t", - "han on a judgment about what was best for children. \n \n Now, they ve since rev", - "ised that to allow only cereals that are 38 percent sugar by weight. But even i", - "f they are only one-third sugar, that means kids are effectively eating one s", - "poonful of sugar in every three spoons of cereal not exactly a healthier dietary", - " choice.\n \n The Federal Trade Commission tried stepping in back in 1978, but t", - "he industry poured in so many millions of dollars in lobbying might that Congr", - "ess basically threatened to yank the entire agency s funding should the FTC mess", - " with Big Cereal, demonstrating just how powerful market forces are compared to", - " those that can be mobilized on behalf of children. The political post-traumat", - "ic stress induced by the aggressive attacks on the FTC led to a twenty-five-ye", - "ar hiatus in federal efforts to rein in food marketing aimed at children. \n \n F", - "inally, enter the Interagency Working Group with members from four federal age", - "ncies the FTC, CDC, FDA, and USDA. The group developed a set of voluntary princ", - "iples [that] are designed to encourage stronger and more meaningful self-regul", - "ation by the food industry and to support parents efforts to get their kids to ", - "eat healthier foods. It proposed the radical suggestion of not marketing to", - " children cereals that are more than 26 percent pure sugar. \n \n As you can see b", - "elow and at 4:02 in my video , the top ten breakfast cereals marketed to childr", - "en are Cinnamon Toast Crunch, Lucky Charms, Honey Nut Cheerios, Froot Loops, Ree", - "se s Puffs, Trix, Frosted Flakes, Fruity Pebbles, Cocoa Puffs, and Cookie Crisp ", - "and not a single one would meet that standard. General Mills shot back: The", - " Proposal s nutrition standards are arbitrary, capricious, and fundamentally fla", - "wed. No surprise since literally all cereals marketed by General Mills would", - " be barred from advertising not a single one would make the cut. To suggest volu", - "ntary standards unconstitutionally restrains commercial speech in violation of ", - "the First Amendment, to which the FTC basically replied : Let me get you a di", - "ctionary . How could suggesting voluntary guidelines violate the Constitution?", - " But that s how freaked out the industry is at even the notion of meaningful g", - "uidelines. One grocer s association actually called the proposed nutrition p", - "rinciples the most bizarre and unconscionable it had ever seen.\n \n So, what ha", - "ppened? Again, agency funding was jeopardized ,\n so the FTC called off the inte", - "ragency proposal. \n \n At every level of government, the food and beverage indust", - "ries won fight after fight .They have never lost a significant political battl", - "e in the United States Said a director of one of the child advocacy organizatio", - "ns: We just got beat. Money wins. And it took a lot of money $175 million of B", - "ig Food lobbying funds. It was apparently enough to buy the White House s silenc", - "e as the interagency proposal got killed off. As one Obama advisor put it, You ", - "can tell someone to eat less fat, consume more fiber, more fruits and vegetables", - ", and less sugar. But if you start naming foods, you cross the line. \n \n I m up", - "set with the White House, said Senator Tom Harkin (D-Iowa), chairman of the Sen", - "ate Health Committee. They went wobbly in the knees, and when it comes to kids ", - " health, they shouldn t go wobbly in the knees. \n \n For more on breakfast cerea", - "ls, click here . And click here for more on sugar. \n \n I am all in favor of ", - "Taking Personal Responsibility for Your Health , but the strong-arm tobacco-styl", - "e tactics of the multitrillion-dollar food industry are contributing to the deat", - "hs of an estimated 14 million people every year.\n \n On a brighter note, check ou", - "t How We Won the Fight to Ban Trans Fat .\n \n For more on sugar specifically, se", - "e Flashback Friday: Sugar Industry Attempts to Manipulate the Science.\n \n Chec", - "k out my other videos on breakfast cereals: Flashback Friday: The Worst Food f", - "or Tooth Decay and How to Stop Tooth Decay .\n Are there any healthy cereals? A", - " few make the cut. See Flashback Friday: The Five-to-One Fiber Rule .\n \n " - ], - "key take away": [ - "\n Key Takeaways \n For about 50 years, there have been calls to ban the advertisi", - "ng of sugar cereals to kids, products referred to as sugar-coated nothings and", - " cereal-flavored candy by a Harvard nutrition professor and the number one foo", - "d marketed to children.\n Major manufacturers of children s cereal accepted invit", - "ations to testify at a Senate hearing on nutrition education, but later declined", - ", feeling they didn t have persuasive answers for why they were trying to sell", - " kids breakfast candy.\n \n Said an executive of the Kellogg s ad firm: Our prima", - "ry goal is to sell products to children, not educate them. When you sell a woma", - "n on a product and she goes into the store and finds your brand isn t in stock, ", - "she ll probably forget about it. But when you sell a kid on your product, if he ", - "can t get it, he will throw himself on the floor, stamp his feet and cry. You ca", - "n t get a reaction like that out of an adult. \n Big Cereal decided to self-regul", - "ate and pledged to market only healthier dietary choices in child-directed adve", - "rtising. Big Candy joined, too, but after the initiative went into effect, kids", - " actually saw more candy ads.\n \n The cereal companies created their own defini", - "tions of healthier dietary choices, classifying Froot Loops and Reese s Peanu", - "t Butter Puffs consisting of up to 44% sugar by weight as healthier dietary cho", - "ices. \n Though the level has since been lowered to 38 percent sugar by weight, t", - "hat s still effectively one spoonful of sugar in every three spoons of cereal. ", - "\n \n The FTC attempted to step in in 1978, but the lobbying might of the industry", - " won and the FTC backed down, leading to a twenty-five-year hiatus in federal e", - "fforts to rein in food marketing aimed at children. \n The Interagency Working Gr", - "oup, comprised of members from the FTC, CDC, FDA, and USDA, proposed not allowin", - "g cereals that are more than 26 percent pure sugar to be marketed to children.\n ", - "Not one of the top ten breakfast cereals marketed to kids would meet that standa", - "rd, nor would even one cereal marketed by General Mills.\n The industry fought ba", - "ck with $175 million of Big Food lobbying funds and won again, so the FTC called", - " off the interagency proposal.\n I m upset with the White House, said Senator To", - "m Harkin (D-Iowa), chairman of the Senate Health Committee. They went wobbly in", - " the knees, and when it comes to kids health, they shouldn t go wobbly in the k", - "nees. \n \n" - ], - "images": [ - "https://nutritionfacts.org/app/uploads/2023/10/4-02.jpg" - ], - "url": "https://nutritionfacts.org/blog/the-political-power-of-the-food-industry/" - }, - { - "title": "Preorder How Not to Age Today", - "date": "2023-09-07T12:00:34+00:00", - "author": "Michael Greger M.D. FACLM", - "content": [ - "\n I m thrilled to announce that preorders are now open for How Not to Age , my ", - "new book coming out in December after three years in the making. There was so mu", - "ch juicy research on the many aspects of aging and longevity more than 13,000 ci", - "tations worth! that I couldn t even fit it all in. So, throughout the book, you", - " will find links to special videos I made just for How Not to Age to provide y", - "ou with even more life-changing, life-saving information. \n Preorders are availa", - "ble now . If you plan to buy How Not to Age , please do me the favor of preorde", - "ring. All preorders count towards first week sales, which factor into the New Y", - "ork Times Best Sellers list calculations. So, the more preorders How Not to Ag", - "e receives, the more likely it will make the list, which could help expose mill", - "ions to the message of healthy eating. In fact, the book lands on December 5 per", - "fect timing for holiday gifts for everyone in your life you want to live long an", - "d healthy lives! Please consider getting some of your holiday shopping done earl", - "y by ordering a whole stack of them! \n Preorder and Immediately Receive a Chapte", - "r \n We are also offering a gift! Everyone who preorders a copy of How Not to Ag", - "e can immediately receive the Preserving Your Bones chapter from the book. Afte", - "r you place your order,\n enter your order number on this page to receive your ", - "free chapter. Be among the first to get this sneak peek into the book by preorde", - "ring now. \n As always, all proceeds I receive from all of my books are donated", - " directly to charity. \n P.S. In a few weeks I will be announcing an upcoming web", - "inar where I will share my brand-new How Not to Age presentation before openin", - "g it up for questions. Keep your eyes out for that email at the end of September", - ".\n " - ], - "key take away": [ - "Key Take Away not found" - ], - "images": [ - "https://nutritionfacts.org/app/uploads/2023/08/hnta-free-chapter-960x450.jpeg" - ], - "url": "https://nutritionfacts.org/blog/preorder-how-not-to-age-today/" - } -] \ No newline at end of file diff --git a/src/backend/RAG/LangChain_Implementation/chroma_db/3c3261c1-ed70-4735-b4da-b018031bc59a/data_level0.bin b/src/backend/RAG/LangChain_Implementation/chroma_db/3c3261c1-ed70-4735-b4da-b018031bc59a/data_level0.bin deleted file mode 100644 index ea3192e..0000000 Binary files a/src/backend/RAG/LangChain_Implementation/chroma_db/3c3261c1-ed70-4735-b4da-b018031bc59a/data_level0.bin and /dev/null differ diff --git a/src/backend/RAG/LangChain_Implementation/chroma_db/3c3261c1-ed70-4735-b4da-b018031bc59a/header.bin b/src/backend/RAG/LangChain_Implementation/chroma_db/3c3261c1-ed70-4735-b4da-b018031bc59a/header.bin deleted file mode 100644 index 3e0932a..0000000 Binary files a/src/backend/RAG/LangChain_Implementation/chroma_db/3c3261c1-ed70-4735-b4da-b018031bc59a/header.bin and /dev/null differ diff --git a/src/backend/RAG/LangChain_Implementation/chroma_db/3c3261c1-ed70-4735-b4da-b018031bc59a/length.bin b/src/backend/RAG/LangChain_Implementation/chroma_db/3c3261c1-ed70-4735-b4da-b018031bc59a/length.bin deleted file mode 100644 index 197a882..0000000 Binary files a/src/backend/RAG/LangChain_Implementation/chroma_db/3c3261c1-ed70-4735-b4da-b018031bc59a/length.bin and /dev/null differ diff --git a/src/backend/RAG/LangChain_Implementation/chroma_db/3c3261c1-ed70-4735-b4da-b018031bc59a/link_lists.bin b/src/backend/RAG/LangChain_Implementation/chroma_db/3c3261c1-ed70-4735-b4da-b018031bc59a/link_lists.bin deleted file mode 100644 index e69de29..0000000 diff --git a/src/backend/RAG/LangChain_Implementation/chroma_db/chroma.sqlite3 b/src/backend/RAG/LangChain_Implementation/chroma_db/chroma.sqlite3 deleted file mode 100644 index 01de934..0000000 Binary files a/src/backend/RAG/LangChain_Implementation/chroma_db/chroma.sqlite3 and /dev/null differ diff --git a/src/backend/RAG/LangChain_Implementation/retriever.py b/src/backend/RAG/LangChain_Implementation/retriever.py deleted file mode 100644 index 00cde15..0000000 --- a/src/backend/RAG/LangChain_Implementation/retriever.py +++ /dev/null @@ -1,197 +0,0 @@ -import os -import sys - -from astrapy import DataAPIClient -from astrapy.db import AstraDB -from dotenv import load_dotenv -from langchain.schema import Document -from langchain.text_splitter import RecursiveCharacterTextSplitter - -# from langchain_google_genai import ChatGoogleGenerativeAI -from langchain_community.vectorstores.chroma import Chroma - -# from langchain.embeddings import OpenAIEmbeddings -# from langchain_community.embeddings import OpenAIEmbeddings -# from langchain_community.embeddings import HuggingFaceEmbeddings -# from langchain_huggingface import HuggingFaceEmbeddings -# from langchain_community.llms.openai import OpenAI -from langchain_openai import OpenAI, OpenAIEmbeddings - -load_dotenv() - - -class Retriever: - def __init__(self): - self.openai_api_key = os.environ.get('OPENAI_API_KEY') - self.google_api_key = os.environ.get('GOOGLE_API_KEY') - - default_embedding = OpenAIEmbeddings(openai_api_key=self.openai_api_key) - default_llm = OpenAI(temperature=0.2) - - self.embedding_options = ['OpenAI', 'HuggingFace'] - self.llm_options = ['chatgpt', 'gemini'] - - self.llm = default_llm - self.text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200) - self.embedding_model = default_embedding - - self.document_list = [] - - astra_db_namespace = 'test' - astra_db_collection = 'test_collection_2' - client = DataAPIClient(os.environ.get('ASTRA_DB_TOKEN')) - self.database = client.get_database(os.environ.get('ASTRA_DB_API_ENDPOINT')) - - self.db = AstraDB( - token=os.getenv('ASTRA_DB_TOKEN'), - api_endpoint=os.getenv('ASTRA_DB_API_ENDPOINT'), - namespace=os.getenv(astra_db_namespace), - ) - self.collection = self.db.collection(collection_name=astra_db_collection) - - self.preprompt = """You are a medical AI agent that tries to answer the user's - questions as good as possible. You will receive useful - information about the topic, that might help you give better answers. - Here is the user's question: ---""" - - # similarity search in AstraDB - def astra_similarity_search(self, query: str): - query_embedding = self.embedding_model.embed_query(query) - - sort = {'$vector': query_embedding} - options = {'limit': 2} - projection = {'content': 1, 'metadata': 1, '$vector': 1} - - document_list = self.collection.find(sort=sort, options=options, projection=projection) - return document_list - - def manik_retrieval(self, query): - document_chunks = [] - document_list = self.astra_similarity_search(query) - for document in document_list['data']['documents']: - content = document.get('content', '') - metadata = document.get('metadata', {}) - document_chunks.append(Document(page_content=content, metadata=metadata)) - - # Process documents with LangChain and generate response - response = '' - if document_chunks: - relevant_info = '' - for doc in document_chunks: - relevant_info += f'Content: {doc.page_content}\nMetadata: {doc.metadata}\n\n' - print(f'Content: {doc.page_content}\nMetadata: {doc.metadata}\n\n') - context = ( - 'You are a health consultant specializing in answering' - 'health-related queries and providing comprehensive insights. ' - 'You have access to various documents containing' - 'information about the latest trends in the health industry, ' - 'summarize the key points, and draw conclusions' - 'based on the provided information.' - ) - - # multi_step_reasoning = ('') - # few_shot_examples = ('') - # hypothetical_scenario = ('') - - modified_prompt = ( - f'{context}\n\n' - # f'{multi_step_reasoning}\n\n' - # f'{few_shot_examples}\n\n' - # f'{hypothetical_scenario}\n\n' - f'Here is some relevant information:\n{relevant_info}\n\n' - f'Now, based on the provided information and your expertise' - ', answer the following questions:\n' - f'Q: What are the protective effects mentioned?\n' - f'Q: What can we conclude from this information?' - ) - - response = self.llm.predict(modified_prompt) - return response - - def which_llm(self): - print(self.llm) - - def choose_llm(self, llm_name: str): - if llm_name.lower() not in self.llm_options: - print('Choose one of the available LLM names:') - for i in self.llm_options: - print(i) - return - - if llm_name.lower() == 'ChatGPT'.lower(): - self.llm = OpenAI(temperature=0.2) - print('ChatGPT set as LLM') - elif llm_name.lower() == 'Gemini'.lower(): - self.llm = ChatGoogleGenerativeAI(model='gemini-pro') - print('gemini-pro set as LLM') - return - - def choose_embedding_model(self, embedding_fct: str): - if embedding_fct not in self.embedding_options: - print('Choose one of the available embedding models:') - for i in self.embedding_options: - print(i) - return - - if embedding_fct == 'OpenAI': - self.embedding_model = OpenAIEmbeddings() - print('OpenAI-Embeddings set as embedding model') - elif embedding_fct == 'HuggingFace': - HuggingFaceEmbeddings(model_name='all-MiniLM-L6-v2') - print("HuggingFace 'all-MiniLM-L6-v2' set as embedding model") - return - - def chroma_similarity_search(self, query_text): - embedded_query = self.embedding_model.embed_query(query_text) - chroma_db = Chroma(persist_directory='./chroma_db', embedding_function=self.embedding_model) - - return chroma_db._similarity_search_with_relevance_scores(embedded_query, k=2) - - # def context_enriched_answer(self, query_text): - # results = self.chroma_similarity_search(query_text) - # relevant_info = '' - # for result, score in results: - # doc_content = result.page_content - # metadata = result.metadata - # metadata_str = ', '.join([f'{key}: {value}' for key, value in metadata.items()]) - # relevant_info += f'Content: {doc_content}\nMetadata: {metadata_str}\n\n' - - # modified_prompt = f'{self.preprompt}{query_text}---\n - # Here is some relevant information:\n***{relevant_info}***' - # response = self.llm.predict(modified_prompt) - # print(f'{response}') - # return response - - def simple_response(self, query_text): - response = self.llm.invoke(query_text) - print(f'{response}') - return response - - -### Test: -# retriever = Retriever() -# test_query = "How many corners does a heptagon have?" - -# -# retriever.which_llm() -# retriever.choose_llm('gemini') -# retriever.which_llm() - -# complex_query = "What should I eat for lunch if have diabetes?" -# retriever.manik_retrieval(complex_query) -# retriever.simple_response(test_query) - - -def main(): - input_string = sys.argv[1] - retriever = Retriever() - output_string = retriever.simple_response(input_string) - print(output_string) - - -# def process_input(input_string): -# # Here you can process the input string as needed -# return f"Processed: {input_string}" - -if __name__ == '__main__': - main() diff --git a/src/backend/RAG/create_embeddings/create_embeddings.py b/src/backend/RAG/create_embeddings/create_embeddings.py deleted file mode 100644 index 5f4a4ae..0000000 --- a/src/backend/RAG/create_embeddings/create_embeddings.py +++ /dev/null @@ -1,224 +0,0 @@ -import os -import uuid - -# free local vector store: -import chromadb - -# for alternative using only LangChain syntax: -# from sentence_transformers import SentenceTransformer -from chromadb.utils.embedding_functions import SentenceTransformerEmbeddingFunction -from langchain.embeddings import HuggingFaceBgeEmbeddings # imports for open source modules: -from langchain_community.embeddings import HuggingFaceEmbeddings - -""" imports for modules that need API keys: """ -# from langchain.embeddings.openai import OpenAIEmbeddings - -# from langchain.embeddings import VertexAIEmbeddings -# from langchain_google_genai import GoogleGenerativeAIEmbeddings - -# AstraDB - vector store: -# from langchain_community.storage import AstraDBStore - - -""" -Open source modules: -""" - - -class HFembeddings: - def __init__(self) -> None: - self.model_name = 'BAAI/bge-base-en-v1.5' - self.model_kwargs = {'device': 'cpu'} - self.encode_kwargs = {'normalize_embeddings': True} - - self.hf = HuggingFaceBgeEmbeddings( - model_name=self.model_name, - model_kwargs=self.model_kwargs, - encode_kwargs=self.encode_kwargs, - ) - - def embed_text(self, text): - return self.hf.embed_documents(text) - - def embed_text_list(self, text_list): - return [self.hf.embed_documents(text) for text in text_list] - - -# alternative implementation using only LangChain native syntax: -def hf_embed_text_list(text_list): - embeddings = HuggingFaceEmbeddings() - return [embeddings.embed_documents(text) for text in text_list] - - -def hf_embed_text(text): - embeddings = HuggingFaceEmbeddings() - return embeddings.embed_documents(text) - - -# Chroma DB -def get_local_chroma(): # name="local_chromaDB"): - storage_path = 'storage' # os.getenv('STORAGE_PATH') - if not os.path.exists(storage_path): - os.mkdir(storage_path) - return chromadb.PersistentClient(path=storage_path) # Client - # return client.get_or_create_collection(name=name) - - -def prepare_entry(entry: dict): - if 'document' not in entry: - print( - 'Cannot add embedding: There is no document to embed. ', - 'If there is, check if it is saved under correct value name.', - ) - return - - if '_id' not in entry: - entry['_id'] = str(uuid.uuid4()) - if entry['_id'] == '': - entry['_id'] = str(uuid.uuid4()) - - if 'metadatas' not in entry: - entry['metadatas'] = {} - - # Chroma would actually have its own embedding function which - # could be used alternatively - # use embedding function - currently HuggingFace - - return entry - - -def chroma_add_entry_wth_embedding(entry: dict, collection): - # embedding = hf.embed_text(entry['document']) - collection.add( - ids=[str(entry['_id'])], - documents=[entry['document']], - # embeddings=[embedding], - metadatas=[entry['metadatas']], - ) - return # collection - - -""" -Modules with need of API tokens: -""" - -# OpenAI: -# api_key = os.environ.get('OPENAI_API') -# embeddings = OpenAIEmbeddings(model="text-embedding-ada-002", openai_api_key=api_key) -""" -def hf_embed_text_list(text_list): - embeddings = OpenAIEmbeddings(model="text-embedding-ada-002", openai_api_key=api_key) - return [embeddings.embed_documents(text) for text in text_list] - -def hf_embed_text(text): - embeddings = OpenAIEmbeddings(model="text-embedding-ada-002", openai_api_key=api_key) - return embeddings.embed_documents(text) -""" - - -# Google -# embeddings = GoogleGenerativeAIEmbeddings(model="models/embedding-001") -# vector = embeddings.embed_query("hello, world!") -""" -def gg_embed_text_list(text_list): - embeddings = GoogleGenerativeAIEmbeddings(model="models/embedding-001") - return [embeddings.embed_documents(text) for text in text_list] - -def gg_embed_text(text): - embeddings = GoogleGenerativeAIEmbeddings(model="models/embedding-001") - return embeddings.embed_documents(text) -""" - -# Google Vertex: -# vertex_embeddings = VertexAIEmbeddings(model_name="textembedding-gecko@003") -# vector = vertex_embeddings.embed_query("hello, world!") -""" -def vx_embed_text_list(text_list): - embeddings = VertexAIEmbeddings(model="models/embedding-001") - return [embeddings.embed_documents(text) for text in text_list] - -def vx_embed_text(text): - embeddings = VertexAIEmbeddings(model="models/embedding-001") - return embeddings.embed_documents(text) -""" - -# Storing in AstraDB: -""" -ASTRA_DB_APPLICATION_TOKEN = os.environ.get("ASTRA_DB_APPLICATION_TOKEN") -ASTRA_DB_API_ENDPOINT = os.environ.get("ASTRA_DB_API_ENDPOINT") -# ASTRA_DB_KEYSPACE = os.environ.get("ASTRA_DB_KEYSPACE") - -embedding = hf # use hf for test # OpenAIEmbeddings() -vstore = AstraDBVectorStore( - embedding=embedding, - #namespace=ASTRA_DB_KEYSPACE, - collection_name="test", - token=os.environ["ASTRA_DB_APPLICATION_TOKEN"], - api_endpoint=os.environ["ASTRA_DB_API_ENDPOINT"], -) -""" -""" -# Testing AstraDB: -philo_dataset = load_dataset("datastax/philosopher-quotes")["train"] -print("An example entry:") -print(philo_dataset[16]) -docs = [] -for entry in philo_dataset: - metadata = {"author": entry["author"]} - if entry["tags"]: - # Add metadata tags to the metadata dictionary - for tag in entry["tags"].split(";"): - metadata[tag] = "y" - # Add a LangChain document with the quote and metadata tags - doc = Document(page_content=entry["quote"], metadata=metadata) - docs.append(doc) -""" - - -""" main: """ -if __name__ == '__main__': - test_text = 'this is a text to embed' - test_entry = { - '_id': 'id1', # str(uuid.uuid4()), - 'document': test_text, - 'metadatas': {'test_key': 'test_value'}, - } - test_entry2 = { - '_id': 'id2', # str(uuid.uuid4()), - 'document': 'more text for the test', - 'metadatas': {'test_key': 'test_value'}, - } - hf = HFembeddings() - # test_embedding = hf_embed_text(test_text) - - client = get_local_chroma() # chromadb.Client() - collection = client.get_or_create_collection( - name='local_chroma', - embedding_function=SentenceTransformerEmbeddingFunction(model_name='all-MiniLM-L6-v2'), - ) - # chroma_add_entry_wth_embedding(prepare_entry(test_entry), collection) - - # collection = client.create_collection(name="personal_collection") - entry = prepare_entry(test_entry) - - embedding = hf.embed_text(test_entry['document']) # hf_embed_text(entry['document']) - embedding2 = hf.embed_text(test_entry2['document']) # hf_embed_text(entry['document']) - - collections = client.list_collections() - print(collections) - - collection.add( - ids=[test_entry['_id'], test_entry2['_id']], - documents=[test_entry['document'], test_entry2['document']], - # embeddings=[embedding, embedding2], - metadatas=[test_entry['metadatas'], test_entry2['metadatas']], - ) - - print('collection count') - print(collection.count()) - - results = collection.query(query_texts=['This is a test query'], n_results=2) - print(results) - - # uncomment for demo - # print(collection.peek())