update RAG

.obsidian/workspace.json

@@ -13,7 +13,115 @@
             "state": {
               "type": "markdown",
               "state": {
-                "file": "RAG/RAG Using Llama 3.md",
+                "file": "RAG/RAG Re-Ranking.md",
+                "mode": "source",
+                "source": false
+              }
+            }
+          },
+          {
+            "id": "a974719fe1cc1577",
+            "type": "leaf",
+            "state": {
+              "type": "markdown",
+              "state": {
+                "file": "Rerank.md",
+                "mode": "source",
+                "source": false
+              }
+            }
+          },
+          {
+            "id": "79d7e976d5aefe23",
+            "type": "leaf",
+            "state": {
+              "type": "markdown",
+              "state": {
+                "file": "Embedding Mode.md",
+                "mode": "source",
+                "source": false
+              }
+            }
+          },
+          {
+            "id": "0ffcf615f0cbc21b",
+            "type": "leaf",
+            "state": {
+              "type": "markdown",
+              "state": {
+                "file": "Generation.md",
+                "mode": "source",
+                "source": false
+              }
+            }
+          },
+          {
+            "id": "ab7851afd5287f85",
+            "type": "leaf",
+            "state": {
+              "type": "markdown",
+              "state": {
+                "file": "Augmentation.md",
+                "mode": "source",
+                "source": false
+              }
+            }
+          },
+          {
+            "id": "b89d155e74f30af0",
+            "type": "leaf",
+            "state": {
+              "type": "markdown",
+              "state": {
+                "file": "Retrieval.md",
+                "mode": "source",
+                "source": false
+              }
+            }
+          },
+          {
+            "id": "650ae950226baa04",
+            "type": "leaf",
+            "state": {
+              "type": "markdown",
+              "state": {
+                "file": "LLM-Generated Content.md",
+                "mode": "source",
+                "source": false
+              }
+            }
+          },
+          {
+            "id": "e741fa0af705d39b",
+            "type": "leaf",
+            "state": {
+              "type": "markdown",
+              "state": {
+                "file": "Structured Data.md",
+                "mode": "source",
+                "source": false
+              }
+            }
+          },
+          {
+            "id": "9fb7b72f30a1174a",
+            "type": "leaf",
+            "state": {
+              "type": "markdown",
+              "state": {
+                "file": "Semi-Structured Data.md",
+                "mode": "source",
+                "source": false
+              }
+            }
+          },
+          {
+            "id": "534c379a76c45e97",
+            "type": "leaf",
+            "state": {
+              "type": "markdown",
+              "state": {
+                "file": "Unstructured Data.md",
                 "mode": "source",
                 "source": false
               }
@@ -121,7 +229,7 @@
             "state": {
               "type": "backlink",
               "state": {
-                "file": "RAG/RAG Using Llama 3.md",
+                "file": "RAG/RAG Re-Ranking.md",
                 "collapseAll": false,
                 "extraContext": false,
                 "sortOrder": "alphabetical",
@@ -138,7 +246,7 @@
             "state": {
               "type": "outgoing-link",
               "state": {
-                "file": "RAG/RAG Using Llama 3.md",
+                "file": "RAG/RAG Re-Ranking.md",
                 "linksCollapsed": false,
                 "unlinkedCollapsed": true
               }
@@ -161,7 +269,7 @@
             "state": {
               "type": "outline",
               "state": {
-                "file": "RAG/RAG Using Llama 3.md"
+                "file": "RAG/RAG Re-Ranking.md"
               }
             }
           }
@@ -184,42 +292,43 @@
   },
   "active": "102c8828856b0385",
   "lastOpenFiles": [
-    "Pasted image 20240715222653.png",
-    "RAG/AI Agents Use Case.md",
+    "Rerank.md",
+    "RAG/RAG Re-Ranking.md",
+    "Pasted image 20240716000853.png",
+    "img/Pasted image 20240716000730.png",
+    "LLMs/LLMs APP.md",
+    "RAG/RAG vs FineTuning.md",
+    "img/Pasted image 20240716000624.png",
+    "Accuracy.md",
+    "RAG/Advanced RAG LimaIndex & Claude 3.md",
+    "FineTuning",
+    "img/Pasted image 20240716000446.png",
+    "RAG/Text Splitting RAG.md",
+    "NLP/Service used for NLP in Microsoft Azure.md",
+    "img/Pasted image 20240716000406.png",
+    "RAG/RAG Stack.md",
+    "Embedding Mode.md",
+    "img/Pasted image 20240716000053.png",
+    "RAG/RAG implementing.md",
+    "Generation.md",
+    "Augmentation.md",
+    "Retrieval.md",
+    "img/Pasted image 20240715235811.png",
+    "RAG/RAG Retrieval Sources.md",
+    "LLM-Generated Content.md",
+    "Structured Data.md",
+    "Semi-Structured Data.md",
+    "Unstructured Data.md",
+    "img/Pasted image 20240715235013.png",
+    "LLMs/LLMs, chunking strategies.md",
+    "img/Pasted image 20240715234606.png",
     "RAG/RAG Using Llama 3.md",
-    "Pasted image 20240715222611.png",
+    "img/Pasted image 20240715222653.png",
+    "RAG/AI Agents Use Case.md",
     "RAG/Systematic RAG Workflow.md",
-    "Pasted image 20240715222528.png",
     "LLMs/Agent LLMs Calls.md",
-    "Pasted image 20240715222340.png",
     "RAG/Text & Knowledge Graph Embeddings.md",
-    "Pasted image 20240715222256.png",
     "RAG/Two-Stage Rretrieval System.md",
-    "Pasted image 20240715222219.png",
-    "RAG/Knowledge Graph RAG with SingleStore.md",
-    "Pasted image 20240715221947.png",
-    "NLP/Semantic Search.md",
-    "Nearest Neighbor Search.md",
-    "Vector Database.md",
-    "Text Embedding.md",
-    "Pasted image 20240715221527.png",
-    "LLMs/Developing an LLMs.md",
-    "RAG/RAG Enhancement Technique.md",
-    "LLMs/LLMs agent.md",
-    "LLMs/Enhancing LLMs.md",
-    "LLMs/Components of an LLMs.md",
-    "Pasted image 20240715221155.png",
-    "RAG/Types of Embedding Models for RAG.md",
-    "Pasted image 20240715220454.png",
-    "RAG/Advanced RAG Techniques.md",
-    "Pasted image 20240715220423.png",
-    "RAG/Modular RAG Framework.md",
-    "LLMs/Self-Learning LLMs.md",
-    "LLMs/How LLMs Are Built.md",
-    "RAG/Improving RAG Pipeline.md",
-    "RAG/Adaptive RAG.md",
-    "RAG/Multmodal RAG.md",
-    "RAG/Agentic RAG.md",
     "LangChain",
     "LLMs",
     "NLP",

LLMs/LLMs APP.md

@@ -0,0 +1 @@
+![[Pasted image 20240716000730.png]]

LLMs/LLMs, chunking strategies.md

@@ -0,0 +1,4 @@
+![[Pasted image 20240715234606.png]]
+
+[[LLMs, chunking strategies]]
+

RAG/Advanced RAG LimaIndex & Claude 3.md

@@ -0,0 +1 @@
+![[Pasted image 20240716000446.png]]

RAG/RAG Re-Ranking.md

@@ -0,0 +1,23 @@
+![[Pasted image 20240716000853.png]]
+
+LLMs can acquire new information in at least two ways:  
+1. Fine-tuning  
+2. RAG (retrieval augmented generation)  
+
+Retrieval-augmented generation (RAG) is the practice of extending the “memory” or knowledge of LLM by providing access to information from an external data source.  
+
+Traditional semantic search consists of a two-part process.  
+First, an initial retrieval mechanism does an approximate sweep over a collection of documents and creates a document list.  
+
+Then, a re-ranker mechanism will take this candidate document list and re-rank the elements. With [[Rerank]], we can improve your models by re-organizing your results based on certain parameters.  
+
+Why is Re-Ranking Required ?  
+⮕ The recall performance for LLMs decreases as we add more context resulting in increased context window(context stuffing)  
+
+⮕ Basic Idea behind reranking is to filter down the total number of documents into a fixed number .  
+
+⮕ The re-ranker will re-rank the records and get the most relevant items at the top and they can be sent to the LLM  
+
+⮕ The Reranking offers a solution by finding those records that may not be within the top 3 results and put them into a smaller set of results that can be further fed into the LLM  
+
+Reranking basically enhance the relevance and precision of retrieved results.

RAG/RAG Retrieval Sources.md

@@ -0,0 +1,11 @@
+![[Pasted image 20240715235013.png]]
+
+⮕ [[Unstructured Data]] (Text): This includes plain text documents, web pages, and other free-form textual sources.  
+
+⮕ [[Semi-Structured Data]] (PDF): PDF documents, such as research papers, reports, and manuals, contain a mix of textual and structural information.  
+
+⮕ [[Structured Data]] (Knowledge Graphs): Knowledge graphs, such as Wikipedia and Freebase, represent information in a structured and interconnected format.  
+
+⮕ [[LLM-Generated Content]]: Recent advancements have shown that LLMs themselves can generate high-quality content that can be used as a retrieval source. This approach leverages the knowledge captured within the LLM's parameters to generate relevant information.  
+
+All this data gets converted into embeddings and gets stored in a vector database. When a user query comes in, it also gets converted into an embedding (query embedding) and the most relevant answer will be retrieved using semantic search. The vector database becomes knowledge base to search for the contextually relevant answer.

RAG/RAG Stack.md

@@ -0,0 +1,18 @@
+![[Pasted image 20240716000053.png]]
+
+But to make RAG work perfectly, here are some key points to consider:  
+1. Quality of External Knowledge Source  
+
+2. [[Embedding Mode]]l: The choice of the embedding model used for retrieving relevant documents.  
+
+3. [[Chunking]] Size & Retrieval Strategy: Experiment with different chunk sizes to find the optimal length for context retrieval.  
+
+4. Integration with Language Model: The way the retrieved information is integrated with the language model's generation process is crucial.  
+
+5. Evaluation & Fine-tuning: Evaluating the performance of the RAG model on relevant datasets and tasks is important to identify areas for improvement.  
+
+6. Ethical Considerations: Ensure that the external knowledge source is unbiased and does not contain offensive or misleading information.  
+
+7. Vector database: Having a vector database that supports fast ingestion, retrieval performance, hybrid search is utmost important.  
+
+8. LLM models: Consider LLM models that are robust and fast enough to build your RAG application.

RAG/RAG implementing.md

@@ -0,0 +1,15 @@
+![[Pasted image 20240715235811.png]]
+
+Retrieval component, Augmentation component, Generation component.  
+
+⮕ [[Retrieval]]: This component helps you fetch the relevant information from the external knowledge base like a vector database for any given user query. This component is very crucial as this is the first step in curating the meaningful and contextually correct responses.  
+
+⮕ [[Augmentation]]: This part involves enhancing and adding more relevant context to the retrieved response for the user query.  
+
+⮕ [[Generation]]: Finally, a final output is presented to the user with the help of a large language model (LLM). The LLM uses its own knowledge and the provided context and comes up with an apt response to the user’s query.  
+
+These three components are the basis of a RAG pipeline to help users to get the contextually-rich and accurate responses they are looking for. That is the reason why RAG is so special when it comes to building chatbots, question-answering systems, etc.  
+
+Along with RAG, AI frameworks like LangChain and LlamaIndex make the LLM-applications more efficient by providing the required toolkit.  
+
+Let’s build a simple AI application that can fetch the contextually relevant information from our own data for any given user query.

RAG/RAG vs FineTuning.md

@@ -0,0 +1 @@
+![[Pasted image 20240716000624.png]]

RAG/Text Splitting RAG.md

@@ -0,0 +1 @@
+![[Pasted image 20240716000406.png]]