Upload helper.py

helper.py

@@ -42,8 +42,8 @@ llm = HuggingFacePipeline(pipeline=pipe)
 # # Initialize instructor embeddings using the Hugging Face model
 # instructor_embeddings = HuggingFaceInstructEmbeddings(model_name="C:/Users/arasu/Workspace/Projects/GenAI/embeddings/hkunlp_instructor-large")
 instructor_embeddings = HuggingFaceInstructEmbeddings(model_name="hkunlp/instructor-large")
+# db_path = "vector_db"
 vector_db = ""
-
 def create_vector_db():
     # Load data from pdf
     raw_text = ""
@@ -53,14 +53,13 @@ def create_vector_db():
         chunk_overlap  = 100,
         length_function = len,
     )
-    for root, dirs, files in os.walk("docs"):        
-        for file in files:
-            if file.endswith(".pdf"):
-                pdf = PdfReader("./docs/"+file)
-                for i, page in enumerate(pdf.pages):
-                    content = page.extract_text()
-                    if content:
-                        raw_text += content
+    from PyPDF2 import PdfReader
+    pdf = PdfReader("employment-agreement2018.pdf")
+    raw_text = ""
+    for i, page in enumerate(pdf.pages):
+        content = page.extract_text()
+        if content:
+            raw_text += content
     texts = text_splitter.split_text(raw_text)
 
     # Create a  vector database from 'text'