Update Ingest.py

Ingest.py

@@ -17,9 +17,21 @@ logging.info("Loading documents...")
 loader = DirectoryLoader('data', glob="./*.txt")
 documents = loader.load()
 
+# Extract text from documents and split into manageable texts with logging
+#logging.info("Extracting and splitting texts from documents...")
+#text_splitter = RecursiveCharacterTextSplitter(chunk_size=1024, chunk_overlap=200)
+#texts = []
+#for document in documents:
+#    if hasattr(document, 'get_text'):
+#        text_content = document.get_text()  # Adjust according to actual method
+#    else:
+#        text_content = ""  # Default to empty string if no text method is available
+#
+#    texts.extend(text_splitter.split_text(text_content))
+
+
 # Extract text from documents and split into manageable texts with logging
 logging.info("Extracting and splitting texts from documents...")
-text_splitter = RecursiveCharacterTextSplitter(chunk_size=1024, chunk_overlap=200)
 texts = []
 for document in documents:
     if hasattr(document, 'get_text'):
@@ -27,13 +39,33 @@ for document in documents:
     else:
         text_content = ""  # Default to empty string if no text method is available
 
-    texts.extend(text_splitter.split_text(text_content))
+    # Check if text_content is valid before splitting
+    if text_content and isinstance(text_content, str):
+        valid_chunks = text_splitter.split_text(text_content)
+        texts.extend(valid_chunks)
+    else:
+        logging.warning(f"Invalid document or empty content encountered: {document}")
+
+
+
+
+# Define embedding function
+#def embedding_function(text):
+#    embeddings_model = HuggingFaceEmbeddings(model_name="law-ai/InLegalBERT")
+#    return embeddings_model.embed_query(text)
 
 # Define embedding function
 def embedding_function(text):
     embeddings_model = HuggingFaceEmbeddings(model_name="law-ai/InLegalBERT")
+    
+    # Ensure input is valid
+    if not text or not isinstance(text, str):
+        raise ValueError(f"Invalid text for embedding: {text}")
+    
     return embeddings_model.embed_query(text)
 
+
+
 # Create FAISS index for embeddings
 index = IndexFlatL2(768)  # Dimension of embeddings, adjust as needed
 
@@ -45,10 +77,25 @@ index_to_docstore_id = {i: i for i in range(len(texts))}
 faiss_db = FAISS(embedding_function, index, docstore, index_to_docstore_id)
 
 # Process and store embeddings
+#logging.info("Storing embeddings in FAISS...")
+#for i, text in enumerate(texts):
+#    embedding = embedding_function(text)
+#    faiss_db.add_documents([embedding])
+
+
+# Store embeddings in FAISS
 logging.info("Storing embeddings in FAISS...")
 for i, text in enumerate(texts):
-    embedding = embedding_function(text)
-    faiss_db.add_documents([embedding])
+    try:
+        if text:  # Check that the text is not None or empty
+            embedding = embedding_function(text)
+            faiss_db.add_documents([embedding])
+        else:
+            logging.warning(f"Skipping invalid or empty text at index {i}.")
+    except Exception as e:
+        logging.error(f"Error while processing text at index {i}: {text}, Error: {e}")
+
+
 
 # Exporting the vector embeddings database with logging
 logging.info("Exporting the vector embeddings database...")
@@ -58,4 +105,4 @@ faiss_db.save_local("ipc_embed_db")
 logging.info("Process completed successfully.")
 
 # Shutdown Ray after the process
-ray.shutdown()
+ray.shutdown()