# Example usage of our RAG pipelinefrom rag_pipeline import RAGPipeline, FileConfig# Initialize with smart defaultspipeline = RAGPipeline(    persist_directory="./chroma_db",    collection_name="enterprise_docs",    config=FileConfig(        chunk_size=1000,        chunk_overlap=200,        whisper_model_size="base"    ))# Process entire directories of mixed contentresult = pipeline.process_directory("./company_data")

# Example queryresults = pipeline.db_manager.query(    collection_name="enterprise_docs",    query_texts=["What were our key achievements in Q4?"],    n_results=3)

pip install -r requirements.txt

# Core dependencieschromadb>=0.4.0langchain>=0.1.0pandas>=1.5.0numpy>=1.24.0sentence-transformers>=2.2.0
# Document processingpypdf>=3.0.0python-docx>=0.8.11openpyxl>=3.1.0python-pptx>=0.6.21beautifulsoup4>=4.12.0markdown>=3.4.0lxml>=4.9.0
# Media processingopenai-whisper>=20231117moviepy>=1.0.3pydub>=0.25.1torch>=2.0.0
# Optional but recommendedtqdm>=4.65.0python-magic>=0.4.27pyyaml>=6.0.0

def process_file(self, file_path: str) -> List[Document]:    """    Process a single file and return chunks with metadata.        Args:        file_path (str): Path to the file to process            Returns:        List[Document]: List of processed document chunks            Raises:        ValueError: If file type is not supported    """

class DocumentProcessor:    def __init__(self, config: FileConfig):        self.config = config        self.text_splitter = RecursiveCharacterTextSplitter(            chunk_size=config.chunk_size,            chunk_overlap=config.chunk_overlap        )
    def process_file(self, file_path: str) -> List[LangchainDocument]:        """Process a single file into searchable chunks"""        # Extract text based on file type        text = self._extract_text(file_path)                # Split into manageable chunks        chunks = self.text_splitter.split_text(text)                # Add metadata for better retrieval        return [            LangchainDocument(                page_content=chunk,                metadata={                    "source": file_path,                    "chunk_index": i,                    "processed_date": datetime.now().isoformat()                }            )            for i, chunk in enumerate(chunks)        ]

class ChromaDBManager:    def __init__(self, persist_directory: str):        self.client = chromadb.PersistentClient(path=persist_directory)        self.embedding_function = embedding_functions.DefaultEmbeddingFunction()
    def add_documents(self, collection_name: str, documents: List[LangchainDocument]):        """Store documents with their vector embeddings"""        collection = self.get_or_create_collection(collection_name)                # Prepare documents for storage        docs = [doc.page_content for doc in documents]        metadatas = [doc.metadata for doc in documents]        ids = [f"{doc.metadata['file_hash']}_{doc.metadata['chunk_index']}"               for doc in documents]
        # Store in ChromaDB        collection.add(            documents=docs,            metadatas=metadatas,            ids=ids        )

def query_database(self, query: str, n_results: int = 3) -> Dict:    """Execute a semantic search query"""    try:        results = self.collection.query(            query_texts=[query],            n_results=n_results        )        return results    except Exception as e:        self.logger.error(f"Error querying database: {str(e)}")        return None

# Traditional Searchresults = database.find({"$text": {"$search": "revenue growth 2023"}})
# RAG Searchresults = rag_pipeline.query(    "How did our revenue grow in 2023 compared to previous years?")

class MediaProcessor:    def __init__(self, config: FileConfig):        self.whisper_model = whisper.load_model(            config.whisper_model_size,            device=config.device        )
    def process_media(self, file_path: str) -> str:        """Process audio and video files"""        file_ext = Path(file_path).suffix.lower()                # Handle video files        if file_ext in ['.mp4', '.avi', '.mov']:            audio_path = self.extract_audio(file_path)            return self.transcribe_audio(audio_path)                    # Handle audio files        elif file_ext in ['.mp3', '.wav', '.m4a']:            return self.transcribe_audio(file_path)