requirements.txt

# Core dependencies
numpy>=1.21.0
pandas>=1.3.0
tqdm>=4.65.0
python-dotenv>=0.19.0
pathlib>=1.0.1

# Web scraping and HTTP
requests>=2.28.0
aiohttp>=3.8.0
beautifulsoup4>=4.9.3
urllib3>=1.26.0

# Text processing and NLP
nltk>=3.6.0
textacy>=0.12.0
PyPDF2>=3.0.0
python-docx>=0.8.11
pytesseract>=0.3.8
Pillow>=9.0.0
PyMuPDF>=1.19.0  # fitz

# Embeddings and ML
torch>=1.9.0
transformers>=4.15.0
sentence-transformers>=2.2.0
fastembed>=0.1.0
voyageai>=0.1.0

# Vector Database
pymilvus>=2.2.0

# File handling and concurrency
filelock>=3.0.12
asyncio>=3.4.3

# Google AI
google-generativeai>=0.1.0

# Development and testing
pytest>=7.0.0
black>=22.0.0
flake8>=4.0.0 
luigi