import argparse
from crawl4ai import AsyncWebCrawler, CacheMode, CrawlerRunConfig, LXMLWebScrapingStrategy
from crawl4ai.deep_crawling import BestFirstCrawlingStrategy
from crawl4ai.deep_crawling.scorers import KeywordRelevanceScorer
import asyncio


async def main():
    # Parse command line arguments
    parser = argparse.ArgumentParser(description='Test iframe processing with output directory')
    parser.add_argument('--url', default="https://www.adhirainternationalschool.co.in/cbse-curriculum.php", help='URL to crawl')
    parser.add_argument('--max-pages', type=int, default=1, help='Maximum number of pages to crawl')
    args = parser.parse_args()
    
    print(f"Crawling URL: {args.url}")
    

    # Create a scorer
    keyword_scorer = KeywordRelevanceScorer(
        keywords=["school", "fee", "information", "student", "admission", "sports"],
        weight=0.7
    )

    # Configure the strategy
    best_strategy = BestFirstCrawlingStrategy(
        max_depth=2,
        include_external=False,
        url_scorer=keyword_scorer,
        max_pages=args.max_pages,            
    )

    config = CrawlerRunConfig(
            deep_crawl_strategy=best_strategy,
            scraping_strategy=LXMLWebScrapingStrategy(),
            cache_mode=CacheMode.BYPASS,
            stream=True,
            remove_forms=False,
            process_iframes=False,
            scan_full_page=True,
            wait_for_images=False,
            exclude_all_images=True,
            wait_until="domcontentloaded",
        )
    

    async with AsyncWebCrawler(verbose=True, headless=False) as crawler:
        async for result in await crawler.arun(args.url, config=config):        
            print(f"Links: {result.links['internal']}")

if __name__ == "__main__":
    asyncio.run(main())