o
    h]                  
   @   s   d dl Z d dlmZ d dlmZ d dlmZ d dlZd dlZd dlZej	
ej	ej	ej	e d dlmZ d dlmZmZ d dlmZ d dlmZmZmZ e  d	ed
edededef
ddZdd Zedkrte e  dS dS )    N)BrowserConfig)AsyncWebCrawler)load_dotenv)h_to_x)CSS_SELECTORREQUIRED_KEYS)save_products_to_csv)fetch_and_process_pageget_browser_configget_llm_strategyurlsearch_namecsv_filenamehtml_filenamenum_productsc                    s  t  }tdi |}t }d}d}	g }
t }t|d4 I dH >}	 t||	| t||t||d	I dH \}}|r;td n|
	| |	d7 }	|	|d krKn	t
dI dH  q"W d  I dH  n1 I dH sdw   Y  |
r~t|
|  td	t|
 d
| d |  dS )z=
    Main function to crawl venue data from the website.
    alibaba_crawl_session   )browser_configNT)r   z%No more products found. Ending crawl.
   zSaved z products to 'z'. )r
   r   r   setr   r	   r   r   printextendasynciosleepr   len
show_usage)r   r   r   r   r   browser_config_dictr   llm_strategy
session_idpage_numberall_products
seen_namescrawlerproductsno_results_foundr   r   #/var/www/Befach/backend/bot/main.pycrawl_products   sL   
(#r'   c                      s   t  I dH  dS )z$
    Entry point of the script.
    N)r'   r   r   r   r&   mainT   s   r(   __main__)r   crawl4air   r   dotenvr   sysosuuidpathappenddirnameabspath__file__!bot.alibaba.products_page_scraperr   
bot.configr   r   bot.utils.data_utilsr   bot.utils.scraper_utilsr	   r
   r   strintr'   r(   __name__runr   r   r   r&   <module>   s$    $>