
    h]              
          d dl Z d dlmZ d dlmZ d dlmZ d dlZd dlZd dlZej                  j                  ej                  j                  ej                  j                  ej                  j                  e                         d dlmZ d dlmZmZ d dlmZ d dlmZmZmZ  e        d	ed
edededef
dZd Zedk(  r e j<                   e              yy)    N)BrowserConfig)AsyncWebCrawler)load_dotenv)h_to_x)CSS_SELECTORREQUIRED_KEYS)save_products_to_csv)fetch_and_process_pageget_browser_configget_llm_strategyurlsearch_namecsv_filenamehtml_filenamenum_productsc                 H  K   t               }t        di |}t               }d}d}	g }
t               }t	        |      4 d{   }	 t        ||	| t        ||t        ||	       d{   \  }}|rt        d       n=|
j                  |       |	dz  }	|	|dz   k(  rnt        j                  d       d{    rddd      d{    |
r(t        |
|        t        dt        |
       d	| d
       |j                          y7 7 7 T7 E# 1 d{  7  sw Y   UxY ww)z=
    Main function to crawl venue data from the website.
    alibaba_crawl_session   )browser_configN)r   z%No more products found. Ending crawl.
   zSaved z products to 'z'. )r   r   r   setr   r
   r   r   printextendasynciosleepr	   len
show_usage)r   r   r   r   r   browser_config_dictr   llm_strategy
session_idpage_numberall_products
seen_namescrawlerproductsno_results_founds                  #/var/www/Befach/backend/bot/main.pycrawl_productsr)      sO    
 -."9%89N#%L(J KLJ %
 !$ !$/E+
0 
*&H&  => )1Kl1n, --###= !$ !$F \l^=s<()~RHI
 U!$

*8 $C!$ !$ !$ !$sn   A D"DD"!D'D(AD4D	5D:D"DA D"D	DD"DDDD"c                  4   K   t                d{    y7 w)z$
    Entry point of the script.
    N)r)   r       r(   mainr,   T   s      
s   __main__)r   crawl4air   r   dotenvr   sysosuuidpathappenddirnameabspath__file__!bot.alibaba.products_page_scraperr   
bot.configr   r   bot.utils.data_utilsr	   bot.utils.scraper_utilsr
   r   r   strintr)   r,   __name__runr   r+   r(   <module>r@      s     " $  
 	  0I JK L 4 3  ;c ; ;3 ;WZ ;jm ;| zGKK r+   