
    jhWO                     R   d dl mZ d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z
 ej                  e      Zd dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZ ddlmZ ddlm Z  d dl!m"Z" d dlmZ d dl#m$Z$ ed        Z%d dl&m'Z' ed        Z(ed        Z)d Z*y)    )shared_taskN)MadeInChinaScraper)crawl_products_new)h_to_x)async_to_sync)datetime)settings)File)get_channel_layer   )ProductScrapeTask)ProductScrapeTaskSerializer)timezone)extract_product_infoc           	      v	   t               }d| }		 t        j                  d| d| d       t        j                         j
                  d d dz   }
t        j                         j
                  d d dz   }t        j                  d       t        j                  d	       t        j                  j                  t        j                  d
      }t        j                  |d       t        j                  j                  ||      }t        j                  j                  ||
      }t        | |
||       t        |
|||||       t!        |dd      5 }t#        t%        j&                  |            }t)        |      dk  rt        j+                  d       t,        j.                  j1                  |      j3                         }d| d}|r7d|_        ||_        t9        j:                         |_        |j?                          t        j@                  |       t        j@                  |       d|tC        |      jD                  d} tG        |jH                        |	d||d       dddcd d d        S 	 d d d        t        j                  d| d|        t!        |d      5 }t,        j.                  jK                  |      }|jL                  j?                  |tO        |             d|_        d|_        t9        j:                         |_        |j?                          d d d        t        j@                  |       t        j@                  |       t        j                  d | d!       d"}d|tC              jD                  d} tG        |jH                        |	d||d       dd#tQ        |jR                        d$S # 1 sw Y   MxY w# 1 sw Y   xY w# tT        $ r}t        jW                  d%| d&tQ        |       d'       t,        j.                  j1                  |      j3                         }|kd(|_        t9        j:                         |_        |j?                          d)}d*|tC        |      jD                  d} tG        |jH                        |	d||d       d+tQ        |      dcY d }~S d }~ww xY w),Nuser_zStarted scraping:  (Task ID: )   z.json.csv   zStarting async crawl...output_filesTexist_ok)	html_filecsv_filenamenum_productsscrape_task_iduser_idskip_in_pagerutf-8)encodingr   z"CSV is empty. No products matched.idz
                "Your search 'z' did not match any products. You may consider to:
                        - Check the spelling  
                        - Use fewer keywords  
                        - Use different keywords
                	COMPLETEDscrapper_completeeventmessagepayloadtyper*   data3No Products found with the given search parameters.successr*   zSaving files: HTML=z, CSV=rbzScrapping completed sucessfullyz!Files removed after saving. Task z
 complete.zTask Completed Successfully !Products Scrapped Successfullyr1   r*   r   zError in scraping task : exc_infoFAILEDScraping Failed!...scrapper_failedF),r   loggerinfouuiduuid4hextimesleepospathjoinr	   
MEDIA_ROOTmakedirsr   r   openlistcsvreaderlenwarningr   objectsfilterfirststatusr*   r   nowcompleted_atsaveremover   r.   r   
group_sendgetresult_filer
   strr%   	Exceptionerror)	final_urlsearch_namer   r   	num_pages
start_pager   r    channel_layer
group_namehtml_filenamer   
output_dircsv_file_pathhtml_file_pathfrJ   scrape_taskr*   r.   es                        $/var/www/Befach/backend/bot/tasks.pyrun_alibaba_scraperri      sV    &'M	"Jn5([@PPQRS

(("-7zz|'',v5

1-.WW\\("5"5~F

J.Z>j-@ 	9mY
K\P\m{  FM  \h  	i -w7 "	1#**Q-(F6{aCD/77>>.>QWWY*m , )4K&*1K'/7||~K,$$&		-(		.)/%9+FKK
 8m667 3#* $  $T?"	 "	"	H 	).)9OP-& 	!+3377! 8 K ##((tAw?!,K"CK'/||~K$	 			- 
		.!77GzRS1'1+>CC

 	0m../+"	
  -K`cdodrdr`sttC"	 "	J	 	<  5.~.>bQITXY'//66! 7 

%' 	 "!)K'/||~K$+G)!5kBGGD
 4M-223/&  !SV4415sX   E O DN='	O 2/O !A=O
BO =OO 
OO 	R8CR3-R83R8)save_images_to_uploadthingc                 B   t               }d| }	 t        j                  d|  d| d       t        j                         j
                  d d dz   }t        j                  j                  t        j                  d      }	t        j                  |	d	       t        j                  j                  |	|      }
t               }t        j                  |j                  | d
|||            }|st        j!                  d       t"        j$                  j'                  |      j)                         }d|  d}|r7d|_        ||_        t/        j0                         |_        |j5                          d|t7        |      j8                  d} t;        |j<                        |d||d       dddS g d}g }g }|d | D ]/  }	 t        j                         j
                  d d }|j?                  dg       }|jA                  ||d       |j?                  dd      }|j?                  dd      }|j?                  dd      }|j?                  dd      }|j?                  dd      }|j?                  dd      }|j?                  d d      }|j?                  d!d      }|j?                  d"d      }|j?                  d#d      }|j?                  d$i       } |j?                  d d      }!|j?                  d%d&      }"|j?                  d'd(      }#|j?                  d)i       }$|j?                  d*i       }%|j?                  d+i       }&|j?                  d,g       }'|j?                  d-i       }(|jA                  ||||d|||||||tC        jD                  |$d./      tC        jD                  |%d./      tC        jD                  |&d./      tC        jD                  |'d./      tC        jD                  |(d./      |!|"|#g       2 tM        |
d1d(d23      5 }*tO        jP                  |*      }+|+jS                  |       |+jU                  |       d d d        tM        |
d4      5 }*t"        j$                  j?                  |      }|jV                  j5                  |tY        |*             d|_        d5|_        t/        j0                         |_        |j5                          d d d        t        jZ                  j]                         d6v rt_        j`                  |||       t        jb                  |
       d7}d|t7              j8                  d} t;        |j<                        |d||d       dd8tK        |jd                        d9S # tF        $ r,})t        jI                  d0tK        |)              Y d })~)d })~)ww xY w# 1 sw Y   sxY w# 1 sw Y   xY w# tF        $ r})t        jI                  d:| d;tK        |)       d<       t"        j$                  j'                  |      j)                         }|kd=|_        t/        j0                         |_        |j5                          d>}d?|t7        |      j8                  d} t;        |j<                        |d||d       d.tK        |)      dcY d })~)S d })~)ww xY w)@Nr   Started scraping MadeInChina: r   r   r   r   r   Tr   r   r^   r   r    filters No products found on MadeInChinar$   
            Your search '' or filters did not match any products on MadeInChina. You may consider to:
                    - Check the spelling  
                    - Use fewer keywords
                    - Use different keywords
            r&   r'   r(   r,   r/   r0   )IDSKUURLTitle	Image URLDescriptionPriceMOQCompanyRatingTypeCategory
AttributesFeaturesSample InfozRange PriceVariantsz
Brand NameUnitMXQimages
image_urls
product_idskuN/Alinknamedescriptionpricemoqcompanyr{   r-   
categoriesvariantunitpiecemxq 
properties
basic_infor   zprice rangesvariantsF)ensure_asciiError processing product: wr"   newliner#   r2    Scrapping completed successfully)yestrue1Task Completed Successfully!r3   r4   #Error in MadeInChina scraping task r5   r6   r8   r9   r:   )3r   r;   r<   r=   r>   r?   rB   rC   rD   r	   rE   rF   r   asynciorunscrape_made_in_chinarL   r   rM   rN   rO   rP   r*   r   rQ   rR   rS   r   r.   r   rU   rV   appendjsondumpsrY   rZ   rX   rG   rI   writerwriterow	writerowsrW   r
   SCRAPE_IMAGElowerrj   delayrT   r%   ),r\   r   r   r   r    rn   r_   r`   r   rb   rc   scraperproductsrf   r*   r.   csv_headerscsv_dataimages_tasksproductr   r   r   urltitler   r   r   r   rating	prod_typecategoryr   brandr   r   
attributesfeaturessample_infoprice_rangesr   rg   re   r   s,                                               rh   run_made_in_china_scraperr      s>    &'M	"JA54[M^L\\]^_zz|'',v5WW\\("5"5~F

J.Z> %& ;;(()) ) 
 NN=>+33::n:MSSUK% 'G %0"&-#+3<<>(  " ,!5kBGGD
 4M-223/&   P 
 .G:!ZZ\--cr2
 Xr2##"(",%  kk%/kk&%0FE2%kk-?GU3kk%/!++i7 Xu5#KK6	";;|U;!++i4Iu5{{673kk%, %[[r:
";;|R8%kk-<&{{>2>";;z26 JJz>JJxe<JJ{?JJ|%@JJxe<)! E /| -b7C 	'qZZ]FOOK(X&	' -& 	!+3377>7JK##((tAw?!,K"DK'/||~K$	   &&(,>>&,,^\7S 			-  1'1+>CC

 	0m../+"	
  ,L`cdodrdr`sttU  9#a&BC
	' 	'	 	B  5:>:J"SQRVHU`de'//66.6IOOQ"!)K'/||~K$+G)!5kBGGD
 4M-223/&  !SV44/5s   FV< +V< =H+U+(V< :8V#2V< A=V0B'V< +	V 4!VV< V  V< #V-(V< 0V95V< <	ZCZZZc                 *   t               }d| }	 t        j                  d|  d| d       t        j                         j
                  d d dz   }t        j                  j                  t        j                  d      }	t        j                  |	d	       t        j                  j                  |	|      }
t               }t        j                  |j                  | d
|||            }|st        j!                  d       t"        j$                  j'                  |      j)                         }d|  d}|r7d|_        ||_        t/        j0                         |_        |j5                          d|t7        |      j8                  d} t;        |j<                        |d||d       dddS g d}g }g }|d | D ]=  }	 t        j                         j
                  d d }|j?                  dg       }|jA                  ||d       |j?                  di       }g }|r7|jC                  |jE                         D cg c]  \  }}| d|  c}}       |j?                  d      r|jA                  |d          |rdj                  |      nd}|jA                  ||j?                  dd      |j?                  dd      |d|j?                  d d      |j?                  d!d      |j?                  d"d      ddg
       @ tM        |
d$d%d&'      5 }tO        jP                  |      }|jS                  |       |jU                  |       d d d        tM        |
d(      5 }t"        j$                  j?                  |      }|jV                  j5                  |tY        |             d|_        d)|_        t/        j0                         |_        |j5                          d d d        t[        j\                  |||       t        j^                  |
       d*}d|t7              j8                  d} t;        |j<                        |d||d       dd+tK        |j`                        d,S c c}}w # tF        $ r,}t        jI                  d#tK        |              Y d }~d }~ww xY w# 1 sw Y   YxY w# 1 sw Y   xY w# tF        $ r}t        jI                  d-| dtK        |       d.       t"        j$                  j'                  |      j)                         }|kd/|_        t/        j0                         |_        |j5                          d0}d1|t7        |      j8                  d} t;        |j<                        |d||d       d2tK        |      dcY d }~S d }~ww xY w)3Nr   rl   r   r   r   r   r   Tr   r   rm   ro   r$   rp   rq   r&   r'   r(   r,   r/   r0   )rr   ru   rt   rw   rv   zUploadThings URLrx   ry   rz   zSupplier Infor{   r   r   r   r5   r   
r   r   r   r   r   r   r   r   r   r"   r   r2   r   r   r3   r4   r   r6   r8   r9   r:   F)1r   r;   r<   r=   r>   r?   rB   rC   rD   r	   rE   rF   r   r   r   r   rL   r   rM   rN   rO   rP   r*   r   rQ   rR   rS   r   r.   r   rU   rV   r   extenditemsrY   rZ   rX   rG   rI   r   r   r   rW   r
   rj   r   rT   r%   )r\   r   r   r   r    rn   r_   r`   r   rb   rc   r   r   rf   r*   r.   r   r   r   r   r   r   r   description_partskvr   rg   re   r   s                                 rh   run_1688_scrapperr   [  sY    &'M	"Ji54[M^L\\]^_zz|'',v5WW\\("5"5~F

J.Z> %& ;;(()) ) 
 NN=>+33::n:MSSUK% 'G %0"&-#+3<<>(  " ,!5kBGGD
 4M-223/&   P 

 .G$!ZZ\--cr2
 Xr2 ##"(",%  %[[r:
$&!%,,jFVFVFX-Yda2aSk-YZ;;}-%,,W]-CD>Odii(9:UZ KK.KK.KK/KKu-KK	51! - /P -b7C 	'qZZ]FOOK(X&	' -& 	!+3377>7JK##((tAw?!,K"DK'/||~K$	 	#((wO 			-  1'1+>CC

 	0m../+"	
  ,L`cdodrdr`stt{ .Z(  9#a&BC
	' 	'	 	@  5:>:J"SQRVHU`de'//66.6IOOQ"!)K'/||~K$+G)!5kBGGD
 4M-223/&  !SV44/5s   FR0 +R0 =A;Q8Q	B-Q6R0 8R R0 A=R$BR0 Q	R(!R	R0 RR0 R!R0 $R-)R0 0	V9CVVVc                 0    t        j                  | |       y )N)rj   r   )task_idr   s     rh   scrape_imagesr     s    $$Wl;    )+celeryr   r=   rB   r   r@   loggingrI   r   pandaspd	getLogger__name__r;   bot.china.made_in_chinar   bot.utils.scraper_utilsr   !bot.alibaba.products_page_scraperr   asgiref.syncr   r   django.confr	   django.core.filesr
   channels.layersr   modelsr   serializersr   django.utilsr   bot.utils.scrape_detailsr   ri   bot.utils.scrape_imagerj   r   r   r    r   rh   <module>r      s         
  			8	$ 7 6 4 &    " - % 4 !   9r5 r5h >E5 E5P m5 m5^<r   