
    [Ah#                         d dl mZ d dlZd dlZd dlZd dlZd dlZd dlmZ d dl	m
Z
 d dlmZ d dlmZmZ deded	ed
efdZy)    )BeautifulSoupN)extract_query_name)settings)extract_product_info)scrap_detail_pagesave_images_to_uploadthing	html_filecsv_filenamenum_productsskip_in_pagec                 `	   t         j                  j                  t        j                  d      }t        j
                  |d       t         j                  j                  ||      }t         j                  j                  ||       }t         j                  j                  |      st        d| d       y t        |dd      5 }	t        j                  |	      }
d d d        
}d	}d	}g }g d
}g }||d  }|D ]  }	 ||k\  r n|j                  d      }t        j                  dd|      }d|j                  d      z   }t        |      }|s%t        d|       |dz  }|dk\  rt        d        nC{|r|j                  d      nd}t        j                  d      }|j!                  |      }|r|j#                  d      }|j                  d      }|j                  di       j                  d      }|j                  d      }|j                  d      }|j                  di       j                  d      }|j                  d      }|j                  d      } |j                  d      }!|j                  di       j                  d      }"|j                  di       j                  d       }#|}|j                  di       j                  d!      }$|j                  d"      }%|j                  di       j                  d#      }&	 |j                  di       j                  di       j                  d$      }'|j                  di       j                  d%      }(t%        j&                         j(                  d d& })|j+                  |(|)d'       |j+                  |)||d(|||||||| |!|"|#|$|%|&|'g       |dz  } t        |d*dd+      5 },t3        j4                  |,      }-|-j7                  |       |-j9                  |       d d d        t        j:                  j=                         d,v rt?        j@                  |||       t        d-tC        |       d.|        y # 1 sw Y   }xY w#  d}'Y 8xY w# t,        $ r-}*d	d l}+|+j1                          t        d)|*        Y d }*~*d }*~*ww xY w# 1 sw Y   xY w)/Noutput_filesT)exist_oku   [❌] HTML file z not found.rzutf-8)encodingr   )IDSKUURLTitlez	Image URLDescriptionPriceMOQCompanyRatingTypeCategory
AttributesFeatureszSample InfozRange PriceVariantsz
Brand NameUnitMXQtitlez<[^>]+> zhttps:
productUrlzResult not found   d   zBroken after 100 retriesdescriptionz_(\d+)\.htmlpriceproductmoqcompanyNameproductScoreproduct_typecategory
attributesfeaturessample_infoladder_pricevariants
brand_nameunitmaxQuantityimages   )
image_urls
product_idzN/Az [Error] Skipping a card due to: w)newliner   )yestrue1u   ✅ Extracted z products and saved to )"ospathjoinr   
MEDIA_ROOTmakedirsexistsprintopenjsonloadgetresubr   compilesearchgroupuuiduuid4hexappend	Exception	traceback	print_exccsvwriterwriterow	writerowsSCRAPE_IMAGElowerr   delaylen).r	   r
   r   scrape_task_iduser_idr   
output_dircsv_path	html_pathfilehtmlcards	processedretriedcsv_datacsv_headersimages_taskscard	raw_titler"   urlresultr'   patternmatchskur(   r*   companyratingr-   r.   r/   r0   r1   range_pricer3   r4   r5   mxqr7   r:   erU   frX   s.                                                 </var/www/Befach/backend/bot/alibaba/products_page_scraper.pyh_to_xrz      sG   h11>BJKK
T*ww||J5HZ3I77>>)$ ;78	iw	/ 4yy EIG HK
 L,- EH	:L()IFF:b)4ETXXl33C)#.F(11c>457=&**]32Kjj1GNN3'Ekk!n
 HHW%E**Y+//6C hh}-G XXn-F "::i377GLzz*-HL1Jzz*-H **Yr266}EK **Yr266~FK%Kzz)R044Z@HL1J::i,008DjjB/33M2FJJ=Y ZZ	2.228<F))#2.J#'! 
 OO3UE;sG,*hXz4 
 NIK X 
hR'	: #aA$"#
 ""$(::"((gN	N3x=/)@
OPU X  	:!4QC899	:# #sW   Q;Q+A*Q+1FQ+?1Q!0B Q+8R$Q!Q(%Q++	R!4"RR!$R-)bs4r   rW   rP   rK   r@   rH   bot.utils.data_utilsr   django.confr   bot.utils.scrape_detailsr   bot.utils.scrape_imager   r   strintrz        ry   <module>r      sO     
  	 	  3   9 PuQc uQ uQC uQhk uQr   