o jhWO@s.ddlmZddlZddlZddlZddlZddlZddlZddlZddl Z e e Z ddlmZddlmZddlmZddlmZddlmZddlmZdd lmZdd lmZd d lmZd d lm Z ddl!m"Z"ddlmZddl#m$Z$eddZ%ddl&m'Z'eddZ(eddZ)ddZ*dS)) shared_taskN)MadeInChinaScraper)crawl_products_new)h_to_x) async_to_sync)datetime)settings)File)get_channel_layer)ProductScrapeTask)ProductScrapeTaskSerializer)timezone)extract_product_infoc CsNt}d|} zCtd|d|dtjddd} tjddd} tdtd tj t j d } tj | d d tj | | } tj | | }t|| ||t| | ||||d t| dddf}tt|}t|dkrtdtjj|d}d|d}|rd|_||_t|_|t | t |d|t!|j"d}t#|j$| d||dd ddWdWSWdn1swYtd|d| t| d'}tjj%|d}|j&| t'|d|_d|_t|_|Wdn 1swYt | t |td |d!d"}d|t!|j"d}t#|j$| d||dd d#t(|j)d$WSt*y}zMtj+d%|d&t(|d d'tjj|d}|durd(|_t|_|d)}d*|t!|j"d}t#|j$| d||dd+t(|dWYd}~Sd}~ww),Nuser_zStarted scraping: (Task ID: )z.json.csvzStarting async crawl... output_filesTexist_ok) html_file csv_filename num_productsscrape_task_iduser_id skip_in_pagerutf-8)encodingr z"CSV is empty. No products matched.idz "Your search 'z' did not match any products. You may consider to: - Check the spelling - Use fewer keywords - Use different keywords COMPLETEDscrapper_completeeventmessagepayloadtyper(data3No Products found with the given search parameters.successr(zSaving files: HTML=z, CSV=rbzScrapping completed sucessfullyz!Files removed after saving. Task z complete.zTask Completed Successfully !Products Scrapped Successfullyr/r(rzError in scraping task : exc_infoFAILEDScraping Failed!...scrapper_failedF),r loggerinfouuiduuid4hextimesleepospathjoinr MEDIA_ROOTmakedirsrropenlistcsvreaderlenwarningr objectsfilterfirststatusr(rnow completed_atsaveremover r,r group_sendget result_filer strr# Exceptionerror) final_url search_namerr num_pages start_pagerr channel_layer group_name html_filenamer output_dir csv_file_pathhtml_file_pathfrH scrape_taskr(r,erf$/var/www/Befach/backend/bot/tasks.pyrun_alibaba_scrapers         $      rh)save_images_to_uploadthingc,Cst}d|}ztd|d|dtjddd}tjt j d} tj | dd tj| |} t } t | j|d |||d } | std tjj|d } d|d}| rnd| _|| _t| _| d|t| jd}t|j|d||ddddWSgd}g}g}| d|D]}ztjdd}|dg}| ||d|dd}|dd}|dd}|dd}|dd}|dd}|d d}|d!d}|d"d}|d#d}|d$i} |d d}!|d%d&}"|d'd(}#|d)i}$|d*i}%|d+i}&|d,g}'|d-i}(| ||||d|||||||t!j"|$d.d/t!j"|%d.d/t!j"|&d.d/t!j"|'d.d/t!j"|(d.d/|!|"|#gWqt#ys})zt$d0t%|)WYd})~)qd})~)wwt&| d1d(d2d3}*t'(|*}+|+)||+*|Wdn 1swYt&| d4'}*tjj|d } | j+|t,|*d| _d5| _t| _| Wdn 1swYt j-.d6vrt/0|||t1| d7}d|t| jd}t|j|d||ddd8t%| j2d9WSt#y_})zMtj$d:|d;t%|)dd<tjj|d } | durNd=| _t| _| d>}d?|t| jd}t|j|d||dd.t%|)dWYd})~)Sd})~)ww)@NrStarted scraping MadeInChina: rrrrrTrr r\rrfilters No products found on MadeInChinar" Your search '' or filters did not match any products on MadeInChina. You may consider to: - Check the spelling - Use fewer keywords - Use different keywords r$r%r&r*r-r.)IDSKUURLTitle Image URL DescriptionPriceMOQCompanyRatingTypeCategory AttributesFeatures Sample Infoz Range PriceVariantsz Brand NameUnitMXQimages image_urls product_idskuN/Alinkname descriptionpricemoqcompanyryr+ categoriesvariantunitpiecemxq properties basic_infor~z price rangesvariantsF) ensure_asciiError processing product: wr newliner!r0 Scrapping completed successfully)yestrue1Task Completed Successfully!r1r2#Error in MadeInChina scraping task r3r4r6r7r8)3r r9r:r;r<r=r@rArBrrCrDrasynciorunscrape_made_in_chinarJr rKrLrMrNr(rrOrPrQr r,rrSrTappendjsondumpsrWrXrVrErGwriterwriterow writerowsrUr SCRAPE_IMAGElowerridelayrRr#),rZrrrrrlr]r^rr`rascraperproductsrdr(r, csv_headerscsv_data images_tasksproductrrrurltitlerrrrrating prod_typecategoryrbrandrr attributesfeatures sample_info price_rangesrrercrrfrfrgrun_made_in_china_scrapers.                                           rcCst}d|}ztd|d|dtjddd}tjt j d} tj | dd tj| |} t } t | j|d |||d } | std tjj|d } d|d}| rnd| _|| _t| _| d|t| jd}t|j|d||ddddWSgd}g}g}| d|D]}zgtjdd}|dg}| ||d|di}g}|r|!dd|"D|dr| |d|rd|nd}| ||dd|d d|d|d!d|d"d|d#dddg Wqt#y}zt$d$t%|WYd}~qd}~wwt&| d%d&d'd(}t'(|}|)||*|Wdn 1sAwYt&| d)'}tjj|d } | j+|t,|d| _d*| _t| _| Wdn 1svwYt-.|||t/| d+}d|t| jd}t|j|d||ddd,t%| j0d-WSt#y}zMtj$d.|d/t%|dd0tjj|d } | durd1| _t| _| d2}d3|t| jd}t|j|d||dd4t%|dWYd}~Sd}~ww)5NrrjrrrrrTrr rkrmr"rnror$r%r&r*r-r.) rprsrrrurtzUploadThings URLrvrwrxz Supplier InforyrrrcSsg|] \}}|d|qS)r3rf).0kvrfrfrg sz%run_1688_scrapper..r rrrrrrrrrr rr0rrr1r2rr3r4r6r7r8F)1r r9r:r;r<r=r@rArBrrCrDrrrrrJr rKrLrMrNr(rrOrPrQr r,rrSrTrextenditemsrWrXrVrErGrrrrUr rirrRr#)rZrrrrrlr]r^rr`rarrrdr(r,rrrrrrrdescription_partsrrercrrfrfrgrun_1688_scrapper[s                          rcCst||dS)N)rir)task_idrrfrfrg scrape_images sr)+celeryrr;r@rr>loggingrGrpandaspd getLogger__name__r9bot.china.made_in_chinarbot.utils.scraper_utilsr!bot.alibaba.products_page_scraperr asgiref.syncrr django.confrdjango.core.filesr channels.layersr modelsr serializersr django.utilsrbot.utils.scrape_detailsrrhbot.utils.scrape_imagerirrrrfrfrfrgs@                 t I  0