o h#@sBddlmZddlmZddlZddlZddlZddlZddlTddl Z ddl Z ddl m Z ejejeejeZeddZed d Zed d Zd dZeddZeddZed%defddZeddZeddZeddZeddZ d&d d!Z!d"d#Z"e#d$kre"dSdS)') lru_cache)PathN)*)MODEL_REPO_BRANCHcCs2ddl}|jdkr|j|jS|jdkrdSdS)Nrcudampsl0)torchtyperget_device_properties total_memory)devicer rR/var/www/Befach/backend/venv/lib/python3.10/site-packages/crawl4ai/model_loader.pyget_available_memory s   rcCsLt|}|jdkr dS|jdvr$|dkrdS|dkrdS|dkr"d Sd SdS) Ncpu)rrlll@ )rr )r available_memoryrrrcalculate_batch_sizes  rcCsHddl}|jr|d}|S|jjr|d}|S|d}|S)Nrrrr)r r is_availabler backendsr)r r rrr get_device)s    rcCst}||||fS)N)rto)modelr rrrset_model_device4s rcCsTtjtdtd}tj|ddtj|dddtj|ddd|S)NCRAWL4_AI_BASE_DIRECTORYz .crawl4aiT)exist_okz/cachez/models)ospathjoingetenvrhomemakedirs) home_folderrrrget_home_folder9s r(cCsPddlm}m}m}m}|jddd}|jddd}|t|\}}||fS)Nr BertTokenizer BertModel AutoTokenizer AutoModelzbert-base-uncasedresume_download transformersr*r+r,r-from_pretrainedevalr)r*r+r,r- tokenizerrr rrrload_bert_base_uncasedAs  r5BAAI/bge-small-en-v1.5returncCsPddlm}m}m}m}|j|dd}|j|dd}|t|\}}||fS)zLoad the Hugging Face model for embedding. Args: model_name (str, optional): The model name to load. Defaults to "BAAI/bge-small-en-v1.5". Returns: tuple: The tokenizer and model. rr)Nr.r0) model_namer*r+r,r-r4rr rrrload_HF_embedding_modelJs   r9cCs^ddlm}m}ddlm}ddl}|d}|d}|t|\}}|d||d}|S)Nr)r,"AutoModelForSequenceClassification)pipelinez1dstefa/roberta-base_topic_classification_nyt_newsztext-classification)rr4)r1r,r:r;r r2r3r)r,r:r;r r4rr piperrrload_text_classifier[s    r=csddlm}m}ddl}ddlmddld}|j|dd|j|ddt \j j d fdd }|fS) Nr)r:r,)expitzcardiffnlp/tweet-topic-21-multir.?rc s|ddd|d}fdd|D}d i|}Wdn1s*wY|j}|}||kd}g}|D]}fddt|D} || qF|S) NptT)return_tensorspadding truncation max_lengthcsi|] \}}||qSr)r).0keyval)r rr szHload_text_multilabel_classifier.._classifier..rcs g|] \}}|dkr|qS)rr)rEivalue) class_mappingrr s zHload_text_multilabel_classifier.._classifier..r)itemsno_gradlogitsdetachrnumpy enumerateappend) texts thresholdrDtokensoutputscores predictions batch_labels predictionlabelsrKr r>rr4r rr _classifiers   z4load_text_multilabel_classifier.._classifier)r?r) r1r:r,rQ scipy.specialr>r r2r3rconfigid2label)r:r,npMODELr^rr]rload_text_multilabel_classifierhs   rdcCsBddl}z|jdWnty|dYnw|jdS)Nrztokenizers/punktpunkt)nltkdatafind LookupErrordownload)rfrrrload_nltk_punkts  rkc Csddl}d}t}t||}|rt|sd}t}t|d}td|rXzt ||r:t |Wnt yWtdtd|td|YdSwz7t j dd d ||t |gt jt jd d t|d }|jd d d|d d}t||t |tdWn4t jy} ztd| WYd} ~ dSd} ~ wty} ztd| WYd} ~ dSd} ~ wwz|t |WSty} ztd| WYd} ~ dSd} ~ ww)Nrmodels/reutersz)https://github.com/unclecode/crawl4ai.gitcrawl4aiu7[LOG] ⏬ Downloading Spacy model for the first time...zh[WARNING] Unable to remove existing folders. Please manually delete the following folders and try again:z- gitclonez-bT)stdoutstderrcheckmodels)parentsr reutersu-[LOG] ✅ Spacy Model downloaded successfullyz0An error occurred while cloning the repository: zAn error occurred: zError loading spacy model: )spacyr(rexistsanyiterdirrprintshutilrmtreePermissionError subprocessrunstrDEVNULLmkdircopytreeCalledProcessError Exceptionload) rvnamer' model_folderrepo_urlbranch repo_folder models_folder source_foldererrrload_spacy_modelsd          rFcCs|r+tdt}tj|dtj|dg}|D] }t|r&t|qtdtdt \}}td|tdt tdd S) z*Download all models required for Crawl4AI.z![LOG] Removing existing models...rlrsz[LOG] Existing models removed.z$[LOG] Downloading text classifier...z [LOG] Text classifier loaded on z,[LOG] Downloading custom NLTK Punkt model...u-[LOG] ✅ All models downloaded successfully.N) rzr(r!r"r#rrwr{r|rdrk)remove_existingr' model_foldersfolder_r rrrdownload_all_modelss"      rcCsDtdtdtjdd}|jdddd|}t|jd dS) Nz/[LOG] Welcome to the Crawl4AI Model Downloader!zE[LOG] This script will download all the models required for Crawl4AI.zCrawl4AI Model Downloader) descriptionz--remove-existing store_truez)Remove existing models before downloading)actionhelp)r)rzargparseArgumentParser add_argument parse_argsrr)parserargsrrrmains  r__main__)r6)F)$ functoolsrpathlibrr~r!r{tarfile model_loaderrurllib.requesturllibcrawl4ai.configrr"realpathr#getcwddirname__file__ __location__rrrrr(r5tupler9r=rdrkrrr__name__rrrrsH  "        +   <