o ›Œh ã@s@ddlZddlmZddlZddlmZmZej e  de  ¡¡d¡a ej t ddej t d¡a d d „Z d+d efd d„Zdd„Zdedeeeeeeeeeeef fdd„Zd,dedededededed edededefdd „Zdefd!d"„Zd#d$„Zd%d&„Zd-d ed'efd(d)„Zed*kržej t ¡r™e t ¡e ƒdSdS).éN)ÚPath)ÚOptionalÚTupleÚCRAWL4_AI_BASE_DIRECTORYz .crawl4aiT)Úexist_okz crawl4ai.dbcCs0t t¡}| ¡}| d¡| ¡| ¡dS)NaŽ CREATE TABLE IF NOT EXISTS crawled_data ( url TEXT PRIMARY KEY, html TEXT, cleaned_html TEXT, markdown TEXT, extracted_content TEXT, success BOOLEAN, media TEXT DEFAULT "{}", links TEXT DEFAULT "{}", metadata TEXT DEFAULT "{}", screenshot TEXT DEFAULT "" ) )Úsqlite3ÚconnectÚDB_PATHÚcursorÚexecuteÚcommitÚclose)Úconnr ©rúN/var/www/Befach/backend/venv/lib/python3.10/site-packages/crawl4ai/database.pyÚinit_db s   rÚmediaÚ new_columnc Csvtƒzt t¡}| ¡}| d|›d¡| ¡| ¡WdSty:}zt d|›ƒWYd}~dSd}~ww)Nz$ALTER TABLE crawled_data ADD COLUMN z TEXT DEFAULT ""z2Error altering database to add screenshot column: © Ú check_db_pathrrr r r r r Ú ExceptionÚprint)rrr ÚerrrÚalter_db_add_screenshots €ÿrcCststdƒ‚dS)Nz%Database path is not set or is empty.)r Ú ValueErrorrrrrr*sÿrÚurlÚreturnc Csrtƒzt t¡}| ¡}| d|f¡| ¡}| ¡|WSty8}zt d|›ƒWYd}~dSd}~ww)NzˆSELECT url, html, cleaned_html, markdown, extracted_content, success, media, links, metadata, screenshot FROM crawled_data WHERE url = ?zError retrieving cached URL: © rrrr r r Úfetchoner rr)rrr ÚresultrrrrÚget_cached_url.s €þr ú{}ÚÚhtmlÚ cleaned_htmlÚmarkdownÚextracted_contentÚsuccessÚlinksÚmetadataÚ screenshotc Cs„tƒz$t t¡} |  ¡} |  d|||||||||| f ¡|  ¡|  ¡WdStyA} zt d| ›ƒWYd} ~ dSd} ~ ww)Na¤ INSERT INTO crawled_data (url, html, cleaned_html, markdown, extracted_content, success, media, links, metadata, screenshot) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(url) DO UPDATE SET html = excluded.html, cleaned_html = excluded.cleaned_html, markdown = excluded.markdown, extracted_content = excluded.extracted_content, success = excluded.success, media = excluded.media, links = excluded.links, metadata = excluded.metadata, screenshot = excluded.screenshot zError caching URL: r) rr#r$r%r&r'rr(r)r*rr rrrrÚ cache_url;s  ó€ÿr+c Csrtƒzt t¡}| ¡}| d¡| ¡}| ¡|dWSty8}zt d|›ƒWYd}~dSd}~ww)Nz!SELECT COUNT(*) FROM crawled_datarzError getting total count: r)rr rrrrrÚget_total_countSs   €þr,c Cóntƒzt t¡}| ¡}| d¡| ¡| ¡WdSty6}zt d|›ƒWYd}~dSd}~ww)NzDELETE FROM crawled_datazError clearing database: r©rr rrrrÚclear_db`ó  €ÿr/c Cr-)NzDROP TABLE crawled_datazError flushing database: rr.rrrÚflush_dbkr0r1Ú default_valuec Cs|tƒz t t¡}| ¡}| d|›d|›d¡| ¡| ¡WdSty=}zt d|›ƒWYd}~dSd}~ww)NzUPDATE crawled_data SET z = "z" WHERE screenshot IS NULLz!Error updating existing records: r)rr2rr rrrrÚupdate_existing_recordsvs €ÿr3Ú__main__)r)r!r!r!r")rr!)ÚosÚpathlibrrÚtypingrrÚpathÚjoinÚgetenvÚhomer ÚmakedirsrÚstrrrÚboolr r+Úintr,r/r1r3Ú__name__ÚexistsÚremoverrrrÚs*  ,4       ü