o h@sddlZddlZddlZddlmZddlZddlmZddlZddl Z ddl Z ddl Z ddl m Z ddl mZmZeejddZGd d d Zd ed efd dZdd eefddZddZedkrhedSdS)N)Path)Optional)datetime) AsyncLoggerLogLevelT) log_levelverbosec@s\eZdZdefddZdedefddZdedefd d Zded edefd d ZddZ dS)DatabaseMigrationdb_pathcCs||_|tj||_dSN)r _ensure_content_dirsospathdirname content_paths)selfr rP/var/www/Befach/backend/venv/lib/python3.10/site-packages/crawl4ai/migrations.py__init__szDatabaseMigration.__init__ base_pathreturncCsNdddddd}i}|D]\}}tj||}tj|dd|||<q|S) N html_content cleaned_htmlmarkdown_contentextracted_content screenshots)htmlcleanedmarkdown extractedrT)exist_ok)itemsrrjoinmakedirs)rrdirsrkeyrrrrrr s z&DatabaseMigration._ensure_content_dirscontentcCs"t}|||}|Sr )xxhashxxh64updateencode hexdigest)rr'x content_hashrrr_generate_content_hash(sz(DatabaseMigration._generate_content_hash content_typec s|sdS||}tj|j||}tj|sEtj|ddd4IdH}||IdHWdIdH|S1IdHs@wY|S)Nwzutf-8)encoding) r/rrr#rexistsaiofilesopenwrite)rr'r0r. file_pathfrrr_store_content/s  z DatabaseMigration._store_contentc stjdddzt|j4IdH}|d4IdH}|IdH}WdIdHn 1IdHs5wYd}|D][}|\}}}} } } ||dIdH} ||dIdH} || d IdH}|| d IdH}|| d IdH}|d | | ||||fIdH|d 7}|ddkrtjd|dddq>|IdHtj d|dddWdIdHWdS1IdHswYWdSt y}ztj dddt |id|d}~ww)z/Migrate existing database to file-based storagezStarting database migration...INITtagNzqSELECT url, html, cleaned_html, markdown, extracted_content, screenshot FROM crawled_datarrrrr raE UPDATE crawled_data SET html = ?, cleaned_html = ?, markdown = ?, extracted_content = ?, screenshot = ? WHERE url = ? rdz Migrated z records...zMigration completed. z records processed.COMPLETEMigration failed: {error}ERRORerrormessager=params) loggerinfo aiosqliteconnectr executefetchallr:commitsuccess ExceptionrBstr)rdbcursorrowsmigrated_countrowurlrrrr screenshot html_hash cleaned_hash markdown_hashextracted_hashscreenshot_hasherrrmigrate_database<sN (  2' z"DatabaseMigration.migrate_databaseN) __name__ __module__ __qualname__rOrdictr r/r:r]rrrrr s  r r rc stj|stjddddStd}|d|}zt dIdHt ||tjd|d d|WSt yS}ztj d d d t|id |d}~ww)z"Create backup of existing databasez,No existing database found. Skipping backup.r;r<Nz %Y%m%d_%H%M%Sz.backup_rzDatabase backup created at: r?r@rArBrC)rrr4rFrGrnowstrftimeasynciosleepshutilcopy2rNrBrO)r timestamp backup_pathr\rrrbackup_databasers(   rjcsl|durtjtdd}tj|stjddddSt|IdH}|s)dSt |}| IdHdS)zRun database migrationNz .crawl4aiz crawl4ai.dbz/No existing database found. Skipping migration.r;r<) rrr#rhomer4rFrGrjr r])r ri migrationrrr run_migrations rmcCs>ddl}|jdd}|jddd|}tt|jdS)zCLI entry point for migrationrNz/Migrate Crawl4AI database to file-based storage) descriptionz --db-pathzCustom database path)help)argparseArgumentParser add_argument parse_argsrdrunrmr )rpparserargsrrrmains  rw__main__r )rrdloggingpathlibrrHtypingrr(r5rftimer async_loggerrrDEBUGrFr rOrjrmrwr^rrrrs(   ^