
    (i0                         d dl Z d dlZd dlZd dlZd dlZd dlmZ d dlmZ d dl	m
Z
 d dlmZ d dlmZmZ d dlmZ d dlmZ d dlmZ  G d	 d
      Zy)    N)create_bookfile)strip_symbols)transaction)gettext)fb2parseopdsdb)inpx_parser)configc                   R    e Zd ZddZd Zd Zd Zd Zd Zd Z	d	 Z
d
 Zd ZddZy)opdsScannerNc                     d | _         | j                          |r|| _        nCt        j                  d      | _        | j                  j                  t        j                         | j                          y )N )	fb2parserinit_parserloggerlogging	getLoggersetLevelCRITICAL
init_stats)selfr   s     #/opt/sopds/opds_catalog/sopdscan.py__init__zopdsScanner.__init__   sS     DK!++B/DKKK  !1!12    c                    t        j                  t        j                               | _        | j                  | _        | j                  | _        d| _        d| _        d| _        d| _	        d| _
        d| _        d| _        d| _        y )Nsecondsr   )datetime	timedeltatimet1t2t3books_addedbooks_skippedbooks_deletedarch_scannedarch_skippedbad_archives	bad_booksbooks_in_archivesr   s    r   r   zopdsScanner.init_stats!   so    ""499;7!"r   c                 8    t        j                  d      | _        y )NF)r   r   r,   s    r   r   zopdsScanner.init_parser.   s    ))%0r   c                    | j                   j                  d       | j                   j                  d       t        j                  d k7  r,| j                   j                  dt        j                  z         t        j
                  d k7  r,| j                   j                  dt        j
                  z         t        j                  d k7  r,| j                   j                  dt        j                  z         t        j                  d k7  r,| j                   j                  dt        j                  z         t        j                  d k7  r-| j                   j                  dt        j                  z         y y )Nz ***** Starting sopds-scan...zOPTIONS SETzroot_lib = %szfb2toepub = %szfb2tomobi = %sztemp_dir = %szFB2SAX = %s)	r   infodebugr
   SOPDS_ROOT_LIBSOPDS_FB2TOEPUBSOPDS_FB2TOMOBISOPDS_TEMP_DIRSOPDS_FB2SAXr,   s    r   log_optionszopdsScanner.log_options1   s    89-(  $&dkk.?.?PVPePe@e.f!!4'):):;KFLbLb;b)c!!4'):):;KFLbLb;b)c  $&dkk.?.?PVPePe@e.f$&dkk.>.>}VM`M`?`.a&r   c                    t        j                  t        j                               | _        | j                  j                  dt        | j                        z          | j                  j                  dt        | j                        z          | j                  j                  dt        | j                        z          t        j                  r2| j                  j                  dt        | j                        z          n1| j                  j                  dt        | j                        z          | j                  j                  dt        | j                        z          | j                  j                  dt        | j                        z          | j                  j                  d	t        | j                        z          | j                  j                  d
t        | j                         z          | j                  | j"                  z
  }|j$                  dz  }|j$                  |z
  dz  dz  }|j$                  dz  }| j                  j                  dt        |      z   dz   t        |      z   dz   t        |      z   dz          y )Nr   zBooks added      : zBooks skipped    : zBad books        : zBooks deleted    : zBooks DB entries deleted : zBooks in archives: zArchives scanned : zArchives skipped : zBad archives     : <   i  zTime estimated:z hours, z
 minutes, z	 seconds.)r   r   r    r"   r   r/   strr$   r%   r*   r
   SOPDS_DELETE_LOGICALr&   r+   r'   r(   r)   r!   r   )r   tr   minuteshourss        r   	log_statszopdsScanner.log_stats;   s   ""499;7.s43C3C/DDE.s43E3E/FFG.s4>>/BBC&&KK23t7I7I3JJKKK:3t?Q?Q;RRS.s43I3I/JJK.s43D3D/EEF.s43D3D/EEF.s43D3D/EEF
''$''/		"))G#b(",iio*3u:5j@WMlZ[^_f[gghsstr   c           
         | j                          | j                          d | _        d | _        d | _        t        j                          t        j                  t        j                  d      D ]0  \  }}}t        j                  rc|D cg c]  }t        j                  d|      s| }}|r;|D ]5  }t        j                  j                  ||      }| j!                  |||       7 {|D ]  }t        j                  j                  ||      }t        j                  j#                  |      \  }	}
|
j%                         dk(  r%t        j&                  si| j)                  |||       }t        j                  j+                  |      }| j-                  |||d d|        3 t        j.                         | _        | j3                          y c c}w )NT)followlinksz.*(.inpx|.INPX)$z.zipr   )r   r6   inp_catzip_filerel_pathr   avail_check_prepareoswalkr
   r1   SOPDS_INPX_ENABLErematchpathjoinprocessinpxsplitextlowerSOPDS_ZIPSCAN
processzipgetsizeprocessfilebooks_del_phisicalr&   r>   )r   	full_pathdirsfilesinpx
inpx_files	inpx_filefilenamene	file_sizes               r   scan_allzopdsScanner.scan_allO   s~   ""$&(ggf.C.CQU&V 	K"ItU''/4[tASUY8Zd[
[%/ E	!ww||IyA((ItDE  KWW\\)D1gg&&t,1GGI'++Yt< ggood3I$$T)Da	JK	K4 "4463 \s   G'Gc                    t         j                  j                  t         j                  j                  ||      t        j
                        | _        t        j                  rEt        j                  | j                  |      r%| j                  j                  d|z   dz          d}|S | j                  j                  d|z          t        j                  | j                  t        j                  |      | _        d}|S )NzSkip INP metafile . Not changed.   zStart process INP metafile = r   )rE   rJ   relpathrK   r
   r1   rC   SOPDS_INPX_SKIP_UNCHANGEDr   inp_skipr   r/   
addcattreeCAT_INPXrA   )r   rW   inp_fileinp_sizeresults        r   inpskip_callbackzopdsScanner.inpskip_callbackv   s    ggoobggll4&A&BWBWX++h0WKK1(:;KKLF 	 KK<XEF!,,T]]FOOXVDLFr   c                    |t         j                     d|t         j                     }|t         j                     j	                  t
              }|t         j                     j	                  t
              }d}|t         j                     j	                  t
              }t        j                  j                  | j                  |t         j                           }	t        j                  ||	d      d k(  rt        j                  |	t        j                         }
t        j"                  ||	|
|t         j                     |||||t         j$                     t        j                   
      }| xj&                  dz  c_        | xj(                  dz  c_        | j*                  j-                  d|	z   dz   |z   dz          |t         j.                     D ]=  }t        j0                  |j3                  dd            }t        j4                  ||       ? |t         j6                     D ]L  }t        j8                  |t        j:                  |j=                         j	                  t
                           N |t         j>                     D ]<  }t        j@                  |j	                               }t        jB                  ||d	       > y y )
N.r   rb   Book /
 Added ok., r   )"r	   sFilesExtsLangstripr   sTitlesDaterE   rJ   rK   rC   sFolderr   findbookrf   CAT_INPaddbooksSizer$   r+   r   r0   sAuthor	addauthorreplace
addbauthorsGenre	addbgenreaddgenrerN   sSeries	addseries
addbseries)r   rW   inp	meta_datar[   langtitle
annotationdocdaterel_path_currentcatbookaauthorgssers                    r   inpx_callbackzopdsScanner.inpx_callback   sI   !+"3"34Y{?O?O5PQ{(()//>**+11-@
+++,22=A77<<i@S@S6TU??4!115=##$4fnnEC%5c)KDTDT:UV[\fgnost}  J  P  P  uQ  RX  R`  R`  aDa""A%"KKg&66s:4?LM{223 /''		#c(:;!!$v./ {112 W  fooaggioom6T&UVW {223 .$$QWWY/!!$s1-. >r   c                 ~   t         j                  j                  |t        j                        }t         j                  j                  |      }t        j                  r8t        j                  ||      r"| j                  j                  d|z   dz          y | j                  j                  d|z          t        j                  |t        j                  |       t        j                  || j                  | j                         }t        j"                  |_        t        j&                  |_        |j+                          y )NzSkip INPX file = ra   zStart process INPX file = )rE   rJ   rc   r
   r1   rQ   rd   r   	inpx_skipr   r/   rf   rg   r	   Inpxr   rk   SOPDS_INPX_TEST_ZIPINPX_TEST_ZIPSOPDS_INPX_TEST_FILESINPX_TEST_FILESparse)r   r[   rT   rZ   rel_file	inpx_sizerW   s          r   rL   zopdsScanner.processinpx   s    f&;&;<GGOOD)	++0@0@)0TKK056FFGKK9$>?hC##D$*<*<d>S>STD!'!;!;D#)#?#?D JJLr   c           	      \   t         j                  j                  |t        j                        }t         j                  j                  |      }t        j                  ||      r7| xj                  dz  c_        | j                  j                  d|z   dz          y d}	 t        j                  |dd      }|j                         }t        j                  |t        j                  |      }	|D ]  }
	 | j                  j                  d|z   d	z   |
z          |j!                  |
      j"                  }|j%                  |
      }| j'                  |
|||	t        j                  |       |j)                           |j)                          | xj.                  dz  c_        | xj0                  |z  c_        y # t        j*                  $ r* | j                  j-                  d
|z   d	z   |
z          d}Y w xY w# t        j*                  $ r& | j                  j-                  d|z   dz          d}Y w xY w)Nrb   zSkip ZIP archive z. Already scanned.r   rT)
allowZip64zStart process ZIP file = z book file = zError processing ZIP file = z#Error while read ZIP archive. File z	 corrupt.)rE   rJ   rc   r
   r1   rQ   r   arc_skipr(   r   r0   zipfileZipFilenamelistrf   CAT_ZIPgetinfor^   openrR   close
BadZipFilewarningr'   r)   )r   r[   rT   rZ   r   zsizezip_process_errorzfilelistr   r\   r^   bookfiles                r   rP   zopdsScanner.processzip   s   f&;&;<%??8E*q KK1(:;OOP !&OOD#$?::<''&..%H! 	.A.))*Ed*J?*Z[\*\]"#))A,"8"8	#$66!9((4V^^IV (	. 	!!1$! 00 #-- .++,J4,OP_,_`a,ab,-).
 %% &##$I$$N{$Z[$%!&s8   AG2 1BF25'G2 29G/+G2 .G//G2 26H+*H+c                 R	   t         j                  j                  |      \  }}|j                         t        j
                  j                         v rt         j                  j                  |t        j                        }	| j                  j                  d|	z   dz   |z          	 t        j                  ||	d      d k(  r|dk(  rt        j                  |	|      }	 t        ||      }
|
r|
j$                  r|
j$                  j'                  t(              nd}|
j*                  r|
j*                  j'                  t(              n|}|
j,                  r|
j,                  nd}t/        |t0              r|j'                  t(              n#|j3                  d      j'                  t(              }|
j4                  r|
j4                  nd}t        j6                  ||	||dd  ||||||
      }| xj8                  dz  c_        |dk7  r| xj:                  dz  c_        | j                  j                  d	|	z   dz   |z   d
z          |
j<                  D ]  }|j?                  dtA        d            j'                  t(              }|rL|jC                  d      dk  r8|j                         }djE                  |d   djE                  |d d       g      }t        jF                  |      }t        jH                  ||        |
jJ                  D ]L  }t        jL                  |t        jN                  |j                         j'                  t(                           N |
jP                  rt        jR                  |
jP                  d         }|
jP                  d   xs d}|jU                         rtW        |      nd}t        jX                  |||       y | xjZ                  dz  c_-        | j                  j                  d	|	z   dz   |z   dz          y y y y # t        $ rI}d }
| j                  j!                  |	dz   |z   d|z  z          | xj"                  dz  c_        Y d }~gd }~ww xY w# t\        $ rF}| j                  j!                  |	dz   |z   d|z  z          | xj"                  dz  c_        Y d }~y d }~ww xY w)NzAttempt to add book ro   rb   r   z - z* Book parse error, skipping... (Error: %s)r   utf8rn   rp   r[   zUnknown authorrq   rr   r   index0z Already in DB.z7 Book UnicodeEncodeError error, skipping... (Error: %s))/rE   rJ   rM   rN   r
   SOPDS_BOOK_EXTENSIONSsplitrc   r1   r   r0   r   rz   rf   r   	Exceptionr   r*   language_coderv   r   r   description
isinstancer9   decoder   r|   r$   r+   authorsget_findrK   r   r   tagsr   r   series_infor   isdigitintr   r%   UnicodeEncodeError)r   r[   rT   rZ   r   archiver^   r\   r]   rC   	book_dataerrr   r   r   r   r   r   author_nameauthor_namesr   genrer   ser_nos                           r   rR   zopdsScanner.processfile   s   !!$'A77944::<<WW__Yv/D/DEHKK4X=cA$FG1$??415=z"--hw?,$3D$$?	 !OXOfOfy66<<]KlnHQ	 5 5m D]^>G>S>SY%:%:Y[
HRS]_bHcZ%5%5m%Disiziz  |B  jC  jI  jI  JW  jX
7@7H7H)"3"3b#^^D#aeE*U\]abklst((!+("A: 22A52))'(*:3*>t*CL*PQ!*!2!2 ;A*+%%q9I7J*K*Q*QR_*`K*{/?/?/DQ/F/:/@/@/B.1hhR8HR^_b`bRcId7e.f#)#3#3K#@F"--d6:; &/^^ gE",,T&//%++-BUBUVcBd2efg %00"("2"293H3H3Q"RC%.%:%:7%C%KF4:NN4DS[!F"--d3v>&&)&KK%%gh&6s&:4&?@Q&QR 17 ! = % ,$(	++Hu,<t,CFrsvFv,vw!+,P & $##Hu$4t$;>wz}>}$}~!#$sI   %6Q P (KQ <Q 	Q>Q	Q QQ 	R& <R!!R&)N)r   r   )__name__
__module____qualname__r   r   r   r6   r>   r_   rk   r   rL   rP   rR    r   r   r   r      s<    	#1bu(%N.:1:6$r   r   )rE   r    r   r   rH   book_tools.formatr   book_tools.format.utilr   	django.dbr   django.utils.translationr   r   opds_catalogr   r   r	   opds_catalog.zipfzipfr   	constancer
   r   r   r   r   <module>r      s:    
    	 - 0 ! 1 ) $ # l$ l$r   