ĮiX@ddlmZddlZddlmZddlmZddlmZm Z m Z m Z ddl m Z mZmZmZmZddlmZdd lmZmZdd lmZmZmZmZmZmZej<d Zej@Z!e!jEejFd  d dd Z$ d ddZ% d ddZ& d ddZ'y)) annotationsN)PathLike)BinaryIO)coherence_ratioencoding_languagesmb_encoding_languagesmerge_coherence_ratios)IANA_SUPPORTEDIANA_SUPPORTED_SIMILARTOO_BIG_SEQUENCETOO_SMALL_SEQUENCETRACE) mess_ratio) CharsetMatchCharsetMatches)any_specified_encodingcut_sequence_chunks iana_nameidentify_sig_or_bomis_multi_byte_encodingshould_strip_sig_or_bomcharset_normalizerz)%(asctime)s | %(levelname)s | %(message)sc "t|ttfs#tdj t ||rBt j} t jtt jtt|} | dk(r_t jd|r.t jtt j tt!|dddgdgS|Dt j#td d j%||D cgc]} t'| d}} ng}|Dt j#td d j%||D cgc]} t'| d}} ng}| ||zkr!t j#td ||| d }| }|d kDr| |z |krt)| |z }t|t*k} t|t,k\}| r*t j#tdj | n+|r)t j#tdj | g}|r t/|nd}|,|j1|t j#td|t3}g}g}t3}d}d}d}t}t}t5|\}}|6|j1|t j#tdt|||j1dd|vr|j1d|t6zD]g}|r||vr |r||vr||vr|j9|d}||k(}|xr t;|}|dvr|st j#td|`|dvr|st j#td|||vrt j#td| t=|} |r9| dur5tC|dur|dt)dn|t|t)d|ntC|dur|n |t|d|}tI|sdn t|| t)| |z }"| xr|duxrt|| k}#|#rt j#td|t)t|"dz }$tK|$d}$d}%d}&g}'g}( tM|||"||||| | D]g})|'j1|)|(j1tO|)||duxrd t|cxkxrdknc|(d |k\r|%d z }%|%|$k\s|sb|dusgn|&s$|r"| s |t)d"djQ|d#$|(rtS|(t|(z nd}*|*|k\s|%|$k\r|j1||tTvr|jWtT|t j#td&||%tY|*d'zd()| r/|dd|d*d+fvr&|&s$t!||||g||,}+||k(r|+}n |dk(r|+}n|+}t j#td-|tY|*d'zd()| s t[|},n t]|},|,r3t j#td.j |tC|,g}-|dk7r8|'D]3})t_|)||,rd/j%|,nd}.|-j1|.5ta|-}/|/r*t j#td0j |/|t!|||*||/|dus||ddfvr|nd|,}0|j1|0||ddfvry|*d1krt|*dk(r^t jd2|0jb|r.t jtt j t|0gcS|j1|0t|r||||vrvd|vrrd|vrn|je}1t jd2|1jb|r.t jtt j t|1gcS||k(st jd3||r.t jtt j t||gcSt|dk(r|s|s|rt j#td4|r2t jd5|jb|j1|nr|r||r|r|jf|jfk7s|'t jd6|j1|n(|r&t jd7|j1||rt@f$rt j#td|YwxYw#tDtFf$rQ}!t|!tFs%t j#td|tC|!|j1|Yd}!~!Zd}!~!wwxYw#tD$r4}!t j#td!|tC|!|$}%d}&Yd}!~!d}!~!wwxYw#tD$rA}!t j#td%|tC|!|j1|Yd}!~!d}!~!wwxYw):af Given a raw bytes sequence, return the best possibles charset usable to render str objects. If there is no results, it is a strong indicator that the source is binary/not text. By default, the process will extract 5 blocks of 512o each to assess the mess and coherence of a given sequence. And will give up a particular code page after 20% of measured mess. Those criteria are customizable at will. The preemptive behavior DOES NOT replace the traditional detection workflow, it prioritize a particular code page but never take it for granted. Can improve the performance. You may want to focus your attention to some code page or/and not others, use cp_isolation and cp_exclusion for that purpose. This function will strip the SIG in the payload/sequence every time except on UTF-16, UTF-32. By default the library does not setup any handler other than the NullHandler, if you choose to set the 'explain' toggle to True it will alter the logger configuration to add a StreamHandler that is suitable for debugging. Custom logging format and handler can be set manually. z3Expected object of type bytes or bytearray, got: {}rzTrying to detect encoding from a tiny portion of ({}) byte(s).zIUsing lazy str decoding because the payload is quite large, ({}) byte(s).z@Detected declarative mark in sequence. Priority +1 given for %s.zIDetected a SIG or BOM mark on first %i byte(s). Priority +1 given for %s.ascii>utf_16utf_32z\Encoding %s won't be tested as-is because it require a BOM. Will try some sub-encoder LE/BE.>utf_7zREncoding %s won't be tested as-is because detection is unreliable without BOM/SIG.zY%s is deemed too similar to a code page that was already considered unsuited. Continuing!z2Encoding %s does not provide an IncrementalDecodergA)encodingz9Code page %s does not fit given bytes sequence at ALL. %szpCode page %s is a multi byte encoding table and it appear that at least one character was encoded using n-bytes.TzaLazyStr Loading: After MD chunk decode, code page %s does not fit given bytes sequence at ALL. %sgj@strict)errorsz^LazyStr Loading: After final lookup, code page %s does not fit given bytes sequence at ALL. %szc%s was excluded because of initial chaos probing. Gave up %i time(s). Computed mean chaos is %f %%.d)ndigitsrr)preemptive_declarationz=%s passed initial chaos probing. Mean measured chaos is %f %%z&{} should target any language(s) of {},z We detected language {} using {}皙?z.Encoding detection: %s is most likely the one.zoEncoding detection: %s is most likely the one as we detected a BOM or SIG within the beginning of the sequence.zONothing got out of the detection process. Using ASCII/UTF-8/Specified fallback.z7Encoding detection: %s will be used as a fallback matchz:Encoding detection: utf_8 will be used as a fallback matchz:Encoding detection: ascii will be used as a fallback matchz]Encoding detection: Found %s as plausible (best-candidate) for content. With %i alternatives.z=Encoding detection: Unable to determine any suitable charset.)4 isinstance bytearraybytes TypeErrorformattypeloggerlevel addHandlerexplain_handlersetLevelrlendebug removeHandlerrrlogjoinrintrr rappendsetrr addrrModuleNotFoundError ImportErrorstrUnicodeDecodeError LookupErrorrangemaxrrdecodesumr updateroundrr rr r!best fingerprint)2 sequencessteps chunk_size threshold cp_isolation cp_exclusionpreemptive_behaviourexplainlanguage_thresholdenable_fallbackprevious_logger_levellengthcpis_too_small_sequenceis_too_large_sequenceprioritized_encodingsspecified_encodingtestedtested_but_hard_failuretested_but_soft_failuresoft_failure_skipfallback_ascii fallback_u8fallback_specifiedresultsearly_stop_results sig_encoding sig_payload encoding_ianadecoded_payloadbom_or_sig_availablestrip_sig_or_bomis_multi_byte_decoderer_multi_byte_bonusmax_chunk_gave_upearly_stop_countlazy_str_hard_failure md_chunks md_ratioschunkmean_mess_ratiofallback_entrytarget_languages cd_ratioschunk_languagescd_ratios_merged current_matchprobable_results2 I/var/www/html/venv/lib/python3.12/site-packages/charset_normalizer/api.py from_bytesr&sI < i)U!3 4 A H HY   %+\\/*i.F { ST    1 OO1 2|IwUBPRSTUU  5 IIl #  8DD| "e,| D   6 IIl #  8DD| "e,| D  *u$%  l       qyVe^j0%( "%i.3E"E"%i.4D"D  L S S    W ^ ^  (*.By)t%$$%78  N  uF)+)+"%%*.N'+K.2,.G)7)9 3I >L+$$\2  W       )++$$W-.? M=  M\9  F "  =!&*%1]%B!5" :Q ;  0 09M JJn   I %.B JJd   - - JJk   *@*O ! $)>%)G,u4"+CI.&s;'7#d)D* #&,u4"&s;'7'9:* #&)As;/?     " .t+ .O$v-   JJ-  "%SWq[!1 115 ! %!  ' ),$ %   '  !4GA\1B,Ga,GR=I-$)$$(99(-=-F7 V&%) #d)+&--mH-MENY#i.!@SV i '+;?P+P # * *= 9 66!(()? )NO JJ0 o+Q7   !W&8(HMN-!-!(#+="!$66)7&"g-%3N"0K   K  /C' 3  %*<]*K 4]C   JJ8??!3'7#8   G #""1&2BCHH-.#   1#2)<  JJ299$m %    *U2$);Wg(NN #5 " }% 0'7C C#%#% D!**((9OO$9:%}o66  % %m 4 " ##+/AV/K6!6!,>,C,C,EO LL@(( $$_5 56!?"34 4 L ( LL1  $$_5 56!7=#9":; ;w @z  7|q .,> JJa   LLI"++  NN- . ^3"++~/I/II' LLU V NN; '  LLU V NN> * k LLN # # L1   TU_--. NAEEx$[1  JJD    6#K0 a- O!F  $ * *= 9  L  ) JJsA   1 $( ! )*&  t!F  (..}= sug&%g+ g0Ah!2A4j'j,j6k0*hh!j0Ai<<j k )j<<k l 6l  lc Ft|j||||||||| S)z Same thing than the function from_bytes but using a file pointer that is already ready. Will not close the file pointer. )rread) fprOrPrQrRrSrTrUrVrWs rfrom_fpr#s5      c nt|d5} t| ||||||||| cdddS#1swYyxYw)z Same thing than the function from_bytes but with one extra step. Opening and reading given file path in binary mode. Can raise IOError. rbN)openr) pathrOrPrQrRrSrTrUrVrWrs r from_pathrAsB dD R              s+4c t|ttfrt||||||||||  } | St|tt frt ||||||||||  } | St||||||||||  } | S)a) Detect if the given input (file, bytes, or path) points to a binary file. aka. not a string. Based on the same main heuristic algorithms and default kwargs at the sole exception that fallbacks match are disabled to be stricter around ASCII-compatible but unlikely to be a string. ) rOrPrQrRrSrTrUrVrW)r-rCrrr/r.rr) fp_or_path_or_payloadrOrPrQrRrSrTrUrVrWguessess r is_binaryr`s"'#x9 !!%%!51+  Z;C      !!%%!51+  4; !!%%!51+  ;r) 皙?NNTFr,T)rNzbytes | bytearrayrOr=rPr=rQfloatrRlist[str] | NonerSrrTboolrUrrVrrWrreturnr)rrrOr=rPr=rQrrRrrSrrTrrUrrVrrWrrr)rzstr | bytes | PathLikerOr=rPr=rQrrRrrSrrTrrUrrVrrWrrr) rrrNNTFr,F)rz!PathLike | str | BinaryIO | bytesrOr=rPr=rQrrRrrSrrTrrUrrVrrWrrr)( __future__rloggingosrtypingrcdrrr r constantr r r rrmdrmodelsrrutilsrrrrrr getLoggerr3 StreamHandlerr6 setFormatter Formatterrrrrrrrs" 0   / 0'''')GAB%)%)!% # z z zz z # z # zzzzzz~%)%)!% #    #  # @%)%)!% #       #  #       B%)%)!% #!?<? ?? ? # ? # ????? ?r