Dj*7UddlmZddlZddlZddlZddlmZddlmZddl m Z ddl m Z ddl mZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%d.d Z&d/d Z'd0d Z(e)d ejTDZ+d e,d<e+Dcgc]}|d c}Z-de,d<d1dZ.d/dZ/d/dZ0d/dZ1d/dZ2d/dZ3d/dZ4d/dZ5d/dZ6d/dZ7d/dZ8d/dZ9d/dZ:d/dZ;d/dZ d3 d4d"Z?e d#d5d$Z@d6d%ZAd7d&ZBd8d9d'ZCd:d(ZDd;d)ZEd*ejd+f dd-ZHycc}w)?) annotationsN) bisect_right)IncrementalDecoder) lru_cache) Generator)ENCODING_MARKSIANA_SUPPORTED_SIMILARRE_POSSIBLE_ENCODING_INDICATIONUNICODE_RANGES_COMBINED_SECONDARY_RANGE_NAMESCOMMON_CJK_CHARACTERS_LATIN_CJK_HANGUL _KATAKANA _HIRAGANA_HALFWIDTH_KATAKANA_THAI_ARABIC_ARABIC_ISOLATED_FORM _LIGATURE _SUPERSCRIPT_SENTENCE_OPEN_PUNCTUATION_ACCENT_KEYWORDS _ACCENTUATED_KNOWN_MB_DECODERS_KNOWN_MB_CLASSES _IANA_NAMES_MULTIBYTE_SEARCH_RADIUSc tj|}d}d|vr |tz}d|vr |tz}d|vr |t z}d|vr|t z}d|vr |tz}d|vr |tz}d|vr |tz}d |vr|tz}d |vr |tz}d |vs|jd r |tz}d |vr |tz}|dvr |tz}t D]}||vs|t"z}|S|S#t$rYywxYw)zRCompute all name-based classification flags with a single unicodedata.name() call.rLATINCJKHANGULKATAKANA HALFWIDTHHIRAGANATHAIARABICz ISOLATED FORMLIGATUREz LETTER AE SUPERSCRIPT>INVERTED QUESTION MARKINVERTED EXCLAMATION MARK) unicodedataname ValueErrorrrrrrrrrrendswithrrrrr) characterdescflagskws V/root/workspace/ytshorts/venv/lib/python3.12/site-packages/charset_normalizer/utils.py_character_flagsr7'sC$$Y/E$  }  4 T  $  ( (ET  ~ 4  d " * *ETT]];7   FF ++ : \ !E  L  LG sC## C/.C/c8tt|tzSN)boolr7rr2s r6is_accentuatedr<Qs  +l: ;;ctj|}|s|S|jd}tt |ddS)N r)r. decompositionsplitchrint)r2 decomposedcodess r6 remove_accentrGUsA!// :J !'',E s58R !!r=c#TK|] \}}|j|j|f"ywr9)startstop).0r/ ord_ranges r6 rMas+<i__innd+<s&(zlist[tuple[int, int, str]]_UNICODE_RANGES_SORTEDz list[int]_UNICODE_RANGE_STARTSct|}|dkry|dkrytt|dz }|dk\rt|\}}}||kr|Sy)zK Retrieve the Unicode range official name from a single character. zControl characterz Basic LatinrrN)ordrrOrN)r2 character_ordidxrIrJr/s r6 unicode_rangerVhs_YMr"s ,m ｜+<>TZ>PcPdPo)isspacer.r])r2r_s r6 is_separatorrvsBi+AA)229= $ $ P(:>P(PPr=cD|j|jk7Sr9)islowerisupperr;s r6is_case_variablerzs    )"3"3"5 55r=c8tt|tzSr9)r:r7rr;s r6is_cjkr|s  +d2 33r=c8tt|tzSr9)r:r7rr;s r6 is_hiraganar~  +i7 88r=c8tt|tzSr9)r:r7rr;s r6 is_katakanarrr=c8tt|tzSr9)r:r7rr;s r6 is_hangulr  +g5 66r=c8tt|tzSr9)r:r7rr;s r6is_thairs  +e3 44r=c8tt|tzSr9)r:r7rr;s r6 is_arabicrrr=c8tt|tzSr9)r:r7rr;s r6is_arabic_isolated_formrs  +.CC DDr=c|tvSr9)rr;s r6is_cjk_uncommonrs 1 11r=c|tvSr9)r ) range_names r6is_unicode_range_secondaryrs / //r=cf|j xr|j xr |dk7xr|dk7S)Nu)ru isprintabler;s r6is_unprintablersF     "%%' ' "   "  ! r=ct|ttfstt |}|dt ||}|j }d|vrd|vry|jdd}tj|D]L}|jdj jdd }tj|}|J|cSy) zW Extract using ASCII-only decoder any specified encoding in the first n-bytes. Nscodingscharsetasciiignoreerrorsr-_) isinstancebytes bytearray TypeErrorlenminlowerdecoder finditergroupreplacerget) sequence search_zoneseq_len search_bytes lowered_bytes decoded_zonematchspecified_encoding encoding_ianas r6any_specified_encodingrs h 2 3x=G7c';78L &&(M %*M*I$++GH+EL099,G!"[[^113;;CE#(:;  $ ! r=)maxsizec|tvrytD](} tj|j |yy#t t tf$rYBwxYw)zQ Verify is a specific encoding is a multi byte one based on it IANA name TF)rr importlib import_modulegetcodec ImportErrorAttributeError LookupError)r/providers r6is_multi_byte_encodingrsd  !!&   # #H - 6 6t <   ^[9   s$<AActD]>}t|}t|tr|g}|D]}|j|s||fccS@y)z9 Identify and extract SIG/BOM in given sequence. )Nr=)r rr startswith)r iana_encodingmarksmarks r6identify_sig_or_bomrs[ (+ %3M%B eU #GE +D""4($d** + + r=c |dvS)N>utf_16utf_32)rs r6should_strip_sig_or_bomr(s  4 44r=c|jjdd}tj|}||S|rt d|d|S)zIReturns the Python normalized encoding name (Not the IANA official name).rrzUnable to retrieve IANA for '')rrrrr0)cp_namestrictrs r6 iana_namer,sQmmo%%c3/GOOG,M  8 CDD Nr=ctt|s t|rytjd|j}tjd|j}|d}|d}d}t dD]7}t |g}|j ||j |k(s3|dz }9|dz S)Ngz encodings.rrrr)rrrrrangerr) iana_name_a iana_name_b decoder_a decoder_bid_aid_bcharacter_match_counti to_be_decodeds r6 cp_similarityr:sk*.D[.Q''*[M(BCVVI''*[M(BCVVI(9D(9D!" 3Z'$aSz ;;} %])C C !Q & !' !3 &&r=c,|tvxr |t|vS)z Determine if two code page are at least 80% similar. IANA_SUPPORTED_SIMILAR dict was generated using the function cp_similarity. )r )rrs r6 is_cp_similarrNs% -- ? 1+> >r=charset_normalizerz)%(asctime)s | %(levelname)s | %(message)sctj|}|j|tj}|j tj ||j |yr9)logging getLoggersetLevel StreamHandler setFormatter Formatter addHandler)r/level format_stringloggerhandlers r6set_logging_handlerrYsU   t $F OOE##%G **=9: gr=c #PK|rG|jdr6t|} t|} |D]} | | z| z} || | |z}|sy|y|r|s|D]} || | |z}|sy|y| r4|s|n |t|d}|D]} || | |z}|syt||y|D]} | |z}|t|dzkDr|| | |z}|r|s||z}|j||rdnd}|r| dkDrt |d}|d|}d}|rXt|} | | zt|z}t d|t z }t | |t z}|j|||dk\}|rE|sC||vr?t| | d z d D],}|||}|r|s||z}|j|d}|d||vs,n|yw) Niso2022_rrrrr@F) rrstrrrmaxr findr) sequencesroffsets chunk_sizebom_or_sig_availablestrip_sig_or_bom sig_payloadis_multi_byte_decoderdecoded_payloaddeferred_decodingdecoded_lengthsequence_lengthrdecoded_offsetchunk base_bytes cut_sequence chunk_endchunk_partial_size_chk chunk_prefix found_nearbyexpected_offset search_start search_endjs r6cut_sequence_chunksrfsd=33J?_-i. A/?BN#N^j5PQEK   !6 A#AJ7EK   .I9S=M=O3P  3A%a!j.9LlM2 2  3 2 AJI3y>A--$QZ8L#,<*\9  ''#8xh(E%Q.1*b.A&$%<&<= $ "%(%9N&'.&8C N&JO#&q/rs "%8'T<"6<<8288:<623I#IQAaD#IyI*6 , ENQ6499757E2059.18 40$5 '(%D       ,#'#^ ^^^ ^  ^  ^^ ^ ^^ ^C$Js E