U ku fh-@sddlmZddlmZddlmZddlmZmZm Z m Z m Z m Z m Z ddlmZddlmZmZmZGdd d ZGd d d Ze eefZe eZGd d d ZdS))aliases)sha256)dumps)AnyDictIteratorListOptionalTupleUnion)TOO_BIG_SEQUENCE) iana_nameis_multi_byte_encoding unicode_rangec@seZdZd8eeeedeedddZe edddZ e edd d Z e ed d d Z ed ddZed ddZdddddZe ed ddZe eed ddZe ed ddZe ed ddZe eed ddZe ed ddZe ed d d!Ze ed d"d#Ze ed d$d%Ze ed d&d'Ze ed d(d)Ze edd d*d+Ze ed d,d-Ze eed d.d/Ze eed d0d1Z d9eed3d4d5Z!e ed d6d7Z"dS): CharsetMatchNCoherenceMatches)payloadguessed_encodingmean_mess_ratiohas_sig_or_bom languagesdecoded_payloadcCsF||_||_||_||_||_d|_g|_d|_d|_d|_ ||_ dS)N) _payload _encoding_mean_mess_ratio _languages_has_sig_or_bom_unicode_ranges_leavesZ_mean_coherence_ratio_output_payload_output_encoding_string)selfrrrrrrr%=/tmp/pip-unpacked-wheel-fc9lr3jv/charset_normalizer/models.py__init__ s zCharsetMatch.__init__)otherreturncCs>t|ts&tdt|jt|j|j|jko<|j|jkS)Nz&__eq__ cannot be invoked on {} and {}.) isinstancer TypeErrorformatstr __class__encoding fingerprintr$r(r%r%r&__eq__$s zCharsetMatch.__eq__cCst|tstt|j|j}t|j|j}|dkrJ|dkrJ|j|jkS|dkr|dkrt|jtkrt|j|jkS|j |j kS|j|jkS)zQ Implemented to make sorted available upon CharsetMatches items. g{Gz?g{Gz?) r*r ValueErrorabschaos coherencelenrr multi_byte_usage)r$r(Zchaos_differenceZcoherence_differencer%r%r&__lt__-s    zCharsetMatch.__lt__r)cCsdtt|t|jS)Ng?)r7r-rawr$r%r%r&r8CszCharsetMatch.multi_byte_usagecCs"|jdkrt|j|jd|_|jS)Nstrict)r#r-rrr<r%r%r&__str__Gs zCharsetMatch.__str__cCsd|j|jS)Nz)r,r/r0r<r%r%r&__repr__MszCharsetMatch.__repr__cCs8t|tr||kr"td|jd|_|j|dS)Nz;Unable to add instance <{}> as a submatch of a CharsetMatch)r*rr3r,r.r#r appendr1r%r%r& add_submatchPszCharsetMatch.add_submatchcCs|jSN)rr<r%r%r&r/[szCharsetMatch.encodingcCsDg}tD]2\}}|j|kr*||q |j|kr ||q |S)z Encoding name are known by many name, using this could help when searching for IBM855 when it's listed as CP855. )ritemsr/r@)r$Z also_known_asupr%r%r&encoding_aliases_s    zCharsetMatch.encoding_aliasescCs|jSrBrr<r%r%r&bomlszCharsetMatch.bomcCs|jSrBrGr<r%r%r&byte_order_markpszCharsetMatch.byte_order_markcCsdd|jDS)z Return the complete list of possible languages found in decoded sequence. Usually not really useful. Returned list may be empty even if 'language' property return something != 'Unknown'. cSsg|] }|dqS)rr%).0er%r%r& zsz*CharsetMatch.languages..rr<r%r%r&rtszCharsetMatch.languagescCsp|jsbd|jkrdSddlm}m}t|jr8||jn||j}t|dksVd|krZdS|dS|jddS)z Most probable language found in decoded sequence. If none were detected or inferred, the property will return "Unknown". asciiZEnglishr)encoding_languagesmb_encoding_languagesz Latin BasedUnknown)rcould_be_from_charsetZcharset_normalizer.cdrOrPrr/r7)r$rOrPrr%r%r&language|s  zCharsetMatch.languagecCs|jSrB)rr<r%r%r&r5szCharsetMatch.chaoscCs|js dS|jddS)Nrrr rMr<r%r%r&r6szCharsetMatch.coherencecCst|jdddSNd)ndigits)roundr5r<r%r%r& percent_chaosszCharsetMatch.percent_chaoscCst|jdddSrT)rXr6r<r%r%r&percent_coherenceszCharsetMatch.percent_coherencecCs|jS)z+ Original untouched bytes. )rr<r%r%r&r;szCharsetMatch.rawcCs|jSrB)r r<r%r%r&submatchszCharsetMatch.submatchcCst|jdkSNr)r7r r<r%r%r& has_submatchszCharsetMatch.has_submatchcCs@|jdk r|jSddt|D}ttdd|D|_|jS)NcSsg|] }t|qSr%)r)rJcharr%r%r&rLsz*CharsetMatch.alphabets..cSsh|] }|r|qSr%r%)rJrr%r%r& sz)CharsetMatch.alphabets..)rr-sortedlist)r$Zdetected_rangesr%r%r& alphabetss zCharsetMatch.alphabetscCs|jgdd|jDS)z The complete list of encoding that output the exact SAME str result and therefore could be the originating encoding. This list does include the encoding available in property 'encoding'. cSsg|] }|jqSr%)r/)rJmr%r%r&rLsz6CharsetMatch.could_be_from_charset..)rr r<r%r%r&rRsz"CharsetMatch.could_be_from_charsetutf_8)r/r)cCs2|jdks|j|kr,||_t||d|_|jS)z Method to get re-encoded bytes payload using given target encoding. Default to UTF-8. Any errors will be simply ignored by the encoder NOT replaced. Nreplace)r"r-encoder!)r$r/r%r%r&outputszCharsetMatch.outputcCst|S)zw Retrieve the unique SHA256 computed using the transformed (re-encoded) payload. Not the original one. )rrh hexdigestr<r%r%r&r0szCharsetMatch.fingerprint)N)re)#__name__ __module__ __qualname__bytesr-floatboolr r'objectr2r9propertyr8r>r?rAr/rrFrHrIrrSr5r6rYrZr;r[r]rcrRrhr0r%r%r%r&r sb      rc@seZdZdZdeeedddZeedddZ e e e fed d d Z e dd d ZedddZedd ddZeddddZeddddZdS)CharsetMatchesz Container with every CharsetMatch items ordered by default from most probable to the less one. Act like a list(iterable) but does not implements all related methods. N)resultscCs|r t|ng|_dSrB)ra_results)r$rsr%r%r&r'szCharsetMatches.__init__r:ccs|jEdHdSrBrtr<r%r%r&__iter__szCharsetMatches.__iter__)itemr)cCsNt|tr|j|St|trFt|d}|jD]}||jkr.|Sq.tdS)z Retrieve a single item either by its position or encoding name (alias may be used here). Raise KeyError upon invalid index or encoding not present in results. FN)r*intrtr-rrRKeyError)r$rwresultr%r%r& __getitem__s       zCharsetMatches.__getitem__cCs t|jSrBr7rtr<r%r%r&__len__szCharsetMatches.__len__cCst|jdkSr\r|r<r%r%r&__bool__szCharsetMatches.__bool__cCs|t|tstdt|jt|jtkr`|j D],}|j |j kr2|j |j kr2| |dSq2|j |t|j |_ dS)z~ Insert a single match. Will be inserted accordingly to preserve sort. Can be inserted as a submatch. z-Cannot append instance '{}' to CharsetMatchesN)r*rr3r,r-r.r7r;r rtr0r5rAr@ra)r$rwmatchr%r%r&r@s    zCharsetMatches.appendrcCs|js dS|jdS)zQ Simply return the first match. Strict equivalent to matches[0]. Nrrur<r%r%r&bestszCharsetMatches.bestcCs|S)zP Redundant method, call the method best(). Kept for BC reasons. )rr<r%r%r&firstszCharsetMatches.first)N)rjrkrl__doc__r rrr'rrvr rxr-r{r}ror~r@rrr%r%r%r&rrsrrc @sjeZdZeeeeeeeeeeeeeeeed ddZe e ee fdddZ edddZ d S) CliDetectionResult pathr/rFalternative_encodingsrSrcrr5r6 unicode_path is_preferredc CsF||_| |_||_||_||_||_||_||_||_| |_ | |_ dSrB) rrr/rFrrSrcrr5r6r) r$rr/rFrrSrcrr5r6rrr%r%r&r')szCliDetectionResult.__init__r:c Cs2|j|j|j|j|j|j|j|j|j|j |j d S)Nrrr<r%r%r&__dict__CszCliDetectionResult.__dict__cCst|jdddS)NT) ensure_asciiindent)rrr<r%r%r&to_jsonSszCliDetectionResult.to_jsonN)rjrkrlr-r rrornr'rqrrrrr%r%r%r&r(s rN)Zencodings.aliasesrhashlibrjsonrtypingrrrrr r r Zconstantr utilsrrrrrrr-rnZCoherenceMatchrrr%r%r%r&s   $ XC