Bug 1920152 - Re-vendor using ./mach vendor python (which now uses 'uv') r=ahal,mach-reviewers

The dependency resolution in 'poetry' was 'lowest', the default
dependency resolution in 'uv' is 'highest'. This means when we vendor,
we will vendor the newest/highest version possible of all transitive
dependencies that are universally/cross-platform compatible.

We could change from the default of 'highest', but I think this is
better. Various packages have been upgraded, and some have been
removed/replaced by others, and that's why there's various changes in
the <site>.txt files.

Differential Revision: https://phabricator.services.mozilla.com/D222992
This commit is contained in:
ahochheiden
2024-10-03 14:47:55 +00:00
parent 161279367b
commit 45be02805e
923 changed files with 194282 additions and 56357 deletions
+5 -1
View File
@@ -28,9 +28,10 @@ vendored:third_party/python/frozenlist
vendored:third_party/python/giturlparse vendored:third_party/python/giturlparse
vendored:third_party/python/glean_parser vendored:third_party/python/glean_parser
vendored:third_party/python/gyp/pylib vendored:third_party/python/gyp/pylib
vendored:third_party/python/jinja2_time
vendored:third_party/python/json_e vendored:third_party/python/json_e
vendored:third_party/python/Mako vendored:third_party/python/Mako
vendored:third_party/python/markdown_it_py
vendored:third_party/python/mdurl
vendored:third_party/python/mohawk vendored:third_party/python/mohawk
vendored:third_party/python/mozilla_repo_urls vendored:third_party/python/mozilla_repo_urls
vendored:third_party/python/multidict vendored:third_party/python/multidict
@@ -38,11 +39,13 @@ vendored:third_party/python/pathspec
vendored:third_party/python/ply vendored:third_party/python/ply
vendored:third_party/python/pyasn1 vendored:third_party/python/pyasn1
vendored:third_party/python/pyasn1_modules vendored:third_party/python/pyasn1_modules
vendored:third_party/python/pygments
vendored:third_party/python/pylru vendored:third_party/python/pylru
vendored:third_party/python/python_dateutil vendored:third_party/python/python_dateutil
vendored:third_party/python/python_slugify vendored:third_party/python/python_slugify
vendored:third_party/python/redo vendored:third_party/python/redo
vendored:third_party/python/requests_unixsocket vendored:third_party/python/requests_unixsocket
vendored:third_party/python/rich
vendored:third_party/python/rsa vendored:third_party/python/rsa
vendored:third_party/python/slugid vendored:third_party/python/slugid
vendored:third_party/python/taskcluster vendored:third_party/python/taskcluster
@@ -50,6 +53,7 @@ vendored:third_party/python/taskcluster_taskgraph
vendored:third_party/python/taskcluster_urls vendored:third_party/python/taskcluster_urls
vendored:third_party/python/text_unidecode vendored:third_party/python/text_unidecode
vendored:third_party/python/toml vendored:third_party/python/toml
vendored:third_party/python/types_python_dateutil
vendored:third_party/python/voluptuous vendored:third_party/python/voluptuous
vendored:third_party/python/yamllint vendored:third_party/python/yamllint
vendored:third_party/python/yarl vendored:third_party/python/yarl
+5 -1
View File
@@ -31,8 +31,9 @@ vendored:third_party/python/frozenlist
vendored:third_party/python/giturlparse vendored:third_party/python/giturlparse
vendored:third_party/python/glean_parser vendored:third_party/python/glean_parser
vendored:third_party/python/gyp/pylib vendored:third_party/python/gyp/pylib
vendored:third_party/python/jinja2_time
vendored:third_party/python/json_e vendored:third_party/python/json_e
vendored:third_party/python/markdown_it_py
vendored:third_party/python/mdurl
vendored:third_party/python/mohawk vendored:third_party/python/mohawk
vendored:third_party/python/mozilla_repo_urls vendored:third_party/python/mozilla_repo_urls
vendored:third_party/python/multidict vendored:third_party/python/multidict
@@ -40,12 +41,14 @@ vendored:third_party/python/pathspec
vendored:third_party/python/ply vendored:third_party/python/ply
vendored:third_party/python/pyasn1 vendored:third_party/python/pyasn1
vendored:third_party/python/pyasn1_modules vendored:third_party/python/pyasn1_modules
vendored:third_party/python/pygments
vendored:third_party/python/pylru vendored:third_party/python/pylru
vendored:third_party/python/python_dateutil vendored:third_party/python/python_dateutil
vendored:third_party/python/python_slugify vendored:third_party/python/python_slugify
vendored:third_party/python/redo vendored:third_party/python/redo
vendored:third_party/python/requests_unixsocket vendored:third_party/python/requests_unixsocket
vendored:third_party/python/responses vendored:third_party/python/responses
vendored:third_party/python/rich
vendored:third_party/python/rsa vendored:third_party/python/rsa
vendored:third_party/python/slugid vendored:third_party/python/slugid
vendored:third_party/python/taskcluster vendored:third_party/python/taskcluster
@@ -53,6 +56,7 @@ vendored:third_party/python/taskcluster_taskgraph
vendored:third_party/python/taskcluster_urls vendored:third_party/python/taskcluster_urls
vendored:third_party/python/text_unidecode vendored:third_party/python/text_unidecode
vendored:third_party/python/toml vendored:third_party/python/toml
vendored:third_party/python/types_python_dateutil
vendored:third_party/python/voluptuous vendored:third_party/python/voluptuous
vendored:third_party/python/yamllint vendored:third_party/python/yamllint
vendored:third_party/python/yarl vendored:third_party/python/yarl
+5 -1
View File
@@ -37,16 +37,20 @@ vendored:third_party/python/fluent.syntax
vendored:third_party/python/frozenlist vendored:third_party/python/frozenlist
vendored:third_party/python/giturlparse vendored:third_party/python/giturlparse
vendored:third_party/python/gyp/pylib vendored:third_party/python/gyp/pylib
vendored:third_party/python/jinja2_time
vendored:third_party/python/json_e vendored:third_party/python/json_e
vendored:third_party/python/markdown_it_py
vendored:third_party/python/mdurl
vendored:third_party/python/mohawk vendored:third_party/python/mohawk
vendored:third_party/python/mozilla_repo_urls vendored:third_party/python/mozilla_repo_urls
vendored:third_party/python/pygments
vendored:third_party/python/pylru vendored:third_party/python/pylru
vendored:third_party/python/redo vendored:third_party/python/redo
vendored:third_party/python/requests_unixsocket vendored:third_party/python/requests_unixsocket
vendored:third_party/python/responses vendored:third_party/python/responses
vendored:third_party/python/rich
vendored:third_party/python/slugid vendored:third_party/python/slugid
vendored:third_party/python/taskcluster vendored:third_party/python/taskcluster
vendored:third_party/python/taskcluster_taskgraph vendored:third_party/python/taskcluster_taskgraph
vendored:third_party/python/taskcluster_urls vendored:third_party/python/taskcluster_urls
vendored:third_party/python/types_python_dateutil
vendored:third_party/python/voluptuous vendored:third_party/python/voluptuous
+5 -1
View File
@@ -13,22 +13,26 @@ vendored:third_party/python/esprima
vendored:third_party/python/fluent.syntax vendored:third_party/python/fluent.syntax
vendored:third_party/python/frozenlist vendored:third_party/python/frozenlist
vendored:third_party/python/giturlparse vendored:third_party/python/giturlparse
vendored:third_party/python/jinja2_time
vendored:third_party/python/json_e vendored:third_party/python/json_e
vendored:third_party/python/markdown_it_py
vendored:third_party/python/mdurl
vendored:third_party/python/mohawk vendored:third_party/python/mohawk
vendored:third_party/python/mozilla_repo_urls vendored:third_party/python/mozilla_repo_urls
vendored:third_party/python/multidict vendored:third_party/python/multidict
vendored:third_party/python/pathspec vendored:third_party/python/pathspec
vendored:third_party/python/pygments
vendored:third_party/python/python_dateutil vendored:third_party/python/python_dateutil
vendored:third_party/python/python_slugify vendored:third_party/python/python_slugify
vendored:third_party/python/redo vendored:third_party/python/redo
vendored:third_party/python/requests_unixsocket vendored:third_party/python/requests_unixsocket
vendored:third_party/python/rich
vendored:third_party/python/slugid vendored:third_party/python/slugid
vendored:third_party/python/taskcluster vendored:third_party/python/taskcluster
vendored:third_party/python/taskcluster_taskgraph vendored:third_party/python/taskcluster_taskgraph
vendored:third_party/python/taskcluster_urls vendored:third_party/python/taskcluster_urls
vendored:third_party/python/text_unidecode vendored:third_party/python/text_unidecode
vendored:third_party/python/toml vendored:third_party/python/toml
vendored:third_party/python/types_python_dateutil
vendored:third_party/python/voluptuous vendored:third_party/python/voluptuous
vendored:third_party/python/yamllint vendored:third_party/python/yamllint
vendored:third_party/python/yarl vendored:third_party/python/yarl
+5 -1
View File
@@ -39,8 +39,9 @@ vendored:third_party/python/frozenlist
vendored:third_party/python/giturlparse vendored:third_party/python/giturlparse
vendored:third_party/python/glean_parser vendored:third_party/python/glean_parser
vendored:third_party/python/gyp/pylib vendored:third_party/python/gyp/pylib
vendored:third_party/python/jinja2_time
vendored:third_party/python/json_e vendored:third_party/python/json_e
vendored:third_party/python/markdown_it_py
vendored:third_party/python/mdurl
vendored:third_party/python/mohawk vendored:third_party/python/mohawk
vendored:third_party/python/mozilla_repo_urls vendored:third_party/python/mozilla_repo_urls
vendored:third_party/python/multidict vendored:third_party/python/multidict
@@ -48,12 +49,14 @@ vendored:third_party/python/pathspec
vendored:third_party/python/ply vendored:third_party/python/ply
vendored:third_party/python/pyasn1 vendored:third_party/python/pyasn1
vendored:third_party/python/pyasn1_modules vendored:third_party/python/pyasn1_modules
vendored:third_party/python/pygments
vendored:third_party/python/pylru vendored:third_party/python/pylru
vendored:third_party/python/python_dateutil vendored:third_party/python/python_dateutil
vendored:third_party/python/python_slugify vendored:third_party/python/python_slugify
vendored:third_party/python/redo vendored:third_party/python/redo
vendored:third_party/python/requests_unixsocket vendored:third_party/python/requests_unixsocket
vendored:third_party/python/responses vendored:third_party/python/responses
vendored:third_party/python/rich
vendored:third_party/python/rsa vendored:third_party/python/rsa
vendored:third_party/python/slugid vendored:third_party/python/slugid
vendored:third_party/python/taskcluster vendored:third_party/python/taskcluster
@@ -61,6 +64,7 @@ vendored:third_party/python/taskcluster_taskgraph
vendored:third_party/python/taskcluster_urls vendored:third_party/python/taskcluster_urls
vendored:third_party/python/text_unidecode vendored:third_party/python/text_unidecode
vendored:third_party/python/toml vendored:third_party/python/toml
vendored:third_party/python/types_python_dateutil
vendored:third_party/python/voluptuous vendored:third_party/python/voluptuous
vendored:third_party/python/yamllint vendored:third_party/python/yamllint
vendored:third_party/python/yarl vendored:third_party/python/yarl
+5 -1
View File
@@ -31,9 +31,10 @@ vendored:third_party/python/frozenlist
vendored:third_party/python/giturlparse vendored:third_party/python/giturlparse
vendored:third_party/python/glean_parser vendored:third_party/python/glean_parser
vendored:third_party/python/gyp/pylib vendored:third_party/python/gyp/pylib
vendored:third_party/python/jinja2_time
vendored:third_party/python/json_e vendored:third_party/python/json_e
vendored:third_party/python/jsonschema vendored:third_party/python/jsonschema
vendored:third_party/python/markdown_it_py
vendored:third_party/python/mdurl
vendored:third_party/python/mohawk vendored:third_party/python/mohawk
vendored:third_party/python/mozilla_repo_urls vendored:third_party/python/mozilla_repo_urls
vendored:third_party/python/multidict vendored:third_party/python/multidict
@@ -42,6 +43,7 @@ vendored:third_party/python/pkgutil_resolve_name
vendored:third_party/python/ply vendored:third_party/python/ply
vendored:third_party/python/pyasn1 vendored:third_party/python/pyasn1
vendored:third_party/python/pyasn1_modules vendored:third_party/python/pyasn1_modules
vendored:third_party/python/pygments
vendored:third_party/python/pylru vendored:third_party/python/pylru
vendored:third_party/python/pyrsistent vendored:third_party/python/pyrsistent
vendored:third_party/python/python_dateutil vendored:third_party/python/python_dateutil
@@ -49,6 +51,7 @@ vendored:third_party/python/python_slugify
vendored:third_party/python/redo vendored:third_party/python/redo
vendored:third_party/python/requests_unixsocket vendored:third_party/python/requests_unixsocket
vendored:third_party/python/responses vendored:third_party/python/responses
vendored:third_party/python/rich
vendored:third_party/python/rsa vendored:third_party/python/rsa
vendored:third_party/python/slugid vendored:third_party/python/slugid
vendored:third_party/python/taskcluster vendored:third_party/python/taskcluster
@@ -56,6 +59,7 @@ vendored:third_party/python/taskcluster_taskgraph
vendored:third_party/python/taskcluster_urls vendored:third_party/python/taskcluster_urls
vendored:third_party/python/text_unidecode vendored:third_party/python/text_unidecode
vendored:third_party/python/toml vendored:third_party/python/toml
vendored:third_party/python/types_python_dateutil
vendored:third_party/python/voluptuous vendored:third_party/python/voluptuous
vendored:third_party/python/yamllint vendored:third_party/python/yamllint
vendored:third_party/python/yarl vendored:third_party/python/yarl
-16
View File
@@ -1,16 +0,0 @@
arrow/__init__.py,sha256=HxsSJGl56GoeHB__No-kdGmC_Wes_Ttf0ohOy7OoFig,872
arrow/_version.py,sha256=C-D_WWrVkBDmQmApLcm0sWNh2CgIrwWfc8_sB5vvU-Q,22
arrow/api.py,sha256=6tdqrG0NjrKO22_eWHU4a5xerfR6IrZPY-yynGpnvTM,2755
arrow/arrow.py,sha256=CnSXk3GCi1DroUvElSxlwQy9Y-2lCUSV5GKLLrBFmRA,63570
arrow/constants.py,sha256=y3scgWgxiFuQg4DeFlhmexy1BA7K8LFNZyqK-VWPQJs,3238
arrow/factory.py,sha256=dWP3XIYfYjqp7DCOdEYAD7PQfsbpQE70Ph9OS1A1LnE,11435
arrow/formatter.py,sha256=YpYY8jeGZH0sgjc23PBm8HKf-EMHLp-8Ua52XfrVgPQ,5271
arrow/locales.py,sha256=QSi6FJTVdmxDxAUIDMhUp3sJ13tHhF2tB50fA_mve0I,156276
arrow/parser.py,sha256=ingY4axAO40kEYUL8MwqTIhFegCAVouDZk3c4YOs9aI,25720
arrow/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
arrow/util.py,sha256=xnDevqRyNeYWbl3x-n_Tyo4cOgHcdgbxFECFsJ1XoEc,3679
arrow-1.2.3.dist-info/LICENSE,sha256=QNbhJV1xUfXwQaUUcl08lP-owYgeWgwptr6pPwPi47s,11341
arrow-1.2.3.dist-info/METADATA,sha256=gg8GFdfHjX15F_yMrLsluJwKUXTaQi5ECUvwBlZYe9o,6938
arrow-1.2.3.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
arrow-1.2.3.dist-info/top_level.txt,sha256=aCBThK2RIB824ctI3l9i6z94l8UYpFF-BC4m3dDzFFo,6
arrow-1.2.3.dist-info/RECORD,,
@@ -1 +0,0 @@
arrow
@@ -186,7 +186,7 @@
same "printed page" as the copyright notice for easier same "printed page" as the copyright notice for easier
identification within third-party archives. identification within third-party archives.
Copyright 2021 Chris Smith Copyright 2023 Chris Smith
Licensed under the Apache License, Version 2.0 (the "License"); Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License. you may not use this file except in compliance with the License.
@@ -1,31 +1,43 @@
Metadata-Version: 2.1 Metadata-Version: 2.1
Name: arrow Name: arrow
Version: 1.2.3 Version: 1.3.0
Summary: Better dates & times for Python Summary: Better dates & times for Python
Home-page: https://arrow.readthedocs.io Keywords: arrow,date,time,datetime,timestamp,timezone,humanize
Author: Chris Smith Author-email: Chris Smith <crsmithdev@gmail.com>
Author-email: crsmithdev@gmail.com Requires-Python: >=3.8
License: Apache 2.0 Description-Content-Type: text/x-rst
Project-URL: Repository, https://github.com/arrow-py/arrow
Project-URL: Bug Reports, https://github.com/arrow-py/arrow/issues
Project-URL: Documentation, https://arrow.readthedocs.io
Keywords: arrow date time datetime timestamp timezone humanize
Classifier: Development Status :: 5 - Production/Stable Classifier: Development Status :: 5 - Production/Stable
Classifier: Intended Audience :: Developers Classifier: Intended Audience :: Developers
Classifier: Intended Audience :: Information Technology
Classifier: License :: OSI Approved :: Apache Software License Classifier: License :: OSI Approved :: Apache Software License
Classifier: Topic :: Software Development :: Libraries :: Python Modules Classifier: Topic :: Software Development :: Libraries :: Python Modules
Classifier: Programming Language :: Python :: 3 Classifier: Programming Language :: Python :: 3
Classifier: Programming Language :: Python :: 3 :: Only Classifier: Programming Language :: Python :: 3 :: Only
Classifier: Programming Language :: Python :: 3.6
Classifier: Programming Language :: Python :: 3.7
Classifier: Programming Language :: Python :: 3.8 Classifier: Programming Language :: Python :: 3.8
Classifier: Programming Language :: Python :: 3.9 Classifier: Programming Language :: Python :: 3.9
Classifier: Programming Language :: Python :: 3.10 Classifier: Programming Language :: Python :: 3.10
Requires-Python: >=3.6 Classifier: Programming Language :: Python :: 3.11
Description-Content-Type: text/x-rst Classifier: Programming Language :: Python :: 3.12
License-File: LICENSE Classifier: Operating System :: OS Independent
Requires-Dist: python-dateutil (>=2.7.0) Requires-Dist: python-dateutil>=2.7.0
Requires-Dist: typing-extensions ; python_version < "3.8" Requires-Dist: types-python-dateutil>=2.8.10
Requires-Dist: doc8 ; extra == "doc"
Requires-Dist: sphinx>=7.0.0 ; extra == "doc"
Requires-Dist: sphinx-autobuild ; extra == "doc"
Requires-Dist: sphinx-autodoc-typehints ; extra == "doc"
Requires-Dist: sphinx_rtd_theme>=1.3.0 ; extra == "doc"
Requires-Dist: dateparser==1.* ; extra == "test"
Requires-Dist: pre-commit ; extra == "test"
Requires-Dist: pytest ; extra == "test"
Requires-Dist: pytest-cov ; extra == "test"
Requires-Dist: pytest-mock ; extra == "test"
Requires-Dist: pytz==2021.1 ; extra == "test"
Requires-Dist: simplejson==3.* ; extra == "test"
Project-URL: Documentation, https://arrow.readthedocs.io
Project-URL: Issues, https://github.com/arrow-py/arrow/issues
Project-URL: Source, https://github.com/arrow-py/arrow
Provides-Extra: doc
Provides-Extra: test
Arrow: Better dates & times for Python Arrow: Better dates & times for Python
====================================== ======================================
@@ -161,3 +173,4 @@ Support Arrow
------------- -------------
`Open Collective <https://opencollective.com/>`_ is an online funding platform that provides tools to raise money and share your finances with full transparency. It is the platform of choice for individuals and companies to make one-time or recurring donations directly to the project. If you are interested in making a financial contribution, please visit the `Arrow collective <https://opencollective.com/arrow>`_. `Open Collective <https://opencollective.com/>`_ is an online funding platform that provides tools to raise money and share your finances with full transparency. It is the platform of choice for individuals and companies to make one-time or recurring donations directly to the project. If you are interested in making a financial contribution, please visit the `Arrow collective <https://opencollective.com/arrow>`_.
+15
View File
@@ -0,0 +1,15 @@
arrow/__init__.py,sha256=HxsSJGl56GoeHB__No-kdGmC_Wes_Ttf0ohOy7OoFig,872
arrow/_version.py,sha256=F5mW07pSyGrqDNY2Ehr-UpDzpBtN-FsYU0QGZWf6PJE,22
arrow/api.py,sha256=6tdqrG0NjrKO22_eWHU4a5xerfR6IrZPY-yynGpnvTM,2755
arrow/arrow.py,sha256=m9XvNnpQ1aTHZWXPud3W2-QMfilgWXnUCnuZInwf27g,63517
arrow/constants.py,sha256=y3scgWgxiFuQg4DeFlhmexy1BA7K8LFNZyqK-VWPQJs,3238
arrow/factory.py,sha256=qiDSokfcVWJhiJbIkOcU1Ohh4N0PdKxghsJzBnI8AUo,11432
arrow/formatter.py,sha256=0D0-AjBZwuay9312KvY0UnaVBfAZj-vEIqWcG0_3ZDQ,5267
arrow/locales.py,sha256=6g5xHq5UkIAZPF8N2PvzN_xoUvsfNcPhNfJw0TUi8tw,156894
arrow/parser.py,sha256=FO6NWpzjvZcsMhIck6pd7hKe1ijlKUZE9l_OFlyskyw,25790
arrow/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
arrow/util.py,sha256=xnDevqRyNeYWbl3x-n_Tyo4cOgHcdgbxFECFsJ1XoEc,3679
arrow-1.3.0.dist-info/LICENSE,sha256=tIH4cpbLCr2xP9jLuUsUwyi-iA7J5oVHphuE2s_9Bno,11341
arrow-1.3.0.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
arrow-1.3.0.dist-info/METADATA,sha256=P7gh6Gt6pIqBLBP577OoTZWFhRmAOpMyiwpuNEeklac,7534
arrow-1.3.0.dist-info/RECORD,,
+1 -1
View File
@@ -1 +1 @@
__version__ = "1.2.3" __version__ = "1.3.0"
+7 -24
View File
@@ -168,9 +168,9 @@ class Arrow:
isinstance(tzinfo, dt_tzinfo) isinstance(tzinfo, dt_tzinfo)
and hasattr(tzinfo, "localize") and hasattr(tzinfo, "localize")
and hasattr(tzinfo, "zone") and hasattr(tzinfo, "zone")
and tzinfo.zone # type: ignore[attr-defined] and tzinfo.zone
): ):
tzinfo = parser.TzinfoParser.parse(tzinfo.zone) # type: ignore[attr-defined] tzinfo = parser.TzinfoParser.parse(tzinfo.zone)
elif isinstance(tzinfo, str): elif isinstance(tzinfo, str):
tzinfo = parser.TzinfoParser.parse(tzinfo) tzinfo = parser.TzinfoParser.parse(tzinfo)
@@ -495,7 +495,7 @@ class Arrow:
yield current yield current
values = [getattr(current, f) for f in cls._ATTRS] values = [getattr(current, f) for f in cls._ATTRS]
current = cls(*values, tzinfo=tzinfo).shift( # type: ignore current = cls(*values, tzinfo=tzinfo).shift( # type: ignore[misc]
**{frame_relative: relative_steps} **{frame_relative: relative_steps}
) )
@@ -578,7 +578,7 @@ class Arrow:
for _ in range(3 - len(values)): for _ in range(3 - len(values)):
values.append(1) values.append(1)
floor = self.__class__(*values, tzinfo=self.tzinfo) # type: ignore floor = self.__class__(*values, tzinfo=self.tzinfo) # type: ignore[misc]
if frame_absolute == "week": if frame_absolute == "week":
# if week_start is greater than self.isoweekday() go back one week by setting delta = 7 # if week_start is greater than self.isoweekday() go back one week by setting delta = 7
@@ -792,7 +792,6 @@ class Arrow:
return self._datetime.isoformat() return self._datetime.isoformat()
def __format__(self, formatstr: str) -> str: def __format__(self, formatstr: str) -> str:
if len(formatstr) > 0: if len(formatstr) > 0:
return self.format(formatstr) return self.format(formatstr)
@@ -804,7 +803,6 @@ class Arrow:
# attributes and properties # attributes and properties
def __getattr__(self, name: str) -> int: def __getattr__(self, name: str) -> int:
if name == "week": if name == "week":
return self.isocalendar()[1] return self.isocalendar()[1]
@@ -965,7 +963,6 @@ class Arrow:
absolute_kwargs = {} absolute_kwargs = {}
for key, value in kwargs.items(): for key, value in kwargs.items():
if key in self._ATTRS: if key in self._ATTRS:
absolute_kwargs[key] = value absolute_kwargs[key] = value
elif key in ["week", "quarter"]: elif key in ["week", "quarter"]:
@@ -1022,7 +1019,6 @@ class Arrow:
additional_attrs = ["weeks", "quarters", "weekday"] additional_attrs = ["weeks", "quarters", "weekday"]
for key, value in kwargs.items(): for key, value in kwargs.items():
if key in self._ATTRS_PLURAL or key in additional_attrs: if key in self._ATTRS_PLURAL or key in additional_attrs:
relative_kwargs[key] = value relative_kwargs[key] = value
else: else:
@@ -1259,11 +1255,10 @@ class Arrow:
) )
if trunc(abs(delta)) != 1: if trunc(abs(delta)) != 1:
granularity += "s" # type: ignore granularity += "s" # type: ignore[assignment]
return locale.describe(granularity, delta, only_distance=only_distance) return locale.describe(granularity, delta, only_distance=only_distance)
else: else:
if not granularity: if not granularity:
raise ValueError( raise ValueError(
"Empty granularity list provided. " "Empty granularity list provided. "
@@ -1314,7 +1309,7 @@ class Arrow:
def dehumanize(self, input_string: str, locale: str = "en_us") -> "Arrow": def dehumanize(self, input_string: str, locale: str = "en_us") -> "Arrow":
"""Returns a new :class:`Arrow <arrow.arrow.Arrow>` object, that represents """Returns a new :class:`Arrow <arrow.arrow.Arrow>` object, that represents
the time difference relative to the attrbiutes of the the time difference relative to the attributes of the
:class:`Arrow <arrow.arrow.Arrow>` object. :class:`Arrow <arrow.arrow.Arrow>` object.
:param timestring: a ``str`` representing a humanized relative time. :param timestring: a ``str`` representing a humanized relative time.
@@ -1367,7 +1362,6 @@ class Arrow:
# Search input string for each time unit within locale # Search input string for each time unit within locale
for unit, unit_object in locale_obj.timeframes.items(): for unit, unit_object in locale_obj.timeframes.items():
# Need to check the type of unit_object to create the correct dictionary # Need to check the type of unit_object to create the correct dictionary
if isinstance(unit_object, Mapping): if isinstance(unit_object, Mapping):
strings_to_search = unit_object strings_to_search = unit_object
@@ -1378,7 +1372,6 @@ class Arrow:
# Needs to cycle all through strings as some locales have strings that # Needs to cycle all through strings as some locales have strings that
# could overlap in a regex match, since input validation isn't being performed. # could overlap in a regex match, since input validation isn't being performed.
for time_delta, time_string in strings_to_search.items(): for time_delta, time_string in strings_to_search.items():
# Replace {0} with regex \d representing digits # Replace {0} with regex \d representing digits
search_string = str(time_string) search_string = str(time_string)
search_string = search_string.format(r"\d+") search_string = search_string.format(r"\d+")
@@ -1419,7 +1412,7 @@ class Arrow:
# Assert error if string does not modify any units # Assert error if string does not modify any units
if not any([True for k, v in unit_visited.items() if v]): if not any([True for k, v in unit_visited.items() if v]):
raise ValueError( raise ValueError(
"Input string not valid. Note: Some locales do not support the week granulairty in Arrow. " "Input string not valid. Note: Some locales do not support the week granularity in Arrow. "
"If you are attempting to use the week granularity on an unsupported locale, this could be the cause of this error." "If you are attempting to use the week granularity on an unsupported locale, this could be the cause of this error."
) )
@@ -1718,7 +1711,6 @@ class Arrow:
# math # math
def __add__(self, other: Any) -> "Arrow": def __add__(self, other: Any) -> "Arrow":
if isinstance(other, (timedelta, relativedelta)): if isinstance(other, (timedelta, relativedelta)):
return self.fromdatetime(self._datetime + other, self._datetime.tzinfo) return self.fromdatetime(self._datetime + other, self._datetime.tzinfo)
@@ -1736,7 +1728,6 @@ class Arrow:
pass # pragma: no cover pass # pragma: no cover
def __sub__(self, other: Any) -> Union[timedelta, "Arrow"]: def __sub__(self, other: Any) -> Union[timedelta, "Arrow"]:
if isinstance(other, (timedelta, relativedelta)): if isinstance(other, (timedelta, relativedelta)):
return self.fromdatetime(self._datetime - other, self._datetime.tzinfo) return self.fromdatetime(self._datetime - other, self._datetime.tzinfo)
@@ -1749,7 +1740,6 @@ class Arrow:
return NotImplemented return NotImplemented
def __rsub__(self, other: Any) -> timedelta: def __rsub__(self, other: Any) -> timedelta:
if isinstance(other, dt_datetime): if isinstance(other, dt_datetime):
return other - self._datetime return other - self._datetime
@@ -1758,42 +1748,36 @@ class Arrow:
# comparisons # comparisons
def __eq__(self, other: Any) -> bool: def __eq__(self, other: Any) -> bool:
if not isinstance(other, (Arrow, dt_datetime)): if not isinstance(other, (Arrow, dt_datetime)):
return False return False
return self._datetime == self._get_datetime(other) return self._datetime == self._get_datetime(other)
def __ne__(self, other: Any) -> bool: def __ne__(self, other: Any) -> bool:
if not isinstance(other, (Arrow, dt_datetime)): if not isinstance(other, (Arrow, dt_datetime)):
return True return True
return not self.__eq__(other) return not self.__eq__(other)
def __gt__(self, other: Any) -> bool: def __gt__(self, other: Any) -> bool:
if not isinstance(other, (Arrow, dt_datetime)): if not isinstance(other, (Arrow, dt_datetime)):
return NotImplemented return NotImplemented
return self._datetime > self._get_datetime(other) return self._datetime > self._get_datetime(other)
def __ge__(self, other: Any) -> bool: def __ge__(self, other: Any) -> bool:
if not isinstance(other, (Arrow, dt_datetime)): if not isinstance(other, (Arrow, dt_datetime)):
return NotImplemented return NotImplemented
return self._datetime >= self._get_datetime(other) return self._datetime >= self._get_datetime(other)
def __lt__(self, other: Any) -> bool: def __lt__(self, other: Any) -> bool:
if not isinstance(other, (Arrow, dt_datetime)): if not isinstance(other, (Arrow, dt_datetime)):
return NotImplemented return NotImplemented
return self._datetime < self._get_datetime(other) return self._datetime < self._get_datetime(other)
def __le__(self, other: Any) -> bool: def __le__(self, other: Any) -> bool:
if not isinstance(other, (Arrow, dt_datetime)): if not isinstance(other, (Arrow, dt_datetime)):
return NotImplemented return NotImplemented
@@ -1865,7 +1849,6 @@ class Arrow:
def _get_iteration_params(cls, end: Any, limit: Optional[int]) -> Tuple[Any, int]: def _get_iteration_params(cls, end: Any, limit: Optional[int]) -> Tuple[Any, int]:
"""Sets default end and limit values for range method.""" """Sets default end and limit values for range method."""
if end is None: if end is None:
if limit is None: if limit is None:
raise ValueError("One of 'end' or 'limit' is required.") raise ValueError("One of 'end' or 'limit' is required.")
-3
View File
@@ -267,11 +267,9 @@ class ArrowFactory:
raise TypeError(f"Cannot parse single argument of type {type(arg)!r}.") raise TypeError(f"Cannot parse single argument of type {type(arg)!r}.")
elif arg_count == 2: elif arg_count == 2:
arg_1, arg_2 = args[0], args[1] arg_1, arg_2 = args[0], args[1]
if isinstance(arg_1, datetime): if isinstance(arg_1, datetime):
# (datetime, tzinfo/str) -> fromdatetime @ tzinfo # (datetime, tzinfo/str) -> fromdatetime @ tzinfo
if isinstance(arg_2, (dt_tzinfo, str)): if isinstance(arg_2, (dt_tzinfo, str)):
return self.type.fromdatetime(arg_1, tzinfo=arg_2) return self.type.fromdatetime(arg_1, tzinfo=arg_2)
@@ -281,7 +279,6 @@ class ArrowFactory:
) )
elif isinstance(arg_1, date): elif isinstance(arg_1, date):
# (date, tzinfo/str) -> fromdate @ tzinfo # (date, tzinfo/str) -> fromdate @ tzinfo
if isinstance(arg_2, (dt_tzinfo, str)): if isinstance(arg_2, (dt_tzinfo, str)):
return self.type.fromdate(arg_1, tzinfo=arg_2) return self.type.fromdate(arg_1, tzinfo=arg_2)
-4
View File
@@ -29,7 +29,6 @@ FORMAT_W3C: Final[str] = "YYYY-MM-DD HH:mm:ssZZ"
class DateTimeFormatter: class DateTimeFormatter:
# This pattern matches characters enclosed in square brackets are matched as # This pattern matches characters enclosed in square brackets are matched as
# an atomic group. For more info on atomic groups and how to they are # an atomic group. For more info on atomic groups and how to they are
# emulated in Python's re library, see https://stackoverflow.com/a/13577411/2701578 # emulated in Python's re library, see https://stackoverflow.com/a/13577411/2701578
@@ -41,18 +40,15 @@ class DateTimeFormatter:
locale: locales.Locale locale: locales.Locale
def __init__(self, locale: str = DEFAULT_LOCALE) -> None: def __init__(self, locale: str = DEFAULT_LOCALE) -> None:
self.locale = locales.get_locale(locale) self.locale = locales.get_locale(locale)
def format(cls, dt: datetime, fmt: str) -> str: def format(cls, dt: datetime, fmt: str) -> str:
# FIXME: _format_token() is nullable # FIXME: _format_token() is nullable
return cls._FORMAT_RE.sub( return cls._FORMAT_RE.sub(
lambda m: cast(str, cls._format_token(dt, m.group(0))), fmt lambda m: cast(str, cls._format_token(dt, m.group(0))), fmt
) )
def _format_token(self, dt: datetime, token: Optional[str]) -> Optional[str]: def _format_token(self, dt: datetime, token: Optional[str]) -> Optional[str]:
if token and token.startswith("[") and token.endswith("]"): if token and token.startswith("[") and token.endswith("]"):
return token[1:-1] return token[1:-1]
+22 -12
View File
@@ -129,7 +129,6 @@ class Locale:
_locale_map[locale_name.lower().replace("_", "-")] = cls _locale_map[locale_name.lower().replace("_", "-")] = cls
def __init__(self) -> None: def __init__(self) -> None:
self._month_name_to_ordinal = None self._month_name_to_ordinal = None
def describe( def describe(
@@ -174,7 +173,7 @@ class Locale:
# Needed to determine the correct relative string to use # Needed to determine the correct relative string to use
timeframe_value = 0 timeframe_value = 0
for _unit_name, unit_value in timeframes: for _, unit_value in timeframes:
if trunc(unit_value) != 0: if trunc(unit_value) != 0:
timeframe_value = trunc(unit_value) timeframe_value = trunc(unit_value)
break break
@@ -285,7 +284,6 @@ class Locale:
timeframe: TimeFrameLiteral, timeframe: TimeFrameLiteral,
delta: Union[float, int], delta: Union[float, int],
) -> str: ) -> str:
if timeframe == "now": if timeframe == "now":
return humanized return humanized
@@ -425,7 +423,7 @@ class ItalianLocale(Locale):
"hours": "{0} ore", "hours": "{0} ore",
"day": "un giorno", "day": "un giorno",
"days": "{0} giorni", "days": "{0} giorni",
"week": "una settimana,", "week": "una settimana",
"weeks": "{0} settimane", "weeks": "{0} settimane",
"month": "un mese", "month": "un mese",
"months": "{0} mesi", "months": "{0} mesi",
@@ -867,14 +865,16 @@ class FinnishLocale(Locale):
timeframes: ClassVar[Mapping[TimeFrameLiteral, Union[str, Mapping[str, str]]]] = { timeframes: ClassVar[Mapping[TimeFrameLiteral, Union[str, Mapping[str, str]]]] = {
"now": "juuri nyt", "now": "juuri nyt",
"second": "sekunti", "second": {"past": "sekunti", "future": "sekunnin"},
"seconds": {"past": "{0} muutama sekunti", "future": "{0} muutaman sekunnin"}, "seconds": {"past": "{0} sekuntia", "future": "{0} sekunnin"},
"minute": {"past": "minuutti", "future": "minuutin"}, "minute": {"past": "minuutti", "future": "minuutin"},
"minutes": {"past": "{0} minuuttia", "future": "{0} minuutin"}, "minutes": {"past": "{0} minuuttia", "future": "{0} minuutin"},
"hour": {"past": "tunti", "future": "tunnin"}, "hour": {"past": "tunti", "future": "tunnin"},
"hours": {"past": "{0} tuntia", "future": "{0} tunnin"}, "hours": {"past": "{0} tuntia", "future": "{0} tunnin"},
"day": "päivä", "day": {"past": "päivä", "future": "päivän"},
"days": {"past": "{0} päivää", "future": "{0} päivän"}, "days": {"past": "{0} päivää", "future": "{0} päivän"},
"week": {"past": "viikko", "future": "viikon"},
"weeks": {"past": "{0} viikkoa", "future": "{0} viikon"},
"month": {"past": "kuukausi", "future": "kuukauden"}, "month": {"past": "kuukausi", "future": "kuukauden"},
"months": {"past": "{0} kuukautta", "future": "{0} kuukauden"}, "months": {"past": "{0} kuukautta", "future": "{0} kuukauden"},
"year": {"past": "vuosi", "future": "vuoden"}, "year": {"past": "vuosi", "future": "vuoden"},
@@ -1887,7 +1887,7 @@ class GermanBaseLocale(Locale):
future = "in {0}" future = "in {0}"
and_word = "und" and_word = "und"
timeframes = { timeframes: ClassVar[Dict[TimeFrameLiteral, str]] = {
"now": "gerade eben", "now": "gerade eben",
"second": "einer Sekunde", "second": "einer Sekunde",
"seconds": "{0} Sekunden", "seconds": "{0} Sekunden",
@@ -1982,7 +1982,9 @@ class GermanBaseLocale(Locale):
return super().describe(timeframe, delta, only_distance) return super().describe(timeframe, delta, only_distance)
# German uses a different case without 'in' or 'ago' # German uses a different case without 'in' or 'ago'
humanized = self.timeframes_only_distance[timeframe].format(trunc(abs(delta))) humanized: str = self.timeframes_only_distance[timeframe].format(
trunc(abs(delta))
)
return humanized return humanized
@@ -2547,6 +2549,8 @@ class ArabicLocale(Locale):
"hours": {"2": "ساعتين", "ten": "{0} ساعات", "higher": "{0} ساعة"}, "hours": {"2": "ساعتين", "ten": "{0} ساعات", "higher": "{0} ساعة"},
"day": "يوم", "day": "يوم",
"days": {"2": "يومين", "ten": "{0} أيام", "higher": "{0} يوم"}, "days": {"2": "يومين", "ten": "{0} أيام", "higher": "{0} يوم"},
"week": "اسبوع",
"weeks": {"2": "اسبوعين", "ten": "{0} أسابيع", "higher": "{0} اسبوع"},
"month": "شهر", "month": "شهر",
"months": {"2": "شهرين", "ten": "{0} أشهر", "higher": "{0} شهر"}, "months": {"2": "شهرين", "ten": "{0} أشهر", "higher": "{0} شهر"},
"year": "سنة", "year": "سنة",
@@ -3709,6 +3713,8 @@ class HungarianLocale(Locale):
"hours": {"past": "{0} órával", "future": "{0} óra"}, "hours": {"past": "{0} órával", "future": "{0} óra"},
"day": {"past": "egy nappal", "future": "egy nap"}, "day": {"past": "egy nappal", "future": "egy nap"},
"days": {"past": "{0} nappal", "future": "{0} nap"}, "days": {"past": "{0} nappal", "future": "{0} nap"},
"week": {"past": "egy héttel", "future": "egy hét"},
"weeks": {"past": "{0} héttel", "future": "{0} hét"},
"month": {"past": "egy hónappal", "future": "egy hónap"}, "month": {"past": "egy hónappal", "future": "egy hónap"},
"months": {"past": "{0} hónappal", "future": "{0} hónap"}, "months": {"past": "{0} hónappal", "future": "{0} hónap"},
"year": {"past": "egy évvel", "future": "egy év"}, "year": {"past": "egy évvel", "future": "egy év"},
@@ -3934,7 +3940,6 @@ class ThaiLocale(Locale):
class LaotianLocale(Locale): class LaotianLocale(Locale):
names = ["lo", "lo-la"] names = ["lo", "lo-la"]
past = "{0} ກ່ອນຫນ້ານີ້" past = "{0} ກ່ອນຫນ້ານີ້"
@@ -4119,6 +4124,7 @@ class BengaliLocale(Locale):
return f"{n}র্থ" return f"{n}র্থ"
if n == 6: if n == 6:
return f"{n}ষ্ঠ" return f"{n}ষ্ঠ"
return ""
class RomanshLocale(Locale): class RomanshLocale(Locale):
@@ -4137,6 +4143,8 @@ class RomanshLocale(Locale):
"hours": "{0} ura", "hours": "{0} ura",
"day": "in di", "day": "in di",
"days": "{0} dis", "days": "{0} dis",
"week": "in'emna",
"weeks": "{0} emnas",
"month": "in mais", "month": "in mais",
"months": "{0} mais", "months": "{0} mais",
"year": "in onn", "year": "in onn",
@@ -5399,7 +5407,7 @@ class LuxembourgishLocale(Locale):
future = "an {0}" future = "an {0}"
and_word = "an" and_word = "an"
timeframes = { timeframes: ClassVar[Dict[TimeFrameLiteral, str]] = {
"now": "just elo", "now": "just elo",
"second": "enger Sekonn", "second": "enger Sekonn",
"seconds": "{0} Sekonnen", "seconds": "{0} Sekonnen",
@@ -5487,7 +5495,9 @@ class LuxembourgishLocale(Locale):
return super().describe(timeframe, delta, only_distance) return super().describe(timeframe, delta, only_distance)
# Luxembourgish uses a different case without 'in' or 'ago' # Luxembourgish uses a different case without 'in' or 'ago'
humanized = self.timeframes_only_distance[timeframe].format(trunc(abs(delta))) humanized: str = self.timeframes_only_distance[timeframe].format(
trunc(abs(delta))
)
return humanized return humanized
+6 -14
View File
@@ -159,7 +159,6 @@ class DateTimeParser:
_input_re_map: Dict[_FORMAT_TYPE, Pattern[str]] _input_re_map: Dict[_FORMAT_TYPE, Pattern[str]]
def __init__(self, locale: str = DEFAULT_LOCALE, cache_size: int = 0) -> None: def __init__(self, locale: str = DEFAULT_LOCALE, cache_size: int = 0) -> None:
self.locale = locales.get_locale(locale) self.locale = locales.get_locale(locale)
self._input_re_map = self._BASE_INPUT_RE_MAP.copy() self._input_re_map = self._BASE_INPUT_RE_MAP.copy()
self._input_re_map.update( self._input_re_map.update(
@@ -196,7 +195,6 @@ class DateTimeParser:
def parse_iso( def parse_iso(
self, datetime_string: str, normalize_whitespace: bool = False self, datetime_string: str, normalize_whitespace: bool = False
) -> datetime: ) -> datetime:
if normalize_whitespace: if normalize_whitespace:
datetime_string = re.sub(r"\s+", " ", datetime_string.strip()) datetime_string = re.sub(r"\s+", " ", datetime_string.strip())
@@ -236,13 +234,14 @@ class DateTimeParser:
] ]
if has_time: if has_time:
if has_space_divider: if has_space_divider:
date_string, time_string = datetime_string.split(" ", 1) date_string, time_string = datetime_string.split(" ", 1)
else: else:
date_string, time_string = datetime_string.split("T", 1) date_string, time_string = datetime_string.split("T", 1)
time_parts = re.split(r"[\+\-Z]", time_string, 1, re.IGNORECASE) time_parts = re.split(
r"[\+\-Z]", time_string, maxsplit=1, flags=re.IGNORECASE
)
time_components: Optional[Match[str]] = self._TIME_RE.match(time_parts[0]) time_components: Optional[Match[str]] = self._TIME_RE.match(time_parts[0])
@@ -303,7 +302,6 @@ class DateTimeParser:
fmt: Union[List[str], str], fmt: Union[List[str], str],
normalize_whitespace: bool = False, normalize_whitespace: bool = False,
) -> datetime: ) -> datetime:
if normalize_whitespace: if normalize_whitespace:
datetime_string = re.sub(r"\s+", " ", datetime_string) datetime_string = re.sub(r"\s+", " ", datetime_string)
@@ -341,12 +339,11 @@ class DateTimeParser:
f"Unable to find a match group for the specified token {token!r}." f"Unable to find a match group for the specified token {token!r}."
) )
self._parse_token(token, value, parts) # type: ignore self._parse_token(token, value, parts) # type: ignore[arg-type]
return self._build_datetime(parts) return self._build_datetime(parts)
def _generate_pattern_re(self, fmt: str) -> Tuple[List[_FORMAT_TYPE], Pattern[str]]: def _generate_pattern_re(self, fmt: str) -> Tuple[List[_FORMAT_TYPE], Pattern[str]]:
# fmt is a string of tokens like 'YYYY-MM-DD' # fmt is a string of tokens like 'YYYY-MM-DD'
# we construct a new string by replacing each # we construct a new string by replacing each
# token by its pattern: # token by its pattern:
@@ -498,7 +495,6 @@ class DateTimeParser:
value: Any, value: Any,
parts: _Parts, parts: _Parts,
) -> None: ) -> None:
if token == "YYYY": if token == "YYYY":
parts["year"] = int(value) parts["year"] = int(value)
@@ -508,7 +504,7 @@ class DateTimeParser:
elif token in ["MMMM", "MMM"]: elif token in ["MMMM", "MMM"]:
# FIXME: month_number() is nullable # FIXME: month_number() is nullable
parts["month"] = self.locale.month_number(value.lower()) # type: ignore parts["month"] = self.locale.month_number(value.lower()) # type: ignore[typeddict-item]
elif token in ["MM", "M"]: elif token in ["MM", "M"]:
parts["month"] = int(value) parts["month"] = int(value)
@@ -588,7 +584,6 @@ class DateTimeParser:
weekdate = parts.get("weekdate") weekdate = parts.get("weekdate")
if weekdate is not None: if weekdate is not None:
year, week = int(weekdate[0]), int(weekdate[1]) year, week = int(weekdate[0]), int(weekdate[1])
if weekdate[2] is not None: if weekdate[2] is not None:
@@ -712,7 +707,6 @@ class DateTimeParser:
) )
def _parse_multiformat(self, string: str, formats: Iterable[str]) -> datetime: def _parse_multiformat(self, string: str, formats: Iterable[str]) -> datetime:
_datetime: Optional[datetime] = None _datetime: Optional[datetime] = None
for fmt in formats: for fmt in formats:
@@ -740,12 +734,11 @@ class DateTimeParser:
class TzinfoParser: class TzinfoParser:
_TZINFO_RE: ClassVar[Pattern[str]] = re.compile( _TZINFO_RE: ClassVar[Pattern[str]] = re.compile(
r"^([\+\-])?(\d{2})(?:\:?(\d{2}))?$" r"^(?:\(UTC)*([\+\-])?(\d{2})(?:\:?(\d{2}))?"
) )
@classmethod @classmethod
def parse(cls, tzinfo_string: str) -> dt_tzinfo: def parse(cls, tzinfo_string: str) -> dt_tzinfo:
tzinfo: Optional[dt_tzinfo] = None tzinfo: Optional[dt_tzinfo] = None
if tzinfo_string == "local": if tzinfo_string == "local":
@@ -755,7 +748,6 @@ class TzinfoParser:
tzinfo = tz.tzutc() tzinfo = tz.tzutc()
else: else:
iso_match = cls._TZINFO_RE.match(tzinfo_string) iso_match = cls._TZINFO_RE.match(tzinfo_string)
if iso_match: if iso_match:
@@ -1,6 +1,6 @@
Metadata-Version: 2.1 Metadata-Version: 2.1
Name: build Name: build
Version: 1.2.1 Version: 1.2.2
Summary: A simple, correct Python build frontend Summary: A simple, correct Python build frontend
Author-email: Filipe Laíns <lains@riseup.net>, Bernát Gábor <gaborjbernat@gmail.com>, layday <layday@protonmail.com>, Henry Schreiner <henryschreineriii@gmail.com> Author-email: Filipe Laíns <lains@riseup.net>, Bernát Gábor <gaborjbernat@gmail.com>, layday <layday@protonmail.com>, Henry Schreiner <henryschreineriii@gmail.com>
Requires-Python: >= 3.8 Requires-Python: >= 3.8
@@ -86,10 +86,67 @@ This will build the package in an isolated environment, generating a
source-distribution and wheel in the directory `dist/`. source-distribution and wheel in the directory `dist/`.
See the [documentation](https://build.pypa.io) for full information. See the [documentation](https://build.pypa.io) for full information.
### Common arguments
- `--sdist` (`-s`): Produce just an SDist
- `--wheel` (`-w`): Produce just a wheel
- `-C<option>=<value>`: A Config-setting, the PEP 517 way of passing options to a backend. Can be passed multiple times. Matching options will make a list. Note that setuptools has very limited support.
- `--installer`: Pick an installer for the isolated build (`pip` or `uv`).
- `--no-isolation` (`-n`): Disable build isolation.
- `--skip-dependency-check` (`-x`): Disable dependency checking when not isolated; this should be done if some requirements or version ranges are not required for non-isolated builds.
- `--outdir (`-o`): The output directory (defaults to `dist`)
Some common combinations of arguments:
- `--sdist --wheel` (`-sw`): Produce and SDist and a wheel, both from the source distribution. The default (if no flag is passed) is to build an SDist and then build a wheel _from_ the SDist.
- `-nx`: Disable build isolation and dependency checking. Identical to pip and uv's `--no-build-isolation` flag.
### Integration with other tools
#### pipx
If you use [pipx][], such as in GitHub Actions, the following command will download
and run build in one step:
```console
$ pipx run build
```
#### uv
If you want to use [uv][] to speed up the virtual environment creation, you can use
`--installer=uv`. You can get a Python wheel for `uv` with the `[uv]` extra.
Combining both suggestions yields the following:
```console
$ pipx run build[uv] --installer=uv
```
#### cibuildwheel
If you are using [cibuildwheel][], build is integrated and can be use with either (in your `pyproject.toml`):
```toml
[tool.cibuildwheel]
build-frontend = "build"
```
or
```toml
[tool.cibuildwheel]
build-frontend = "build[uv]"
```
(Be sure to pre-install uv before running cibuildwheel for this one!)
### Code of Conduct ### Code of Conduct
Everyone interacting in the build's codebase, issue trackers, chat rooms, and mailing lists is expected to follow Everyone interacting in the build's codebase, issue trackers, chat rooms, and mailing lists is expected to follow
the [PSF Code of Conduct]. the [PSF Code of Conduct].
[psf code of conduct]: https://github.com/pypa/.github/blob/main/CODE_OF_CONDUCT.md [psf code of conduct]: https://github.com/pypa/.github/blob/main/CODE_OF_CONDUCT.md
[pipx]: https://pipx.pypa.io
[uv]: https://docs.astral.sh/uv/
[cibuildwheel]: https://cibuildwheel.pypa.io
@@ -1,9 +1,9 @@
build/__init__.py,sha256=YRN82R3egs0NvKG5RqK8VdmuNLun9CQ6XLrTT11r5xw,837 build/__init__.py,sha256=V-ksyUNV9Nat75ELXGMJnIWKtvMOU9ZUhd_pwe0hFAY,837
build/__main__.py,sha256=pvzlY6-r8VXHSD3h7SgANITl5vDV1ZMdEPx8n_LysuA,14227 build/__main__.py,sha256=pvzlY6-r8VXHSD3h7SgANITl5vDV1ZMdEPx8n_LysuA,14227
build/_builder.py,sha256=9WmH11ybX7HS5CeLNRhqWxPzimJJsr9dam_OGpt0SNk,13393 build/_builder.py,sha256=Z2uDHZxil43HLfhZ6KKc3qjd5ZNiJfK1Zu6WgJLJTA4,13469
build/_ctx.py,sha256=QEBzQYYT2Buyj0eHkTKIQ3GK3LEV6rEC-2iRiuU4sT8,2880 build/_ctx.py,sha256=QEBzQYYT2Buyj0eHkTKIQ3GK3LEV6rEC-2iRiuU4sT8,2880
build/_exceptions.py,sha256=lxEYr42eG-wgfLbcMhlks4cc0u-P3zMM8sbWraINwI4,1625 build/_exceptions.py,sha256=lxEYr42eG-wgfLbcMhlks4cc0u-P3zMM8sbWraINwI4,1625
build/_types.py,sha256=DujHH4_tdW5uvOU-0vwhfZNy1h3C2MVVMHYq134a_CA,372 build/_types.py,sha256=if2-I_0F1HGWhhufZqZiBDvAd_niCMSKaHAmWHUIseg,673
build/_util.py,sha256=D4xFxoA4PT27xalv9a-5flCtZgZHNFaKpiIM2XpdM9s,2325 build/_util.py,sha256=D4xFxoA4PT27xalv9a-5flCtZgZHNFaKpiIM2XpdM9s,2325
build/env.py,sha256=lDe3HfqNr4BI9mMFsGYp4YbF_IJyFBfO74OftF81KmM,13473 build/env.py,sha256=lDe3HfqNr4BI9mMFsGYp4YbF_IJyFBfO74OftF81KmM,13473
build/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 build/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -12,8 +12,8 @@ build/_compat/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
build/_compat/importlib.py,sha256=mdUYYn5ciavBQ6zFE-2ghFDotKZC4QufbKMyPg5h3WI,467 build/_compat/importlib.py,sha256=mdUYYn5ciavBQ6zFE-2ghFDotKZC4QufbKMyPg5h3WI,467
build/_compat/tarfile.py,sha256=8-sDnbG-9mZ5OsmSVPArxE6KI6dPe0wX2yf4etDSqxk,816 build/_compat/tarfile.py,sha256=8-sDnbG-9mZ5OsmSVPArxE6KI6dPe0wX2yf4etDSqxk,816
build/_compat/tomllib.py,sha256=JT-q9fBu_vOt4fPWKh2bIKvfko633z_74F_TP_Y17cs,255 build/_compat/tomllib.py,sha256=JT-q9fBu_vOt4fPWKh2bIKvfko633z_74F_TP_Y17cs,255
build-1.2.1.dist-info/entry_points.txt,sha256=MYbbM6JGYubgIjTgcLpNIZpc0PWbi9mRAW7VhtczlS8,105 build-1.2.2.dist-info/entry_points.txt,sha256=MYbbM6JGYubgIjTgcLpNIZpc0PWbi9mRAW7VhtczlS8,105
build-1.2.1.dist-info/LICENSE,sha256=qvminKWQeXHM8H3gJTdds0U5qNXu684gtGCZgFciEG8,1113 build-1.2.2.dist-info/LICENSE,sha256=qvminKWQeXHM8H3gJTdds0U5qNXu684gtGCZgFciEG8,1113
build-1.2.1.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81 build-1.2.2.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
build-1.2.1.dist-info/METADATA,sha256=n3E4RpE_KOQIGMEg9zAYZEcLPAOHMmmn_hfP8z_Nyek,4324 build-1.2.2.dist-info/METADATA,sha256=Cf_qA3nK0sUX0iRcfjfRoOFJmMTdQr9xbVin70BoPXw,6246
build-1.2.1.dist-info/RECORD,, build-1.2.2.dist-info/RECORD,,
+4
View File
@@ -0,0 +1,4 @@
Wheel-Version: 1.0
Generator: flit 3.9.0
Root-Is-Purelib: true
Tag: py3-none-any
+1 -1
View File
@@ -18,7 +18,7 @@ from ._types import SubprocessRunner as RunnerType
from ._util import check_dependency from ._util import check_dependency
__version__ = '1.2.1' __version__ = '1.2.2'
__all__ = [ __all__ = [
'__version__', '__version__',
+11 -3
View File
@@ -116,7 +116,9 @@ def _parse_build_system_table(pyproject_toml: Mapping[str, Any]) -> Mapping[str,
def _wrap_subprocess_runner(runner: SubprocessRunner, env: env.IsolatedEnv) -> SubprocessRunner: def _wrap_subprocess_runner(runner: SubprocessRunner, env: env.IsolatedEnv) -> SubprocessRunner:
def _invoke_wrapped_runner(cmd: Sequence[str], cwd: str | None, extra_environ: Mapping[str, str] | None) -> None: def _invoke_wrapped_runner(
cmd: Sequence[str], cwd: str | None = None, extra_environ: Mapping[str, str] | None = None
) -> None:
runner(cmd, cwd, {**(env.make_extra_environ() or {}), **(extra_environ or {})}) runner(cmd, cwd, {**(env.make_extra_environ() or {}), **(extra_environ or {})})
return _invoke_wrapped_runner return _invoke_wrapped_runner
@@ -203,7 +205,11 @@ class ProjectBuilder:
""" """
return set(self._build_system['requires']) return set(self._build_system['requires'])
def get_requires_for_build(self, distribution: Distribution, config_settings: ConfigSettings | None = None) -> set[str]: def get_requires_for_build(
self,
distribution: Distribution,
config_settings: ConfigSettings | None = None,
) -> set[str]:
""" """
Return the dependencies defined by the backend in addition to Return the dependencies defined by the backend in addition to
:attr:`build_system_requires` for a given distribution. :attr:`build_system_requires` for a given distribution.
@@ -220,7 +226,9 @@ class ProjectBuilder:
return set(get_requires(config_settings)) return set(get_requires(config_settings))
def check_dependencies( def check_dependencies(
self, distribution: Distribution, config_settings: ConfigSettings | None = None self,
distribution: Distribution,
config_settings: ConfigSettings | None = None,
) -> set[tuple[str, ...]]: ) -> set[tuple[str, ...]]:
""" """
Return the dependencies which are not satisfied from the combined set of Return the dependencies which are not satisfied from the combined set of
+16 -5
View File
@@ -1,12 +1,23 @@
from __future__ import annotations from __future__ import annotations
import os import os
import sys
import typing import typing
__all__ = ['ConfigSettings', 'Distribution', 'StrPath', 'SubprocessRunner']
ConfigSettings = typing.Mapping[str, typing.Union[str, typing.Sequence[str]]] ConfigSettings = typing.Mapping[str, typing.Union[str, typing.Sequence[str]]]
Distribution = typing.Literal['sdist', 'wheel'] Distribution = typing.Literal['sdist', 'wheel', 'editable']
StrPath = typing.Union[str, 'os.PathLike[str]']
SubprocessRunner = typing.Callable[ if typing.TYPE_CHECKING or sys.version_info > (3, 9):
[typing.Sequence[str], typing.Optional[str], typing.Optional[typing.Mapping[str, str]]], None StrPath = typing.Union[str, os.PathLike[str]]
] else:
StrPath = typing.Union[str, os.PathLike]
if typing.TYPE_CHECKING:
from pyproject_hooks import SubprocessRunner
else:
SubprocessRunner = typing.Callable[
[typing.Sequence[str], typing.Optional[str], typing.Optional[typing.Mapping[str, str]]], None
]
@@ -1,49 +0,0 @@
chardet/__init__.py,sha256=mWZaWmvZkhwfBEAT9O1Y6nRTfKzhT7FHhQTTAujbqUA,3271
chardet/big5freq.py,sha256=D_zK5GyzoVsRes0HkLJziltFQX0bKCLOrFe9_xDvO_8,31254
chardet/big5prober.py,sha256=kBxHbdetBpPe7xrlb-e990iot64g_eGSLd32lB7_h3M,1757
chardet/chardistribution.py,sha256=3woWS62KrGooKyqz4zQSnjFbJpa6V7g02daAibTwcl8,9411
chardet/charsetgroupprober.py,sha256=GZLReHP6FRRn43hvSOoGCxYamErKzyp6RgOQxVeC3kg,3839
chardet/charsetprober.py,sha256=KSmwJErjypyj0bRZmC5F5eM7c8YQgLYIjZXintZNstg,5110
chardet/codingstatemachine.py,sha256=VYp_6cyyki5sHgXDSZnXW4q1oelHc3cu9AyQTX7uug8,3590
chardet/compat.py,sha256=40zr6wICZwknxyuLGGcIOPyve8DTebBCbbvttvnmp5Q,1200
chardet/cp949prober.py,sha256=TZ434QX8zzBsnUvL_8wm4AQVTZ2ZkqEEQL_lNw9f9ow,1855
chardet/enums.py,sha256=Aimwdb9as1dJKZaFNUH2OhWIVBVd6ZkJJ_WK5sNY8cU,1661
chardet/escprober.py,sha256=kkyqVg1Yw3DIOAMJ2bdlyQgUFQhuHAW8dUGskToNWSc,3950
chardet/escsm.py,sha256=RuXlgNvTIDarndvllNCk5WZBIpdCxQ0kcd9EAuxUh84,10510
chardet/eucjpprober.py,sha256=iD8Jdp0ISRjgjiVN7f0e8xGeQJ5GM2oeZ1dA8nbSeUw,3749
chardet/euckrfreq.py,sha256=-7GdmvgWez4-eO4SuXpa7tBiDi5vRXQ8WvdFAzVaSfo,13546
chardet/euckrprober.py,sha256=MqFMTQXxW4HbzIpZ9lKDHB3GN8SP4yiHenTmf8g_PxY,1748
chardet/euctwfreq.py,sha256=No1WyduFOgB5VITUA7PLyC5oJRNzRyMbBxaKI1l16MA,31621
chardet/euctwprober.py,sha256=13p6EP4yRaxqnP4iHtxHOJ6R2zxHq1_m8hTRjzVZ95c,1747
chardet/gb2312freq.py,sha256=JX8lsweKLmnCwmk8UHEQsLgkr_rP_kEbvivC4qPOrlc,20715
chardet/gb2312prober.py,sha256=gGvIWi9WhDjE-xQXHvNIyrnLvEbMAYgyUSZ65HUfylw,1754
chardet/hebrewprober.py,sha256=c3SZ-K7hvyzGY6JRAZxJgwJ_sUS9k0WYkvMY00YBYFo,13838
chardet/jisfreq.py,sha256=vpmJv2Bu0J8gnMVRPHMFefTRvo_ha1mryLig8CBwgOg,25777
chardet/jpcntx.py,sha256=PYlNqRUQT8LM3cT5FmHGP0iiscFlTWED92MALvBungo,19643
chardet/langbulgarianmodel.py,sha256=r6tvOtO8FqhnbWBB5V4czcl1fWM4pB9lGiWQU-8gvsw,105685
chardet/langgreekmodel.py,sha256=1cMu2wUgPB8bQ2RbVjR4LNwCCETgQ-Dwo0Eg2_uB11s,99559
chardet/langhebrewmodel.py,sha256=urMmJHHIXtCwaWAqy1zEY_4SmwwNzt730bDOtjXzRjs,98764
chardet/langhungarianmodel.py,sha256=ODAisvqCfes8B4FeyM_Pg9HY3ZDnEyaCiT4Bxyzoc6w,102486
chardet/langrussianmodel.py,sha256=sPqkrBbX0QVwwy6oqRl-x7ERv2J4-zaMoCvLpkSsSJI,131168
chardet/langthaimodel.py,sha256=ppoKOGL9OPdj9A4CUyG8R48zbnXt9MN1WXeCYepa6sc,103300
chardet/langturkishmodel.py,sha256=H3ldicI_rhlv0r3VFpVWtUL6X30Wy596v7_YHz2sEdg,95934
chardet/latin1prober.py,sha256=S2IoORhFk39FEFOlSFWtgVybRiP6h7BlLldHVclNkU8,5370
chardet/mbcharsetprober.py,sha256=AR95eFH9vuqSfvLQZN-L5ijea25NOBCoXqw8s5O9xLQ,3413
chardet/mbcsgroupprober.py,sha256=h6TRnnYq2OxG1WdD5JOyxcdVpn7dG0q-vB8nWr5mbh4,2012
chardet/mbcssm.py,sha256=SY32wVIF3HzcjY3BaEspy9metbNSKxIIB0RKPn7tjpI,25481
chardet/sbcharsetprober.py,sha256=nmyMyuxzG87DN6K3Rk2MUzJLMLR69MrWpdnHzOwVUwQ,6136
chardet/sbcsgroupprober.py,sha256=hqefQuXmiFyDBArOjujH6hd6WFXlOD1kWCsxDhjx5Vc,4309
chardet/sjisprober.py,sha256=IIt-lZj0WJqK4rmUZzKZP4GJlE8KUEtFYVuY96ek5MQ,3774
chardet/universaldetector.py,sha256=DpZTXCX0nUHXxkQ9sr4GZxGB_hveZ6hWt3uM94cgWKs,12503
chardet/utf8prober.py,sha256=IdD8v3zWOsB8OLiyPi-y_fqwipRFxV9Nc1eKBLSuIEw,2766
chardet/version.py,sha256=A4CILFAd8MRVG1HoXPp45iK9RLlWyV73a1EtwE8Tvn8,242
chardet/cli/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
chardet/cli/chardetect.py,sha256=kUPeQCi-olObXpOq5MtlKuBn1EU19rkeenAMwxl7URY,2711
chardet/metadata/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
chardet/metadata/languages.py,sha256=41tLq3eLSrBEbEVVQpVGFq9K7o1ln9b1HpY1l0hCUQo,19474
chardet-4.0.0.dist-info/LICENSE,sha256=YJXp_6d33SKDn3gBqoRbMcntB_PWv4om3F0t7IzMDvM,26432
chardet-4.0.0.dist-info/METADATA,sha256=ySYQAE7NPm3LwxgMqFi1zdLQ48mmwMbrJwqAWCtcbH8,3526
chardet-4.0.0.dist-info/WHEEL,sha256=ADKeyaGyKF5DwBNE0sRE5pvW-bSkFMJfBuhzZ3rceP4,110
chardet-4.0.0.dist-info/entry_points.txt,sha256=fAMmhu5eJ-zAJ-smfqQwRClQ3-nozOCmvJ6-E8lgGJo,60
chardet-4.0.0.dist-info/top_level.txt,sha256=AowzBbZy4x8EirABDdJSLJZMkJ_53iIag8xfKR6D7kI,8
chardet-4.0.0.dist-info/RECORD,,
@@ -1,8 +1,8 @@
GNU LESSER GENERAL PUBLIC LICENSE GNU LESSER GENERAL PUBLIC LICENSE
Version 2.1, February 1999 Version 2.1, February 1999
Copyright (C) 1991, 1999 Free Software Foundation, Inc. Copyright (C) 1991, 1999 Free Software Foundation, Inc.
51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
Everyone is permitted to copy and distribute verbatim copies Everyone is permitted to copy and distribute verbatim copies
of this license document, but changing it is not allowed. of this license document, but changing it is not allowed.
@@ -10,7 +10,7 @@
as the successor of the GNU Library Public License, version 2, hence as the successor of the GNU Library Public License, version 2, hence
the version number 2.1.] the version number 2.1.]
Preamble Preamble
The licenses for most software are designed to take away your The licenses for most software are designed to take away your
freedom to share and change it. By contrast, the GNU General Public freedom to share and change it. By contrast, the GNU General Public
@@ -112,7 +112,7 @@ modification follow. Pay close attention to the difference between a
former contains code derived from the library, whereas the latter must former contains code derived from the library, whereas the latter must
be combined with the library in order to run. be combined with the library in order to run.
GNU LESSER GENERAL PUBLIC LICENSE GNU LESSER GENERAL PUBLIC LICENSE
TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
0. This License Agreement applies to any software library or other 0. This License Agreement applies to any software library or other
@@ -146,7 +146,7 @@ such a program is covered only if its contents constitute a work based
on the Library (independent of the use of the Library in a tool for on the Library (independent of the use of the Library in a tool for
writing it). Whether that is true depends on what the Library does writing it). Whether that is true depends on what the Library does
and what the program that uses the Library does. and what the program that uses the Library does.
1. You may copy and distribute verbatim copies of the Library's 1. You may copy and distribute verbatim copies of the Library's
complete source code as you receive it, in any medium, provided that complete source code as you receive it, in any medium, provided that
you conspicuously and appropriately publish on each copy an you conspicuously and appropriately publish on each copy an
@@ -432,7 +432,7 @@ decision will be guided by the two goals of preserving the free status
of all derivatives of our free software and of promoting the sharing of all derivatives of our free software and of promoting the sharing
and reuse of software generally. and reuse of software generally.
NO WARRANTY NO WARRANTY
15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO 15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO
WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW. WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW.
@@ -455,7 +455,7 @@ FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF
SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH
DAMAGES. DAMAGES.
END OF TERMS AND CONDITIONS END OF TERMS AND CONDITIONS
How to Apply These Terms to Your New Libraries How to Apply These Terms to Your New Libraries
@@ -485,7 +485,7 @@ convey the exclusion of warranty; and each file should have at least the
You should have received a copy of the GNU Lesser General Public You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
Also add information on how to contact you by electronic and paper mail. Also add information on how to contact you by electronic and paper mail.
@@ -500,5 +500,3 @@ necessary. Here is a sample; alter the names:
Ty Coon, President of Vice Ty Coon, President of Vice
That's all there is to it! That's all there is to it!
@@ -1,33 +1,34 @@
Metadata-Version: 2.1 Metadata-Version: 2.1
Name: chardet Name: chardet
Version: 4.0.0 Version: 5.2.0
Summary: Universal encoding detector for Python 2 and 3 Summary: Universal encoding detector for Python 3
Home-page: https://github.com/chardet/chardet Home-page: https://github.com/chardet/chardet
Author: Mark Pilgrim Author: Mark Pilgrim
Author-email: mark@diveintomark.org Author-email: mark@diveintomark.org
Maintainer: Daniel Blanchard Maintainer: Daniel Blanchard
Maintainer-email: dan.blanchard@gmail.com Maintainer-email: dan.blanchard@gmail.com
License: LGPL License: LGPL
Project-URL: Documentation, https://chardet.readthedocs.io/
Project-URL: GitHub Project, https://github.com/chardet/chardet
Project-URL: Issue Tracker, https://github.com/chardet/chardet/issues
Keywords: encoding,i18n,xml Keywords: encoding,i18n,xml
Platform: UNKNOWN
Classifier: Development Status :: 5 - Production/Stable Classifier: Development Status :: 5 - Production/Stable
Classifier: Intended Audience :: Developers Classifier: Intended Audience :: Developers
Classifier: License :: OSI Approved :: GNU Library or Lesser General Public License (LGPL) Classifier: License :: OSI Approved :: GNU Lesser General Public License v2 or later (LGPLv2+)
Classifier: Operating System :: OS Independent Classifier: Operating System :: OS Independent
Classifier: Programming Language :: Python Classifier: Programming Language :: Python
Classifier: Programming Language :: Python :: 2
Classifier: Programming Language :: Python :: 2.7
Classifier: Programming Language :: Python :: 3 Classifier: Programming Language :: Python :: 3
Classifier: Programming Language :: Python :: 3.5
Classifier: Programming Language :: Python :: 3.6
Classifier: Programming Language :: Python :: 3.7 Classifier: Programming Language :: Python :: 3.7
Classifier: Programming Language :: Python :: 3.8 Classifier: Programming Language :: Python :: 3.8
Classifier: Programming Language :: Python :: 3.9 Classifier: Programming Language :: Python :: 3.9
Classifier: Programming Language :: Python :: 3.10
Classifier: Programming Language :: Python :: 3.11
Classifier: Programming Language :: Python :: Implementation :: CPython Classifier: Programming Language :: Python :: Implementation :: CPython
Classifier: Programming Language :: Python :: Implementation :: PyPy Classifier: Programming Language :: Python :: Implementation :: PyPy
Classifier: Topic :: Software Development :: Libraries :: Python Modules Classifier: Topic :: Software Development :: Libraries :: Python Modules
Classifier: Topic :: Text Processing :: Linguistic Classifier: Topic :: Text Processing :: Linguistic
Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.* Requires-Python: >=3.7
License-File: LICENSE
Chardet: The Universal Character Encoding Detector Chardet: The Universal Character Encoding Detector
-------------------------------------------------- --------------------------------------------------
@@ -51,10 +52,10 @@ Detects
- ASCII, UTF-8, UTF-16 (2 variants), UTF-32 (4 variants) - ASCII, UTF-8, UTF-16 (2 variants), UTF-32 (4 variants)
- Big5, GB2312, EUC-TW, HZ-GB-2312, ISO-2022-CN (Traditional and Simplified Chinese) - Big5, GB2312, EUC-TW, HZ-GB-2312, ISO-2022-CN (Traditional and Simplified Chinese)
- EUC-JP, SHIFT_JIS, CP932, ISO-2022-JP (Japanese) - EUC-JP, SHIFT_JIS, CP932, ISO-2022-JP (Japanese)
- EUC-KR, ISO-2022-KR (Korean) - EUC-KR, ISO-2022-KR, Johab (Korean)
- KOI8-R, MacCyrillic, IBM855, IBM866, ISO-8859-5, windows-1251 (Cyrillic) - KOI8-R, MacCyrillic, IBM855, IBM866, ISO-8859-5, windows-1251 (Cyrillic)
- ISO-8859-5, windows-1251 (Bulgarian) - ISO-8859-5, windows-1251 (Bulgarian)
- ISO-8859-1, windows-1252 (Western European languages) - ISO-8859-1, windows-1252, MacRoman (Western European languages)
- ISO-8859-7, windows-1253 (Greek) - ISO-8859-7, windows-1253 (Greek)
- ISO-8859-8, windows-1255 (Visual and Logical Hebrew) - ISO-8859-8, windows-1255 (Visual and Logical Hebrew)
- TIS-620 (Thai) - TIS-620 (Thai)
@@ -63,7 +64,7 @@ Detects
Our ISO-8859-2 and windows-1250 (Hungarian) probers have been temporarily Our ISO-8859-2 and windows-1250 (Hungarian) probers have been temporarily
disabled until we can retrain the models. disabled until we can retrain the models.
Requires Python 2.7 or 3.5+. Requires Python 3.7+.
Installation Installation
------------ ------------
@@ -90,12 +91,7 @@ or more files::
About About
----- -----
This is a continuation of Mark Pilgrim's excellent chardet. Previously, two This is a continuation of Mark Pilgrim's excellent original chardet port from C, and `Ian Cordasco <https://github.com/sigmavirus24>`_'s
versions needed to be maintained: one that supported python 2.x and one that `charade <https://github.com/sigmavirus24/charade>`_ Python 3-compatible fork.
supported python 3.x. We've recently merged with `Ian Cordasco <https://github.com/sigmavirus24>`_'s
`charade <https://github.com/sigmavirus24/charade>`_ fork, so now we have one
coherent version that works for Python 2.7+ and 3.4+.
:maintainer: Dan Blanchard :maintainer: Dan Blanchard
@@ -0,0 +1,56 @@
chardet/__init__.py,sha256=57R-HSxj0PWmILMN0GFmUNqEMfrEVSamXyjD-W6_fbs,4797
chardet/__main__.py,sha256=puNj2o_QfBRKElEkiVp1zEIL1gGYD2o-JuXLFlqHDC4,123
chardet/big5freq.py,sha256=ltcfP-3PjlNHCoo5e4a7C4z-2DhBTXRfY6jbMbB7P30,31274
chardet/big5prober.py,sha256=lPMfwCX6v2AaPgvFh_cSWZcgLDbWiFCHLZ_p9RQ9uxE,1763
chardet/chardistribution.py,sha256=13B8XUG4oXDuLdXvfbIWwLFeR-ZU21AqTS1zcdON8bU,10032
chardet/charsetgroupprober.py,sha256=UKK3SaIZB2PCdKSIS0gnvMtLR9JJX62M-fZJu3OlWyg,3915
chardet/charsetprober.py,sha256=L3t8_wIOov8em-vZWOcbkdsrwe43N6_gqNh5pH7WPd4,5420
chardet/codingstatemachine.py,sha256=K7k69sw3jY5DmTXoSJQVsUtFIQKYPQVOSJJhBuGv_yE,3732
chardet/codingstatemachinedict.py,sha256=0GY3Hi2qIZvDrOOJ3AtqppM1RsYxr_66ER4EHjuMiMc,542
chardet/cp949prober.py,sha256=0jKRV7fECuWI16rNnks0ZECKA1iZYCIEaP8A1ZvjUSI,1860
chardet/enums.py,sha256=TzECiZoCKNMqgwU76cPCeKWFBqaWvAdLMev5_bCkhY8,1683
chardet/escprober.py,sha256=Kho48X65xE0scFylIdeJjM2bcbvRvv0h0WUbMWrJD3A,4006
chardet/escsm.py,sha256=AqyXpA2FQFD7k-buBty_7itGEYkhmVa8X09NLRul3QM,12176
chardet/eucjpprober.py,sha256=5KYaM9fsxkRYzw1b5k0fL-j_-ezIw-ij9r97a9MHxLY,3934
chardet/euckrfreq.py,sha256=3mHuRvXfsq_QcQysDQFb8qSudvTiol71C6Ic2w57tKM,13566
chardet/euckrprober.py,sha256=hiFT6wM174GIwRvqDsIcuOc-dDsq2uPKMKbyV8-1Xnc,1753
chardet/euctwfreq.py,sha256=2alILE1Lh5eqiFJZjzRkMQXolNJRHY5oBQd-vmZYFFM,36913
chardet/euctwprober.py,sha256=NxbpNdBtU0VFI0bKfGfDkpP7S2_8_6FlO87dVH0ogws,1753
chardet/gb2312freq.py,sha256=49OrdXzD-HXqwavkqjo8Z7gvs58hONNzDhAyMENNkvY,20735
chardet/gb2312prober.py,sha256=KPEBueaSLSvBpFeINMu0D6TgHcR90e5PaQawifzF4o0,1759
chardet/hebrewprober.py,sha256=96T_Lj_OmW-fK7JrSHojYjyG3fsGgbzkoTNleZ3kfYE,14537
chardet/jisfreq.py,sha256=mm8tfrwqhpOd3wzZKS4NJqkYBQVcDfTM2JiQ5aW932E,25796
chardet/johabfreq.py,sha256=dBpOYG34GRX6SL8k_LbS9rxZPMjLjoMlgZ03Pz5Hmqc,42498
chardet/johabprober.py,sha256=O1Qw9nVzRnun7vZp4UZM7wvJSv9W941mEU9uDMnY3DU,1752
chardet/jpcntx.py,sha256=uhHrYWkLxE_rF5OkHKInm0HUsrjgKHHVQvtt3UcvotA,27055
chardet/langbulgarianmodel.py,sha256=bGoRpxBYtrbSHa6mX6PkEA26v30pWmhDjemhdxmkew8,104550
chardet/langgreekmodel.py,sha256=3wMlEzQ8oU2MbrL2xN8lkuOB0dCMLBhW6heekxusoc0,98472
chardet/langhebrewmodel.py,sha256=ZUTqusxMvR_earWPs5w-rH10xoe5sPjd9FLMu1DUIvE,98184
chardet/langhungarianmodel.py,sha256=N-YtC2EiswyS7XsUicCPRycrIzRNj47Y048odp9qOoo,101351
chardet/langrussianmodel.py,sha256=6v7RcZKGj0VH0864BHzizKNceAYbHvGts2p00ifC7w4,128023
chardet/langthaimodel.py,sha256=Mr673U9U8rkQFfUDtLP01pp-0TOsl2o6sb75YEjvpcs,102762
chardet/langturkishmodel.py,sha256=LkXCjWhGUEzqKXvfasHN0SFBigwKJ3xeWNVZ0EyI0kA,95360
chardet/latin1prober.py,sha256=p15EEmFbmQUwbKLC7lOJVGHEZwcG45ubEZYTGu01J5g,5380
chardet/macromanprober.py,sha256=9anfzmY6TBfUPDyBDOdY07kqmTHpZ1tK0jL-p1JWcOY,6077
chardet/mbcharsetprober.py,sha256=Wr04WNI4F3X_VxEverNG-H25g7u-MDDKlNt-JGj-_uU,3715
chardet/mbcsgroupprober.py,sha256=iRpaNBjV0DNwYPu_z6TiHgRpwYahiM7ztI_4kZ4Uz9A,2131
chardet/mbcssm.py,sha256=hUtPvDYgWDaA2dWdgLsshbwRfm3Q5YRlRogdmeRUNQw,30391
chardet/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
chardet/resultdict.py,sha256=ez4FRvN5KaSosJeJ2WzUyKdDdg35HDy_SSLPXKCdt5M,402
chardet/sbcharsetprober.py,sha256=-nd3F90i7GpXLjehLVHqVBE0KlWzGvQUPETLBNn4o6U,6400
chardet/sbcsgroupprober.py,sha256=gcgI0fOfgw_3YTClpbra_MNxwyEyJ3eUXraoLHYb59E,4137
chardet/sjisprober.py,sha256=aqQufMzRw46ZpFlzmYaYeT2-nzmKb-hmcrApppJ862k,4007
chardet/universaldetector.py,sha256=xYBrg4x0dd9WnT8qclfADVD9ondrUNkqPmvte1pa520,14848
chardet/utf1632prober.py,sha256=pw1epGdMj1hDGiCu1AHqqzOEfjX8MVdiW7O1BlT8-eQ,8505
chardet/utf8prober.py,sha256=8m08Ub5490H4jQ6LYXvFysGtgKoKsHUd2zH_i8_TnVw,2812
chardet/version.py,sha256=jp8ePp1zC63YxruGcHSuKxtf3-fF1LYAMUZD2eDWYok,244
chardet/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
chardet/cli/chardetect.py,sha256=zibMVg5RpKb-ME9_7EYG4ZM2Sf07NHcQzZ12U-rYJho,3242
chardet/metadata/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
chardet/metadata/languages.py,sha256=FhvBIdZFxRQ-dTwkb_0madRKgVBCaUMQz9I5xqjE5iQ,13560
chardet-5.2.0.dist-info/LICENSE,sha256=3GJlINzVOiL3J68-5Cx3DlbJemT-OtsGN5nYqwMv5VE,26530
chardet-5.2.0.dist-info/METADATA,sha256=PAr2NQ6hQWpjyFnwlI7MoxHt2S_6oRiUsucOKMNhzGw,3418
chardet-5.2.0.dist-info/WHEEL,sha256=AtBG6SXL3KF_v0NxLf0ehyVOh0cold-JbJYXNGorC6Q,92
chardet-5.2.0.dist-info/entry_points.txt,sha256=_cdvYc4jyY68GYfsQAAthNMxO-yodcGkvNC1xOEsLmI,59
chardet-5.2.0.dist-info/top_level.txt,sha256=AowzBbZy4x8EirABDdJSLJZMkJ_53iIag8xfKR6D7kI,8
chardet-5.2.0.dist-info/RECORD,,
@@ -1,5 +1,5 @@
Wheel-Version: 1.0 Wheel-Version: 1.0
Generator: bdist_wheel (0.38.4) Generator: bdist_wheel (0.41.0)
Root-Is-Purelib: true Root-Is-Purelib: true
Tag: py3-none-any Tag: py3-none-any
@@ -1,3 +1,2 @@
[console_scripts] [console_scripts]
chardetect = chardet.cli.chardetect:main chardetect = chardet.cli.chardetect:main
+67 -35
View File
@@ -15,69 +15,101 @@
# 02110-1301 USA # 02110-1301 USA
######################### END LICENSE BLOCK ######################### ######################### END LICENSE BLOCK #########################
from typing import List, Union
from .universaldetector import UniversalDetector from .charsetgroupprober import CharSetGroupProber
from .charsetprober import CharSetProber
from .enums import InputState from .enums import InputState
from .version import __version__, VERSION from .resultdict import ResultDict
from .universaldetector import UniversalDetector
from .version import VERSION, __version__
__all__ = ["UniversalDetector", "detect", "detect_all", "__version__", "VERSION"]
__all__ = ['UniversalDetector', 'detect', 'detect_all', '__version__', 'VERSION'] def detect(
byte_str: Union[bytes, bytearray], should_rename_legacy: bool = False
) -> ResultDict:
def detect(byte_str):
""" """
Detect the encoding of the given byte string. Detect the encoding of the given byte string.
:param byte_str: The byte sequence to examine. :param byte_str: The byte sequence to examine.
:type byte_str: ``bytes`` or ``bytearray`` :type byte_str: ``bytes`` or ``bytearray``
:param should_rename_legacy: Should we rename legacy encodings
to their more modern equivalents?
:type should_rename_legacy: ``bool``
""" """
if not isinstance(byte_str, bytearray): if not isinstance(byte_str, bytearray):
if not isinstance(byte_str, bytes): if not isinstance(byte_str, bytes):
raise TypeError('Expected object of type bytes or bytearray, got: ' raise TypeError(
'{}'.format(type(byte_str))) f"Expected object of type bytes or bytearray, got: {type(byte_str)}"
else: )
byte_str = bytearray(byte_str) byte_str = bytearray(byte_str)
detector = UniversalDetector() detector = UniversalDetector(should_rename_legacy=should_rename_legacy)
detector.feed(byte_str) detector.feed(byte_str)
return detector.close() return detector.close()
def detect_all(byte_str): def detect_all(
byte_str: Union[bytes, bytearray],
ignore_threshold: bool = False,
should_rename_legacy: bool = False,
) -> List[ResultDict]:
""" """
Detect all the possible encodings of the given byte string. Detect all the possible encodings of the given byte string.
:param byte_str: The byte sequence to examine. :param byte_str: The byte sequence to examine.
:type byte_str: ``bytes`` or ``bytearray`` :type byte_str: ``bytes`` or ``bytearray``
:param ignore_threshold: Include encodings that are below
``UniversalDetector.MINIMUM_THRESHOLD``
in results.
:type ignore_threshold: ``bool``
:param should_rename_legacy: Should we rename legacy encodings
to their more modern equivalents?
:type should_rename_legacy: ``bool``
""" """
if not isinstance(byte_str, bytearray): if not isinstance(byte_str, bytearray):
if not isinstance(byte_str, bytes): if not isinstance(byte_str, bytes):
raise TypeError('Expected object of type bytes or bytearray, got: ' raise TypeError(
'{}'.format(type(byte_str))) f"Expected object of type bytes or bytearray, got: {type(byte_str)}"
else: )
byte_str = bytearray(byte_str) byte_str = bytearray(byte_str)
detector = UniversalDetector() detector = UniversalDetector(should_rename_legacy=should_rename_legacy)
detector.feed(byte_str) detector.feed(byte_str)
detector.close() detector.close()
if detector._input_state == InputState.HIGH_BYTE: if detector.input_state == InputState.HIGH_BYTE:
results = [] results: List[ResultDict] = []
for prober in detector._charset_probers: probers: List[CharSetProber] = []
if prober.get_confidence() > detector.MINIMUM_THRESHOLD: for prober in detector.charset_probers:
charset_name = prober.charset_name if isinstance(prober, CharSetGroupProber):
lower_charset_name = prober.charset_name.lower() probers.extend(p for p in prober.probers)
else:
probers.append(prober)
for prober in probers:
if ignore_threshold or prober.get_confidence() > detector.MINIMUM_THRESHOLD:
charset_name = prober.charset_name or ""
lower_charset_name = charset_name.lower()
# Use Windows encoding name instead of ISO-8859 if we saw any # Use Windows encoding name instead of ISO-8859 if we saw any
# extra Windows-specific bytes # extra Windows-specific bytes
if lower_charset_name.startswith('iso-8859'): if lower_charset_name.startswith("iso-8859") and detector.has_win_bytes:
if detector._has_win_bytes: charset_name = detector.ISO_WIN_MAP.get(
charset_name = detector.ISO_WIN_MAP.get(lower_charset_name, lower_charset_name, charset_name
charset_name) )
results.append({ # Rename legacy encodings with superset encodings if asked
'encoding': charset_name, if should_rename_legacy:
'confidence': prober.get_confidence(), charset_name = detector.LEGACY_MAP.get(
'language': prober.language, charset_name.lower(), charset_name
}) )
results.append(
{
"encoding": charset_name,
"confidence": prober.get_confidence(),
"language": prober.language,
}
)
if len(results) > 0: if len(results) > 0:
return sorted(results, key=lambda result: -result['confidence']) return sorted(results, key=lambda result: -result["confidence"])
return [detector.result] return [detector.result]
+6
View File
@@ -0,0 +1,6 @@
"""Wrapper so people can run python -m chardet"""
from .cli.chardetect import main
if __name__ == "__main__":
main()
+3 -3
View File
@@ -42,9 +42,9 @@
BIG5_TYPICAL_DISTRIBUTION_RATIO = 0.75 BIG5_TYPICAL_DISTRIBUTION_RATIO = 0.75
#Char to FreqOrder table # Char to FreqOrder table
BIG5_TABLE_SIZE = 5376 BIG5_TABLE_SIZE = 5376
# fmt: off
BIG5_CHAR_TO_FREQ_ORDER = ( BIG5_CHAR_TO_FREQ_ORDER = (
1,1801,1506, 255,1431, 198, 9, 82, 6,5008, 177, 202,3681,1256,2821, 110, # 16 1,1801,1506, 255,1431, 198, 9, 82, 6,5008, 177, 202,3681,1256,2821, 110, # 16
3814, 33,3274, 261, 76, 44,2114, 16,2946,2187,1176, 659,3971, 26,3451,2653, # 32 3814, 33,3274, 261, 76, 44,2114, 16,2946,2187,1176, 659,3971, 26,3451,2653, # 32
@@ -383,4 +383,4 @@ BIG5_CHAR_TO_FREQ_ORDER = (
890,3669,3943,5791,1878,3798,3439,5792,2186,2358,3440,1652,5793,5794,5795, 941, # 5360 890,3669,3943,5791,1878,3798,3439,5792,2186,2358,3440,1652,5793,5794,5795, 941, # 5360
2299, 208,3546,4161,2020, 330,4438,3944,2906,2499,3799,4439,4811,5796,5797,5798, # 5376 2299, 208,3546,4161,2020, 330,4438,3944,2906,2499,3799,4439,4811,5796,5797,5798, # 5376
) )
# fmt: on
+6 -6
View File
@@ -25,23 +25,23 @@
# 02110-1301 USA # 02110-1301 USA
######################### END LICENSE BLOCK ######################### ######################### END LICENSE BLOCK #########################
from .mbcharsetprober import MultiByteCharSetProber
from .codingstatemachine import CodingStateMachine
from .chardistribution import Big5DistributionAnalysis from .chardistribution import Big5DistributionAnalysis
from .codingstatemachine import CodingStateMachine
from .mbcharsetprober import MultiByteCharSetProber
from .mbcssm import BIG5_SM_MODEL from .mbcssm import BIG5_SM_MODEL
class Big5Prober(MultiByteCharSetProber): class Big5Prober(MultiByteCharSetProber):
def __init__(self): def __init__(self) -> None:
super(Big5Prober, self).__init__() super().__init__()
self.coding_sm = CodingStateMachine(BIG5_SM_MODEL) self.coding_sm = CodingStateMachine(BIG5_SM_MODEL)
self.distribution_analyzer = Big5DistributionAnalysis() self.distribution_analyzer = Big5DistributionAnalysis()
self.reset() self.reset()
@property @property
def charset_name(self): def charset_name(self) -> str:
return "Big5" return "Big5"
@property @property
def language(self): def language(self) -> str:
return "Chinese" return "Chinese"
+86 -58
View File
@@ -25,40 +25,58 @@
# 02110-1301 USA # 02110-1301 USA
######################### END LICENSE BLOCK ######################### ######################### END LICENSE BLOCK #########################
from .euctwfreq import (EUCTW_CHAR_TO_FREQ_ORDER, EUCTW_TABLE_SIZE, from typing import Tuple, Union
EUCTW_TYPICAL_DISTRIBUTION_RATIO)
from .euckrfreq import (EUCKR_CHAR_TO_FREQ_ORDER, EUCKR_TABLE_SIZE, from .big5freq import (
EUCKR_TYPICAL_DISTRIBUTION_RATIO) BIG5_CHAR_TO_FREQ_ORDER,
from .gb2312freq import (GB2312_CHAR_TO_FREQ_ORDER, GB2312_TABLE_SIZE, BIG5_TABLE_SIZE,
GB2312_TYPICAL_DISTRIBUTION_RATIO) BIG5_TYPICAL_DISTRIBUTION_RATIO,
from .big5freq import (BIG5_CHAR_TO_FREQ_ORDER, BIG5_TABLE_SIZE, )
BIG5_TYPICAL_DISTRIBUTION_RATIO) from .euckrfreq import (
from .jisfreq import (JIS_CHAR_TO_FREQ_ORDER, JIS_TABLE_SIZE, EUCKR_CHAR_TO_FREQ_ORDER,
JIS_TYPICAL_DISTRIBUTION_RATIO) EUCKR_TABLE_SIZE,
EUCKR_TYPICAL_DISTRIBUTION_RATIO,
)
from .euctwfreq import (
EUCTW_CHAR_TO_FREQ_ORDER,
EUCTW_TABLE_SIZE,
EUCTW_TYPICAL_DISTRIBUTION_RATIO,
)
from .gb2312freq import (
GB2312_CHAR_TO_FREQ_ORDER,
GB2312_TABLE_SIZE,
GB2312_TYPICAL_DISTRIBUTION_RATIO,
)
from .jisfreq import (
JIS_CHAR_TO_FREQ_ORDER,
JIS_TABLE_SIZE,
JIS_TYPICAL_DISTRIBUTION_RATIO,
)
from .johabfreq import JOHAB_TO_EUCKR_ORDER_TABLE
class CharDistributionAnalysis(object): class CharDistributionAnalysis:
ENOUGH_DATA_THRESHOLD = 1024 ENOUGH_DATA_THRESHOLD = 1024
SURE_YES = 0.99 SURE_YES = 0.99
SURE_NO = 0.01 SURE_NO = 0.01
MINIMUM_DATA_THRESHOLD = 3 MINIMUM_DATA_THRESHOLD = 3
def __init__(self): def __init__(self) -> None:
# Mapping table to get frequency order from char order (get from # Mapping table to get frequency order from char order (get from
# GetOrder()) # GetOrder())
self._char_to_freq_order = None self._char_to_freq_order: Tuple[int, ...] = tuple()
self._table_size = None # Size of above table self._table_size = 0 # Size of above table
# This is a constant value which varies from language to language, # This is a constant value which varies from language to language,
# used in calculating confidence. See # used in calculating confidence. See
# http://www.mozilla.org/projects/intl/UniversalCharsetDetection.html # http://www.mozilla.org/projects/intl/UniversalCharsetDetection.html
# for further detail. # for further detail.
self.typical_distribution_ratio = None self.typical_distribution_ratio = 0.0
self._done = None self._done = False
self._total_chars = None self._total_chars = 0
self._freq_chars = None self._freq_chars = 0
self.reset() self.reset()
def reset(self): def reset(self) -> None:
"""reset analyser, clear any state""" """reset analyser, clear any state"""
# If this flag is set to True, detection is done and conclusion has # If this flag is set to True, detection is done and conclusion has
# been made # been made
@@ -67,7 +85,7 @@ class CharDistributionAnalysis(object):
# The number of characters whose frequency order is less than 512 # The number of characters whose frequency order is less than 512
self._freq_chars = 0 self._freq_chars = 0
def feed(self, char, char_len): def feed(self, char: Union[bytes, bytearray], char_len: int) -> None:
"""feed a character with known length""" """feed a character with known length"""
if char_len == 2: if char_len == 2:
# we only care about 2-bytes character in our distribution analysis # we only care about 2-bytes character in our distribution analysis
@@ -81,7 +99,7 @@ class CharDistributionAnalysis(object):
if 512 > self._char_to_freq_order[order]: if 512 > self._char_to_freq_order[order]:
self._freq_chars += 1 self._freq_chars += 1
def get_confidence(self): def get_confidence(self) -> float:
"""return confidence based on existing data""" """return confidence based on existing data"""
# if we didn't receive any character in our consideration range, # if we didn't receive any character in our consideration range,
# return negative answer # return negative answer
@@ -89,20 +107,21 @@ class CharDistributionAnalysis(object):
return self.SURE_NO return self.SURE_NO
if self._total_chars != self._freq_chars: if self._total_chars != self._freq_chars:
r = (self._freq_chars / ((self._total_chars - self._freq_chars) r = self._freq_chars / (
* self.typical_distribution_ratio)) (self._total_chars - self._freq_chars) * self.typical_distribution_ratio
)
if r < self.SURE_YES: if r < self.SURE_YES:
return r return r
# normalize confidence (we don't want to be 100% sure) # normalize confidence (we don't want to be 100% sure)
return self.SURE_YES return self.SURE_YES
def got_enough_data(self): def got_enough_data(self) -> bool:
# It is not necessary to receive all data to draw conclusion. # It is not necessary to receive all data to draw conclusion.
# For charset detection, certain amount of data is enough # For charset detection, certain amount of data is enough
return self._total_chars > self.ENOUGH_DATA_THRESHOLD return self._total_chars > self.ENOUGH_DATA_THRESHOLD
def get_order(self, byte_str): def get_order(self, _: Union[bytes, bytearray]) -> int:
# We do not handle characters based on the original encoding string, # We do not handle characters based on the original encoding string,
# but convert this encoding string to a number, here called order. # but convert this encoding string to a number, here called order.
# This allows multiple encodings of a language to share one frequency # This allows multiple encodings of a language to share one frequency
@@ -111,13 +130,13 @@ class CharDistributionAnalysis(object):
class EUCTWDistributionAnalysis(CharDistributionAnalysis): class EUCTWDistributionAnalysis(CharDistributionAnalysis):
def __init__(self): def __init__(self) -> None:
super(EUCTWDistributionAnalysis, self).__init__() super().__init__()
self._char_to_freq_order = EUCTW_CHAR_TO_FREQ_ORDER self._char_to_freq_order = EUCTW_CHAR_TO_FREQ_ORDER
self._table_size = EUCTW_TABLE_SIZE self._table_size = EUCTW_TABLE_SIZE
self.typical_distribution_ratio = EUCTW_TYPICAL_DISTRIBUTION_RATIO self.typical_distribution_ratio = EUCTW_TYPICAL_DISTRIBUTION_RATIO
def get_order(self, byte_str): def get_order(self, byte_str: Union[bytes, bytearray]) -> int:
# for euc-TW encoding, we are interested # for euc-TW encoding, we are interested
# first byte range: 0xc4 -- 0xfe # first byte range: 0xc4 -- 0xfe
# second byte range: 0xa1 -- 0xfe # second byte range: 0xa1 -- 0xfe
@@ -125,18 +144,17 @@ class EUCTWDistributionAnalysis(CharDistributionAnalysis):
first_char = byte_str[0] first_char = byte_str[0]
if first_char >= 0xC4: if first_char >= 0xC4:
return 94 * (first_char - 0xC4) + byte_str[1] - 0xA1 return 94 * (first_char - 0xC4) + byte_str[1] - 0xA1
else: return -1
return -1
class EUCKRDistributionAnalysis(CharDistributionAnalysis): class EUCKRDistributionAnalysis(CharDistributionAnalysis):
def __init__(self): def __init__(self) -> None:
super(EUCKRDistributionAnalysis, self).__init__() super().__init__()
self._char_to_freq_order = EUCKR_CHAR_TO_FREQ_ORDER self._char_to_freq_order = EUCKR_CHAR_TO_FREQ_ORDER
self._table_size = EUCKR_TABLE_SIZE self._table_size = EUCKR_TABLE_SIZE
self.typical_distribution_ratio = EUCKR_TYPICAL_DISTRIBUTION_RATIO self.typical_distribution_ratio = EUCKR_TYPICAL_DISTRIBUTION_RATIO
def get_order(self, byte_str): def get_order(self, byte_str: Union[bytes, bytearray]) -> int:
# for euc-KR encoding, we are interested # for euc-KR encoding, we are interested
# first byte range: 0xb0 -- 0xfe # first byte range: 0xb0 -- 0xfe
# second byte range: 0xa1 -- 0xfe # second byte range: 0xa1 -- 0xfe
@@ -144,18 +162,32 @@ class EUCKRDistributionAnalysis(CharDistributionAnalysis):
first_char = byte_str[0] first_char = byte_str[0]
if first_char >= 0xB0: if first_char >= 0xB0:
return 94 * (first_char - 0xB0) + byte_str[1] - 0xA1 return 94 * (first_char - 0xB0) + byte_str[1] - 0xA1
else: return -1
return -1
class JOHABDistributionAnalysis(CharDistributionAnalysis):
def __init__(self) -> None:
super().__init__()
self._char_to_freq_order = EUCKR_CHAR_TO_FREQ_ORDER
self._table_size = EUCKR_TABLE_SIZE
self.typical_distribution_ratio = EUCKR_TYPICAL_DISTRIBUTION_RATIO
def get_order(self, byte_str: Union[bytes, bytearray]) -> int:
first_char = byte_str[0]
if 0x88 <= first_char < 0xD4:
code = first_char * 256 + byte_str[1]
return JOHAB_TO_EUCKR_ORDER_TABLE.get(code, -1)
return -1
class GB2312DistributionAnalysis(CharDistributionAnalysis): class GB2312DistributionAnalysis(CharDistributionAnalysis):
def __init__(self): def __init__(self) -> None:
super(GB2312DistributionAnalysis, self).__init__() super().__init__()
self._char_to_freq_order = GB2312_CHAR_TO_FREQ_ORDER self._char_to_freq_order = GB2312_CHAR_TO_FREQ_ORDER
self._table_size = GB2312_TABLE_SIZE self._table_size = GB2312_TABLE_SIZE
self.typical_distribution_ratio = GB2312_TYPICAL_DISTRIBUTION_RATIO self.typical_distribution_ratio = GB2312_TYPICAL_DISTRIBUTION_RATIO
def get_order(self, byte_str): def get_order(self, byte_str: Union[bytes, bytearray]) -> int:
# for GB2312 encoding, we are interested # for GB2312 encoding, we are interested
# first byte range: 0xb0 -- 0xfe # first byte range: 0xb0 -- 0xfe
# second byte range: 0xa1 -- 0xfe # second byte range: 0xa1 -- 0xfe
@@ -163,18 +195,17 @@ class GB2312DistributionAnalysis(CharDistributionAnalysis):
first_char, second_char = byte_str[0], byte_str[1] first_char, second_char = byte_str[0], byte_str[1]
if (first_char >= 0xB0) and (second_char >= 0xA1): if (first_char >= 0xB0) and (second_char >= 0xA1):
return 94 * (first_char - 0xB0) + second_char - 0xA1 return 94 * (first_char - 0xB0) + second_char - 0xA1
else: return -1
return -1
class Big5DistributionAnalysis(CharDistributionAnalysis): class Big5DistributionAnalysis(CharDistributionAnalysis):
def __init__(self): def __init__(self) -> None:
super(Big5DistributionAnalysis, self).__init__() super().__init__()
self._char_to_freq_order = BIG5_CHAR_TO_FREQ_ORDER self._char_to_freq_order = BIG5_CHAR_TO_FREQ_ORDER
self._table_size = BIG5_TABLE_SIZE self._table_size = BIG5_TABLE_SIZE
self.typical_distribution_ratio = BIG5_TYPICAL_DISTRIBUTION_RATIO self.typical_distribution_ratio = BIG5_TYPICAL_DISTRIBUTION_RATIO
def get_order(self, byte_str): def get_order(self, byte_str: Union[bytes, bytearray]) -> int:
# for big5 encoding, we are interested # for big5 encoding, we are interested
# first byte range: 0xa4 -- 0xfe # first byte range: 0xa4 -- 0xfe
# second byte range: 0x40 -- 0x7e , 0xa1 -- 0xfe # second byte range: 0x40 -- 0x7e , 0xa1 -- 0xfe
@@ -183,28 +214,26 @@ class Big5DistributionAnalysis(CharDistributionAnalysis):
if first_char >= 0xA4: if first_char >= 0xA4:
if second_char >= 0xA1: if second_char >= 0xA1:
return 157 * (first_char - 0xA4) + second_char - 0xA1 + 63 return 157 * (first_char - 0xA4) + second_char - 0xA1 + 63
else: return 157 * (first_char - 0xA4) + second_char - 0x40
return 157 * (first_char - 0xA4) + second_char - 0x40 return -1
else:
return -1
class SJISDistributionAnalysis(CharDistributionAnalysis): class SJISDistributionAnalysis(CharDistributionAnalysis):
def __init__(self): def __init__(self) -> None:
super(SJISDistributionAnalysis, self).__init__() super().__init__()
self._char_to_freq_order = JIS_CHAR_TO_FREQ_ORDER self._char_to_freq_order = JIS_CHAR_TO_FREQ_ORDER
self._table_size = JIS_TABLE_SIZE self._table_size = JIS_TABLE_SIZE
self.typical_distribution_ratio = JIS_TYPICAL_DISTRIBUTION_RATIO self.typical_distribution_ratio = JIS_TYPICAL_DISTRIBUTION_RATIO
def get_order(self, byte_str): def get_order(self, byte_str: Union[bytes, bytearray]) -> int:
# for sjis encoding, we are interested # for sjis encoding, we are interested
# first byte range: 0x81 -- 0x9f , 0xe0 -- 0xfe # first byte range: 0x81 -- 0x9f , 0xe0 -- 0xfe
# second byte range: 0x40 -- 0x7e, 0x81 -- oxfe # second byte range: 0x40 -- 0x7e, 0x81 -- oxfe
# no validation needed here. State machine has done that # no validation needed here. State machine has done that
first_char, second_char = byte_str[0], byte_str[1] first_char, second_char = byte_str[0], byte_str[1]
if (first_char >= 0x81) and (first_char <= 0x9F): if 0x81 <= first_char <= 0x9F:
order = 188 * (first_char - 0x81) order = 188 * (first_char - 0x81)
elif (first_char >= 0xE0) and (first_char <= 0xEF): elif 0xE0 <= first_char <= 0xEF:
order = 188 * (first_char - 0xE0 + 31) order = 188 * (first_char - 0xE0 + 31)
else: else:
return -1 return -1
@@ -215,19 +244,18 @@ class SJISDistributionAnalysis(CharDistributionAnalysis):
class EUCJPDistributionAnalysis(CharDistributionAnalysis): class EUCJPDistributionAnalysis(CharDistributionAnalysis):
def __init__(self): def __init__(self) -> None:
super(EUCJPDistributionAnalysis, self).__init__() super().__init__()
self._char_to_freq_order = JIS_CHAR_TO_FREQ_ORDER self._char_to_freq_order = JIS_CHAR_TO_FREQ_ORDER
self._table_size = JIS_TABLE_SIZE self._table_size = JIS_TABLE_SIZE
self.typical_distribution_ratio = JIS_TYPICAL_DISTRIBUTION_RATIO self.typical_distribution_ratio = JIS_TYPICAL_DISTRIBUTION_RATIO
def get_order(self, byte_str): def get_order(self, byte_str: Union[bytes, bytearray]) -> int:
# for euc-JP encoding, we are interested # for euc-JP encoding, we are interested
# first byte range: 0xa0 -- 0xfe # first byte range: 0xa0 -- 0xfe
# second byte range: 0xa1 -- 0xfe # second byte range: 0xa1 -- 0xfe
# no validation needed here. State machine has done that # no validation needed here. State machine has done that
char = byte_str[0] char = byte_str[0]
if char >= 0xA0: if char >= 0xA0:
return 94 * (char - 0xA1) + byte_str[1] - 0xa1 return 94 * (char - 0xA1) + byte_str[1] - 0xA1
else: return -1
return -1
+22 -23
View File
@@ -25,29 +25,30 @@
# 02110-1301 USA # 02110-1301 USA
######################### END LICENSE BLOCK ######################### ######################### END LICENSE BLOCK #########################
from .enums import ProbingState from typing import List, Optional, Union
from .charsetprober import CharSetProber from .charsetprober import CharSetProber
from .enums import LanguageFilter, ProbingState
class CharSetGroupProber(CharSetProber): class CharSetGroupProber(CharSetProber):
def __init__(self, lang_filter=None): def __init__(self, lang_filter: LanguageFilter = LanguageFilter.NONE) -> None:
super(CharSetGroupProber, self).__init__(lang_filter=lang_filter) super().__init__(lang_filter=lang_filter)
self._active_num = 0 self._active_num = 0
self.probers = [] self.probers: List[CharSetProber] = []
self._best_guess_prober = None self._best_guess_prober: Optional[CharSetProber] = None
def reset(self): def reset(self) -> None:
super(CharSetGroupProber, self).reset() super().reset()
self._active_num = 0 self._active_num = 0
for prober in self.probers: for prober in self.probers:
if prober: prober.reset()
prober.reset() prober.active = True
prober.active = True self._active_num += 1
self._active_num += 1
self._best_guess_prober = None self._best_guess_prober = None
@property @property
def charset_name(self): def charset_name(self) -> Optional[str]:
if not self._best_guess_prober: if not self._best_guess_prober:
self.get_confidence() self.get_confidence()
if not self._best_guess_prober: if not self._best_guess_prober:
@@ -55,17 +56,15 @@ class CharSetGroupProber(CharSetProber):
return self._best_guess_prober.charset_name return self._best_guess_prober.charset_name
@property @property
def language(self): def language(self) -> Optional[str]:
if not self._best_guess_prober: if not self._best_guess_prober:
self.get_confidence() self.get_confidence()
if not self._best_guess_prober: if not self._best_guess_prober:
return None return None
return self._best_guess_prober.language return self._best_guess_prober.language
def feed(self, byte_str): def feed(self, byte_str: Union[bytes, bytearray]) -> ProbingState:
for prober in self.probers: for prober in self.probers:
if not prober:
continue
if not prober.active: if not prober.active:
continue continue
state = prober.feed(byte_str) state = prober.feed(byte_str)
@@ -75,7 +74,7 @@ class CharSetGroupProber(CharSetProber):
self._best_guess_prober = prober self._best_guess_prober = prober
self._state = ProbingState.FOUND_IT self._state = ProbingState.FOUND_IT
return self.state return self.state
elif state == ProbingState.NOT_ME: if state == ProbingState.NOT_ME:
prober.active = False prober.active = False
self._active_num -= 1 self._active_num -= 1
if self._active_num <= 0: if self._active_num <= 0:
@@ -83,22 +82,22 @@ class CharSetGroupProber(CharSetProber):
return self.state return self.state
return self.state return self.state
def get_confidence(self): def get_confidence(self) -> float:
state = self.state state = self.state
if state == ProbingState.FOUND_IT: if state == ProbingState.FOUND_IT:
return 0.99 return 0.99
elif state == ProbingState.NOT_ME: if state == ProbingState.NOT_ME:
return 0.01 return 0.01
best_conf = 0.0 best_conf = 0.0
self._best_guess_prober = None self._best_guess_prober = None
for prober in self.probers: for prober in self.probers:
if not prober:
continue
if not prober.active: if not prober.active:
self.logger.debug('%s not active', prober.charset_name) self.logger.debug("%s not active", prober.charset_name)
continue continue
conf = prober.get_confidence() conf = prober.get_confidence()
self.logger.debug('%s %s confidence = %s', prober.charset_name, prober.language, conf) self.logger.debug(
"%s %s confidence = %s", prober.charset_name, prober.language, conf
)
if best_conf < conf: if best_conf < conf:
best_conf = conf best_conf = conf
self._best_guess_prober = prober self._best_guess_prober = prober
+38 -36
View File
@@ -28,54 +28,62 @@
import logging import logging
import re import re
from typing import Optional, Union
from .enums import ProbingState from .enums import LanguageFilter, ProbingState
INTERNATIONAL_WORDS_PATTERN = re.compile(
b"[a-zA-Z]*[\x80-\xFF]+[a-zA-Z]*[^a-zA-Z\x80-\xFF]?"
)
class CharSetProber(object): class CharSetProber:
SHORTCUT_THRESHOLD = 0.95 SHORTCUT_THRESHOLD = 0.95
def __init__(self, lang_filter=None): def __init__(self, lang_filter: LanguageFilter = LanguageFilter.NONE) -> None:
self._state = None self._state = ProbingState.DETECTING
self.active = True
self.lang_filter = lang_filter self.lang_filter = lang_filter
self.logger = logging.getLogger(__name__) self.logger = logging.getLogger(__name__)
def reset(self): def reset(self) -> None:
self._state = ProbingState.DETECTING self._state = ProbingState.DETECTING
@property @property
def charset_name(self): def charset_name(self) -> Optional[str]:
return None return None
def feed(self, buf): @property
pass def language(self) -> Optional[str]:
raise NotImplementedError
def feed(self, byte_str: Union[bytes, bytearray]) -> ProbingState:
raise NotImplementedError
@property @property
def state(self): def state(self) -> ProbingState:
return self._state return self._state
def get_confidence(self): def get_confidence(self) -> float:
return 0.0 return 0.0
@staticmethod @staticmethod
def filter_high_byte_only(buf): def filter_high_byte_only(buf: Union[bytes, bytearray]) -> bytes:
buf = re.sub(b'([\x00-\x7F])+', b' ', buf) buf = re.sub(b"([\x00-\x7F])+", b" ", buf)
return buf return buf
@staticmethod @staticmethod
def filter_international_words(buf): def filter_international_words(buf: Union[bytes, bytearray]) -> bytearray:
""" """
We define three types of bytes: We define three types of bytes:
alphabet: english alphabets [a-zA-Z] alphabet: english alphabets [a-zA-Z]
international: international characters [\x80-\xFF] international: international characters [\x80-\xFF]
marker: everything else [^a-zA-Z\x80-\xFF] marker: everything else [^a-zA-Z\x80-\xFF]
The input buffer can be thought to contain a series of words delimited The input buffer can be thought to contain a series of words delimited
by markers. This function works to filter all words that contain at by markers. This function works to filter all words that contain at
least one international character. All contiguous sequences of markers least one international character. All contiguous sequences of markers
are replaced by a single space ascii character. are replaced by a single space ascii character.
This filter applies to all scripts which do not use English characters. This filter applies to all scripts which do not use English characters.
""" """
filtered = bytearray() filtered = bytearray()
@@ -83,8 +91,7 @@ class CharSetProber(object):
# This regex expression filters out only words that have at-least one # This regex expression filters out only words that have at-least one
# international character. The word may include one marker character at # international character. The word may include one marker character at
# the end. # the end.
words = re.findall(b'[a-zA-Z]*[\x80-\xFF]+[a-zA-Z]*[^a-zA-Z\x80-\xFF]?', words = INTERNATIONAL_WORDS_PATTERN.findall(buf)
buf)
for word in words: for word in words:
filtered.extend(word[:-1]) filtered.extend(word[:-1])
@@ -94,20 +101,17 @@ class CharSetProber(object):
# similarly across all languages and may thus have similar # similarly across all languages and may thus have similar
# frequencies). # frequencies).
last_char = word[-1:] last_char = word[-1:]
if not last_char.isalpha() and last_char < b'\x80': if not last_char.isalpha() and last_char < b"\x80":
last_char = b' ' last_char = b" "
filtered.extend(last_char) filtered.extend(last_char)
return filtered return filtered
@staticmethod @staticmethod
def filter_with_english_letters(buf): def remove_xml_tags(buf: Union[bytes, bytearray]) -> bytes:
""" """
Returns a copy of ``buf`` that retains only the sequences of English Returns a copy of ``buf`` that retains only the sequences of English
alphabet and high byte characters that are not between <> characters. alphabet and high byte characters that are not between <> characters.
Also retains English alphabet and high byte characters immediately
before occurrences of >.
This filter can be applied to all scripts which contain both English This filter can be applied to all scripts which contain both English
characters and extended ASCII characters, but is currently only used by characters and extended ASCII characters, but is currently only used by
``Latin1Prober``. ``Latin1Prober``.
@@ -115,26 +119,24 @@ class CharSetProber(object):
filtered = bytearray() filtered = bytearray()
in_tag = False in_tag = False
prev = 0 prev = 0
buf = memoryview(buf).cast("c")
for curr in range(len(buf)): for curr, buf_char in enumerate(buf):
# Slice here to get bytes instead of an int with Python 3 # Check if we're coming out of or entering an XML tag
buf_char = buf[curr:curr + 1]
# Check if we're coming out of or entering an HTML tag # https://github.com/python/typeshed/issues/8182
if buf_char == b'>': if buf_char == b">": # type: ignore[comparison-overlap]
prev = curr + 1
in_tag = False in_tag = False
elif buf_char == b'<': # https://github.com/python/typeshed/issues/8182
in_tag = True elif buf_char == b"<": # type: ignore[comparison-overlap]
# If current character is not extended-ASCII and not alphabetic...
if buf_char < b'\x80' and not buf_char.isalpha():
# ...and we're not in a tag
if curr > prev and not in_tag: if curr > prev and not in_tag:
# Keep everything after last non-extended-ASCII, # Keep everything after last non-extended-ASCII,
# non-alphabetic character # non-alphabetic character
filtered.extend(buf[prev:curr]) filtered.extend(buf[prev:curr])
# Output a space to delimit stretch we kept # Output a space to delimit stretch we kept
filtered.extend(b' ') filtered.extend(b" ")
prev = curr + 1 in_tag = True
# If we're not in a tag... # If we're not in a tag...
if not in_tag: if not in_tag:
-1
View File
@@ -1 +0,0 @@
+57 -29
View File
@@ -12,17 +12,21 @@ If no paths are provided, it takes its input from stdin.
""" """
from __future__ import absolute_import, print_function, unicode_literals
import argparse import argparse
import sys import sys
from typing import Iterable, List, Optional
from chardet import __version__ from .. import __version__
from chardet.compat import PY2 from ..universaldetector import UniversalDetector
from chardet.universaldetector import UniversalDetector
def description_of(lines, name='stdin'): def description_of(
lines: Iterable[bytes],
name: str = "stdin",
minimal: bool = False,
should_rename_legacy: bool = False,
) -> Optional[str]:
""" """
Return a string describing the probable encoding of a file or Return a string describing the probable encoding of a file or
list of strings. list of strings.
@@ -31,8 +35,11 @@ def description_of(lines, name='stdin'):
:type lines: Iterable of bytes :type lines: Iterable of bytes
:param name: Name of file or collection of lines :param name: Name of file or collection of lines
:type name: str :type name: str
:param should_rename_legacy: Should we rename legacy encodings to
their more modern equivalents?
:type should_rename_legacy: ``bool``
""" """
u = UniversalDetector() u = UniversalDetector(should_rename_legacy=should_rename_legacy)
for line in lines: for line in lines:
line = bytearray(line) line = bytearray(line)
u.feed(line) u.feed(line)
@@ -41,16 +48,14 @@ def description_of(lines, name='stdin'):
break break
u.close() u.close()
result = u.result result = u.result
if PY2: if minimal:
name = name.decode(sys.getfilesystemencoding(), 'ignore') return result["encoding"]
if result['encoding']: if result["encoding"]:
return '{}: {} with confidence {}'.format(name, result['encoding'], return f'{name}: {result["encoding"]} with confidence {result["confidence"]}'
result['confidence']) return f"{name}: no result"
else:
return '{}: no result'.format(name)
def main(argv=None): def main(argv: Optional[List[str]] = None) -> None:
""" """
Handles command line arguments and gets things started. Handles command line arguments and gets things started.
@@ -60,25 +65,48 @@ def main(argv=None):
""" """
# Get command line arguments # Get command line arguments
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description="Takes one or more file paths and reports their detected \ description=(
encodings") "Takes one or more file paths and reports their detected encodings"
parser.add_argument('input', )
help='File whose encoding we would like to determine. \ )
(default: stdin)', parser.add_argument(
type=argparse.FileType('rb'), nargs='*', "input",
default=[sys.stdin if PY2 else sys.stdin.buffer]) help="File whose encoding we would like to determine. (default: stdin)",
parser.add_argument('--version', action='version', type=argparse.FileType("rb"),
version='%(prog)s {}'.format(__version__)) nargs="*",
default=[sys.stdin.buffer],
)
parser.add_argument(
"--minimal",
help="Print only the encoding to standard output",
action="store_true",
)
parser.add_argument(
"-l",
"--legacy",
help="Rename legacy encodings to more modern ones.",
action="store_true",
)
parser.add_argument(
"--version", action="version", version=f"%(prog)s {__version__}"
)
args = parser.parse_args(argv) args = parser.parse_args(argv)
for f in args.input: for f in args.input:
if f.isatty(): if f.isatty():
print("You are running chardetect interactively. Press " + print(
"CTRL-D twice at the start of a blank line to signal the " + "You are running chardetect interactively. Press "
"end of your input. If you want help, run chardetect " + "CTRL-D twice at the start of a blank line to signal the "
"--help\n", file=sys.stderr) "end of your input. If you want help, run chardetect "
print(description_of(f, f.name)) "--help\n",
file=sys.stderr,
)
print(
description_of(
f, f.name, minimal=args.minimal, should_rename_legacy=args.legacy
)
)
if __name__ == '__main__': if __name__ == "__main__":
main() main()
+17 -15
View File
@@ -27,10 +27,11 @@
import logging import logging
from .codingstatemachinedict import CodingStateMachineDict
from .enums import MachineState from .enums import MachineState
class CodingStateMachine(object): class CodingStateMachine:
""" """
A state machine to verify a byte sequence for a particular encoding. For A state machine to verify a byte sequence for a particular encoding. For
each byte the detector receives, it will feed that byte to every active each byte the detector receives, it will feed that byte to every active
@@ -52,37 +53,38 @@ class CodingStateMachine(object):
negative answer for this encoding. Detector will exclude this negative answer for this encoding. Detector will exclude this
encoding from consideration from here on. encoding from consideration from here on.
""" """
def __init__(self, sm):
def __init__(self, sm: CodingStateMachineDict) -> None:
self._model = sm self._model = sm
self._curr_byte_pos = 0 self._curr_byte_pos = 0
self._curr_char_len = 0 self._curr_char_len = 0
self._curr_state = None self._curr_state = MachineState.START
self.active = True
self.logger = logging.getLogger(__name__) self.logger = logging.getLogger(__name__)
self.reset() self.reset()
def reset(self): def reset(self) -> None:
self._curr_state = MachineState.START self._curr_state = MachineState.START
def next_state(self, c): def next_state(self, c: int) -> int:
# for each byte we get its class # for each byte we get its class
# if it is first byte, we also get byte length # if it is first byte, we also get byte length
byte_class = self._model['class_table'][c] byte_class = self._model["class_table"][c]
if self._curr_state == MachineState.START: if self._curr_state == MachineState.START:
self._curr_byte_pos = 0 self._curr_byte_pos = 0
self._curr_char_len = self._model['char_len_table'][byte_class] self._curr_char_len = self._model["char_len_table"][byte_class]
# from byte's class and state_table, we get its next state # from byte's class and state_table, we get its next state
curr_state = (self._curr_state * self._model['class_factor'] curr_state = self._curr_state * self._model["class_factor"] + byte_class
+ byte_class) self._curr_state = self._model["state_table"][curr_state]
self._curr_state = self._model['state_table'][curr_state]
self._curr_byte_pos += 1 self._curr_byte_pos += 1
return self._curr_state return self._curr_state
def get_current_charlen(self): def get_current_charlen(self) -> int:
return self._curr_char_len return self._curr_char_len
def get_coding_state_machine(self): def get_coding_state_machine(self) -> str:
return self._model['name'] return self._model["name"]
@property @property
def language(self): def language(self) -> str:
return self._model['language'] return self._model["language"]
@@ -0,0 +1,19 @@
from typing import TYPE_CHECKING, Tuple
if TYPE_CHECKING:
# TypedDict was introduced in Python 3.8.
#
# TODO: Remove the else block and TYPE_CHECKING check when dropping support
# for Python 3.7.
from typing import TypedDict
class CodingStateMachineDict(TypedDict, total=False):
class_table: Tuple[int, ...]
class_factor: int
state_table: Tuple[int, ...]
char_len_table: Tuple[int, ...]
name: str
language: str # Optional key
else:
CodingStateMachineDict = dict
+4 -4
View File
@@ -32,8 +32,8 @@ from .mbcssm import CP949_SM_MODEL
class CP949Prober(MultiByteCharSetProber): class CP949Prober(MultiByteCharSetProber):
def __init__(self): def __init__(self) -> None:
super(CP949Prober, self).__init__() super().__init__()
self.coding_sm = CodingStateMachine(CP949_SM_MODEL) self.coding_sm = CodingStateMachine(CP949_SM_MODEL)
# NOTE: CP949 is a superset of EUC-KR, so the distribution should be # NOTE: CP949 is a superset of EUC-KR, so the distribution should be
# not different. # not different.
@@ -41,9 +41,9 @@ class CP949Prober(MultiByteCharSetProber):
self.reset() self.reset()
@property @property
def charset_name(self): def charset_name(self) -> str:
return "CP949" return "CP949"
@property @property
def language(self): def language(self) -> str:
return "Korean" return "Korean"
+16 -7
View File
@@ -4,21 +4,26 @@ All of the Enums that are used throughout the chardet package.
:author: Dan Blanchard (dan.blanchard@gmail.com) :author: Dan Blanchard (dan.blanchard@gmail.com)
""" """
from enum import Enum, Flag
class InputState(object):
class InputState:
""" """
This enum represents the different states a universal detector can be in. This enum represents the different states a universal detector can be in.
""" """
PURE_ASCII = 0 PURE_ASCII = 0
ESC_ASCII = 1 ESC_ASCII = 1
HIGH_BYTE = 2 HIGH_BYTE = 2
class LanguageFilter(object): class LanguageFilter(Flag):
""" """
This enum represents the different language filters we can apply to a This enum represents the different language filters we can apply to a
``UniversalDetector``. ``UniversalDetector``.
""" """
NONE = 0x00
CHINESE_SIMPLIFIED = 0x01 CHINESE_SIMPLIFIED = 0x01
CHINESE_TRADITIONAL = 0x02 CHINESE_TRADITIONAL = 0x02
JAPANESE = 0x04 JAPANESE = 0x04
@@ -29,46 +34,50 @@ class LanguageFilter(object):
CJK = CHINESE | JAPANESE | KOREAN CJK = CHINESE | JAPANESE | KOREAN
class ProbingState(object): class ProbingState(Enum):
""" """
This enum represents the different states a prober can be in. This enum represents the different states a prober can be in.
""" """
DETECTING = 0 DETECTING = 0
FOUND_IT = 1 FOUND_IT = 1
NOT_ME = 2 NOT_ME = 2
class MachineState(object): class MachineState:
""" """
This enum represents the different states a state machine can be in. This enum represents the different states a state machine can be in.
""" """
START = 0 START = 0
ERROR = 1 ERROR = 1
ITS_ME = 2 ITS_ME = 2
class SequenceLikelihood(object): class SequenceLikelihood:
""" """
This enum represents the likelihood of a character following the previous one. This enum represents the likelihood of a character following the previous one.
""" """
NEGATIVE = 0 NEGATIVE = 0
UNLIKELY = 1 UNLIKELY = 1
LIKELY = 2 LIKELY = 2
POSITIVE = 3 POSITIVE = 3
@classmethod @classmethod
def get_num_categories(cls): def get_num_categories(cls) -> int:
""":returns: The number of likelihood categories in the enum.""" """:returns: The number of likelihood categories in the enum."""
return 4 return 4
class CharacterCategory(object): class CharacterCategory:
""" """
This enum represents the different categories language models for This enum represents the different categories language models for
``SingleByteCharsetProber`` put characters into. ``SingleByteCharsetProber`` put characters into.
Anything less than CONTROL is considered a letter. Anything less than CONTROL is considered a letter.
""" """
UNDEFINED = 255 UNDEFINED = 255
LINE_BREAK = 254 LINE_BREAK = 254
SYMBOL = 253 SYMBOL = 253
+23 -22
View File
@@ -25,11 +25,17 @@
# 02110-1301 USA # 02110-1301 USA
######################### END LICENSE BLOCK ######################### ######################### END LICENSE BLOCK #########################
from typing import Optional, Union
from .charsetprober import CharSetProber from .charsetprober import CharSetProber
from .codingstatemachine import CodingStateMachine from .codingstatemachine import CodingStateMachine
from .enums import LanguageFilter, ProbingState, MachineState from .enums import LanguageFilter, MachineState, ProbingState
from .escsm import (HZ_SM_MODEL, ISO2022CN_SM_MODEL, ISO2022JP_SM_MODEL, from .escsm import (
ISO2022KR_SM_MODEL) HZ_SM_MODEL,
ISO2022CN_SM_MODEL,
ISO2022JP_SM_MODEL,
ISO2022KR_SM_MODEL,
)
class EscCharSetProber(CharSetProber): class EscCharSetProber(CharSetProber):
@@ -39,8 +45,8 @@ class EscCharSetProber(CharSetProber):
identify these encodings. identify these encodings.
""" """
def __init__(self, lang_filter=None): def __init__(self, lang_filter: LanguageFilter = LanguageFilter.NONE) -> None:
super(EscCharSetProber, self).__init__(lang_filter=lang_filter) super().__init__(lang_filter=lang_filter)
self.coding_sm = [] self.coding_sm = []
if self.lang_filter & LanguageFilter.CHINESE_SIMPLIFIED: if self.lang_filter & LanguageFilter.CHINESE_SIMPLIFIED:
self.coding_sm.append(CodingStateMachine(HZ_SM_MODEL)) self.coding_sm.append(CodingStateMachine(HZ_SM_MODEL))
@@ -49,17 +55,15 @@ class EscCharSetProber(CharSetProber):
self.coding_sm.append(CodingStateMachine(ISO2022JP_SM_MODEL)) self.coding_sm.append(CodingStateMachine(ISO2022JP_SM_MODEL))
if self.lang_filter & LanguageFilter.KOREAN: if self.lang_filter & LanguageFilter.KOREAN:
self.coding_sm.append(CodingStateMachine(ISO2022KR_SM_MODEL)) self.coding_sm.append(CodingStateMachine(ISO2022KR_SM_MODEL))
self.active_sm_count = None self.active_sm_count = 0
self._detected_charset = None self._detected_charset: Optional[str] = None
self._detected_language = None self._detected_language: Optional[str] = None
self._state = None self._state = ProbingState.DETECTING
self.reset() self.reset()
def reset(self): def reset(self) -> None:
super(EscCharSetProber, self).reset() super().reset()
for coding_sm in self.coding_sm: for coding_sm in self.coding_sm:
if not coding_sm:
continue
coding_sm.active = True coding_sm.active = True
coding_sm.reset() coding_sm.reset()
self.active_sm_count = len(self.coding_sm) self.active_sm_count = len(self.coding_sm)
@@ -67,23 +71,20 @@ class EscCharSetProber(CharSetProber):
self._detected_language = None self._detected_language = None
@property @property
def charset_name(self): def charset_name(self) -> Optional[str]:
return self._detected_charset return self._detected_charset
@property @property
def language(self): def language(self) -> Optional[str]:
return self._detected_language return self._detected_language
def get_confidence(self): def get_confidence(self) -> float:
if self._detected_charset: return 0.99 if self._detected_charset else 0.00
return 0.99
else:
return 0.00
def feed(self, byte_str): def feed(self, byte_str: Union[bytes, bytearray]) -> ProbingState:
for c in byte_str: for c in byte_str:
for coding_sm in self.coding_sm: for coding_sm in self.coding_sm:
if not coding_sm or not coding_sm.active: if not coding_sm.active:
continue continue
coding_state = coding_sm.next_state(c) coding_state = coding_sm.next_state(c)
if coding_state == MachineState.ERROR: if coding_state == MachineState.ERROR:
+200 -185
View File
@@ -12,7 +12,7 @@
# This library is free software; you can redistribute it and/or # This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public # modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either # License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version. # version 2.1 of the License, or (at your option) any later version.
# #
# This library is distributed in the hope that it will be useful, # This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of # but WITHOUT ANY WARRANTY; without even the implied warranty of
@@ -20,227 +20,242 @@
# Lesser General Public License for more details. # Lesser General Public License for more details.
# #
# You should have received a copy of the GNU Lesser General Public # You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software # License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
# 02110-1301 USA # 02110-1301 USA
######################### END LICENSE BLOCK ######################### ######################### END LICENSE BLOCK #########################
from .codingstatemachinedict import CodingStateMachineDict
from .enums import MachineState from .enums import MachineState
# fmt: off
HZ_CLS = ( HZ_CLS = (
1,0,0,0,0,0,0,0, # 00 - 07 1, 0, 0, 0, 0, 0, 0, 0, # 00 - 07
0,0,0,0,0,0,0,0, # 08 - 0f 0, 0, 0, 0, 0, 0, 0, 0, # 08 - 0f
0,0,0,0,0,0,0,0, # 10 - 17 0, 0, 0, 0, 0, 0, 0, 0, # 10 - 17
0,0,0,1,0,0,0,0, # 18 - 1f 0, 0, 0, 1, 0, 0, 0, 0, # 18 - 1f
0,0,0,0,0,0,0,0, # 20 - 27 0, 0, 0, 0, 0, 0, 0, 0, # 20 - 27
0,0,0,0,0,0,0,0, # 28 - 2f 0, 0, 0, 0, 0, 0, 0, 0, # 28 - 2f
0,0,0,0,0,0,0,0, # 30 - 37 0, 0, 0, 0, 0, 0, 0, 0, # 30 - 37
0,0,0,0,0,0,0,0, # 38 - 3f 0, 0, 0, 0, 0, 0, 0, 0, # 38 - 3f
0,0,0,0,0,0,0,0, # 40 - 47 0, 0, 0, 0, 0, 0, 0, 0, # 40 - 47
0,0,0,0,0,0,0,0, # 48 - 4f 0, 0, 0, 0, 0, 0, 0, 0, # 48 - 4f
0,0,0,0,0,0,0,0, # 50 - 57 0, 0, 0, 0, 0, 0, 0, 0, # 50 - 57
0,0,0,0,0,0,0,0, # 58 - 5f 0, 0, 0, 0, 0, 0, 0, 0, # 58 - 5f
0,0,0,0,0,0,0,0, # 60 - 67 0, 0, 0, 0, 0, 0, 0, 0, # 60 - 67
0,0,0,0,0,0,0,0, # 68 - 6f 0, 0, 0, 0, 0, 0, 0, 0, # 68 - 6f
0,0,0,0,0,0,0,0, # 70 - 77 0, 0, 0, 0, 0, 0, 0, 0, # 70 - 77
0,0,0,4,0,5,2,0, # 78 - 7f 0, 0, 0, 4, 0, 5, 2, 0, # 78 - 7f
1,1,1,1,1,1,1,1, # 80 - 87 1, 1, 1, 1, 1, 1, 1, 1, # 80 - 87
1,1,1,1,1,1,1,1, # 88 - 8f 1, 1, 1, 1, 1, 1, 1, 1, # 88 - 8f
1,1,1,1,1,1,1,1, # 90 - 97 1, 1, 1, 1, 1, 1, 1, 1, # 90 - 97
1,1,1,1,1,1,1,1, # 98 - 9f 1, 1, 1, 1, 1, 1, 1, 1, # 98 - 9f
1,1,1,1,1,1,1,1, # a0 - a7 1, 1, 1, 1, 1, 1, 1, 1, # a0 - a7
1,1,1,1,1,1,1,1, # a8 - af 1, 1, 1, 1, 1, 1, 1, 1, # a8 - af
1,1,1,1,1,1,1,1, # b0 - b7 1, 1, 1, 1, 1, 1, 1, 1, # b0 - b7
1,1,1,1,1,1,1,1, # b8 - bf 1, 1, 1, 1, 1, 1, 1, 1, # b8 - bf
1,1,1,1,1,1,1,1, # c0 - c7 1, 1, 1, 1, 1, 1, 1, 1, # c0 - c7
1,1,1,1,1,1,1,1, # c8 - cf 1, 1, 1, 1, 1, 1, 1, 1, # c8 - cf
1,1,1,1,1,1,1,1, # d0 - d7 1, 1, 1, 1, 1, 1, 1, 1, # d0 - d7
1,1,1,1,1,1,1,1, # d8 - df 1, 1, 1, 1, 1, 1, 1, 1, # d8 - df
1,1,1,1,1,1,1,1, # e0 - e7 1, 1, 1, 1, 1, 1, 1, 1, # e0 - e7
1,1,1,1,1,1,1,1, # e8 - ef 1, 1, 1, 1, 1, 1, 1, 1, # e8 - ef
1,1,1,1,1,1,1,1, # f0 - f7 1, 1, 1, 1, 1, 1, 1, 1, # f0 - f7
1,1,1,1,1,1,1,1, # f8 - ff 1, 1, 1, 1, 1, 1, 1, 1, # f8 - ff
) )
HZ_ST = ( HZ_ST = (
MachineState.START,MachineState.ERROR, 3,MachineState.START,MachineState.START,MachineState.START,MachineState.ERROR,MachineState.ERROR,# 00-07 MachineState.START, MachineState.ERROR, 3, MachineState.START, MachineState.START, MachineState.START, MachineState.ERROR, MachineState.ERROR, # 00-07
MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,# 08-0f MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ITS_ME, MachineState.ITS_ME, MachineState.ITS_ME, MachineState.ITS_ME, # 08-0f
MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ERROR,MachineState.ERROR,MachineState.START,MachineState.START, 4,MachineState.ERROR,# 10-17 MachineState.ITS_ME, MachineState.ITS_ME, MachineState.ERROR, MachineState.ERROR, MachineState.START, MachineState.START, 4, MachineState.ERROR, # 10-17
5,MachineState.ERROR, 6,MachineState.ERROR, 5, 5, 4,MachineState.ERROR,# 18-1f 5, MachineState.ERROR, 6, MachineState.ERROR, 5, 5, 4, MachineState.ERROR, # 18-1f
4,MachineState.ERROR, 4, 4, 4,MachineState.ERROR, 4,MachineState.ERROR,# 20-27 4, MachineState.ERROR, 4, 4, 4, MachineState.ERROR, 4, MachineState.ERROR, # 20-27
4,MachineState.ITS_ME,MachineState.START,MachineState.START,MachineState.START,MachineState.START,MachineState.START,MachineState.START,# 28-2f 4, MachineState.ITS_ME, MachineState.START, MachineState.START, MachineState.START, MachineState.START, MachineState.START, MachineState.START, # 28-2f
) )
# fmt: on
HZ_CHAR_LEN_TABLE = (0, 0, 0, 0, 0, 0) HZ_CHAR_LEN_TABLE = (0, 0, 0, 0, 0, 0)
HZ_SM_MODEL = {'class_table': HZ_CLS, HZ_SM_MODEL: CodingStateMachineDict = {
'class_factor': 6, "class_table": HZ_CLS,
'state_table': HZ_ST, "class_factor": 6,
'char_len_table': HZ_CHAR_LEN_TABLE, "state_table": HZ_ST,
'name': "HZ-GB-2312", "char_len_table": HZ_CHAR_LEN_TABLE,
'language': 'Chinese'} "name": "HZ-GB-2312",
"language": "Chinese",
}
# fmt: off
ISO2022CN_CLS = ( ISO2022CN_CLS = (
2,0,0,0,0,0,0,0, # 00 - 07 2, 0, 0, 0, 0, 0, 0, 0, # 00 - 07
0,0,0,0,0,0,0,0, # 08 - 0f 0, 0, 0, 0, 0, 0, 0, 0, # 08 - 0f
0,0,0,0,0,0,0,0, # 10 - 17 0, 0, 0, 0, 0, 0, 0, 0, # 10 - 17
0,0,0,1,0,0,0,0, # 18 - 1f 0, 0, 0, 1, 0, 0, 0, 0, # 18 - 1f
0,0,0,0,0,0,0,0, # 20 - 27 0, 0, 0, 0, 0, 0, 0, 0, # 20 - 27
0,3,0,0,0,0,0,0, # 28 - 2f 0, 3, 0, 0, 0, 0, 0, 0, # 28 - 2f
0,0,0,0,0,0,0,0, # 30 - 37 0, 0, 0, 0, 0, 0, 0, 0, # 30 - 37
0,0,0,0,0,0,0,0, # 38 - 3f 0, 0, 0, 0, 0, 0, 0, 0, # 38 - 3f
0,0,0,4,0,0,0,0, # 40 - 47 0, 0, 0, 4, 0, 0, 0, 0, # 40 - 47
0,0,0,0,0,0,0,0, # 48 - 4f 0, 0, 0, 0, 0, 0, 0, 0, # 48 - 4f
0,0,0,0,0,0,0,0, # 50 - 57 0, 0, 0, 0, 0, 0, 0, 0, # 50 - 57
0,0,0,0,0,0,0,0, # 58 - 5f 0, 0, 0, 0, 0, 0, 0, 0, # 58 - 5f
0,0,0,0,0,0,0,0, # 60 - 67 0, 0, 0, 0, 0, 0, 0, 0, # 60 - 67
0,0,0,0,0,0,0,0, # 68 - 6f 0, 0, 0, 0, 0, 0, 0, 0, # 68 - 6f
0,0,0,0,0,0,0,0, # 70 - 77 0, 0, 0, 0, 0, 0, 0, 0, # 70 - 77
0,0,0,0,0,0,0,0, # 78 - 7f 0, 0, 0, 0, 0, 0, 0, 0, # 78 - 7f
2,2,2,2,2,2,2,2, # 80 - 87 2, 2, 2, 2, 2, 2, 2, 2, # 80 - 87
2,2,2,2,2,2,2,2, # 88 - 8f 2, 2, 2, 2, 2, 2, 2, 2, # 88 - 8f
2,2,2,2,2,2,2,2, # 90 - 97 2, 2, 2, 2, 2, 2, 2, 2, # 90 - 97
2,2,2,2,2,2,2,2, # 98 - 9f 2, 2, 2, 2, 2, 2, 2, 2, # 98 - 9f
2,2,2,2,2,2,2,2, # a0 - a7 2, 2, 2, 2, 2, 2, 2, 2, # a0 - a7
2,2,2,2,2,2,2,2, # a8 - af 2, 2, 2, 2, 2, 2, 2, 2, # a8 - af
2,2,2,2,2,2,2,2, # b0 - b7 2, 2, 2, 2, 2, 2, 2, 2, # b0 - b7
2,2,2,2,2,2,2,2, # b8 - bf 2, 2, 2, 2, 2, 2, 2, 2, # b8 - bf
2,2,2,2,2,2,2,2, # c0 - c7 2, 2, 2, 2, 2, 2, 2, 2, # c0 - c7
2,2,2,2,2,2,2,2, # c8 - cf 2, 2, 2, 2, 2, 2, 2, 2, # c8 - cf
2,2,2,2,2,2,2,2, # d0 - d7 2, 2, 2, 2, 2, 2, 2, 2, # d0 - d7
2,2,2,2,2,2,2,2, # d8 - df 2, 2, 2, 2, 2, 2, 2, 2, # d8 - df
2,2,2,2,2,2,2,2, # e0 - e7 2, 2, 2, 2, 2, 2, 2, 2, # e0 - e7
2,2,2,2,2,2,2,2, # e8 - ef 2, 2, 2, 2, 2, 2, 2, 2, # e8 - ef
2,2,2,2,2,2,2,2, # f0 - f7 2, 2, 2, 2, 2, 2, 2, 2, # f0 - f7
2,2,2,2,2,2,2,2, # f8 - ff 2, 2, 2, 2, 2, 2, 2, 2, # f8 - ff
) )
ISO2022CN_ST = ( ISO2022CN_ST = (
MachineState.START, 3,MachineState.ERROR,MachineState.START,MachineState.START,MachineState.START,MachineState.START,MachineState.START,# 00-07 MachineState.START, 3, MachineState.ERROR, MachineState.START, MachineState.START, MachineState.START, MachineState.START, MachineState.START, # 00-07
MachineState.START,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,# 08-0f MachineState.START, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, # 08-0f
MachineState.ERROR,MachineState.ERROR,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,# 10-17 MachineState.ERROR, MachineState.ERROR, MachineState.ITS_ME, MachineState.ITS_ME, MachineState.ITS_ME, MachineState.ITS_ME, MachineState.ITS_ME, MachineState.ITS_ME, # 10-17
MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR, 4,MachineState.ERROR,# 18-1f MachineState.ITS_ME, MachineState.ITS_ME, MachineState.ITS_ME, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, 4, MachineState.ERROR, # 18-1f
MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ITS_ME,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,# 20-27 MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ITS_ME, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, # 20-27
5, 6,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,# 28-2f 5, 6, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, # 28-2f
MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ITS_ME,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,# 30-37 MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ITS_ME, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, # 30-37
MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ITS_ME,MachineState.ERROR,MachineState.START,# 38-3f MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ITS_ME, MachineState.ERROR, MachineState.START, # 38-3f
) )
# fmt: on
ISO2022CN_CHAR_LEN_TABLE = (0, 0, 0, 0, 0, 0, 0, 0, 0) ISO2022CN_CHAR_LEN_TABLE = (0, 0, 0, 0, 0, 0, 0, 0, 0)
ISO2022CN_SM_MODEL = {'class_table': ISO2022CN_CLS, ISO2022CN_SM_MODEL: CodingStateMachineDict = {
'class_factor': 9, "class_table": ISO2022CN_CLS,
'state_table': ISO2022CN_ST, "class_factor": 9,
'char_len_table': ISO2022CN_CHAR_LEN_TABLE, "state_table": ISO2022CN_ST,
'name': "ISO-2022-CN", "char_len_table": ISO2022CN_CHAR_LEN_TABLE,
'language': 'Chinese'} "name": "ISO-2022-CN",
"language": "Chinese",
}
# fmt: off
ISO2022JP_CLS = ( ISO2022JP_CLS = (
2,0,0,0,0,0,0,0, # 00 - 07 2, 0, 0, 0, 0, 0, 0, 0, # 00 - 07
0,0,0,0,0,0,2,2, # 08 - 0f 0, 0, 0, 0, 0, 0, 2, 2, # 08 - 0f
0,0,0,0,0,0,0,0, # 10 - 17 0, 0, 0, 0, 0, 0, 0, 0, # 10 - 17
0,0,0,1,0,0,0,0, # 18 - 1f 0, 0, 0, 1, 0, 0, 0, 0, # 18 - 1f
0,0,0,0,7,0,0,0, # 20 - 27 0, 0, 0, 0, 7, 0, 0, 0, # 20 - 27
3,0,0,0,0,0,0,0, # 28 - 2f 3, 0, 0, 0, 0, 0, 0, 0, # 28 - 2f
0,0,0,0,0,0,0,0, # 30 - 37 0, 0, 0, 0, 0, 0, 0, 0, # 30 - 37
0,0,0,0,0,0,0,0, # 38 - 3f 0, 0, 0, 0, 0, 0, 0, 0, # 38 - 3f
6,0,4,0,8,0,0,0, # 40 - 47 6, 0, 4, 0, 8, 0, 0, 0, # 40 - 47
0,9,5,0,0,0,0,0, # 48 - 4f 0, 9, 5, 0, 0, 0, 0, 0, # 48 - 4f
0,0,0,0,0,0,0,0, # 50 - 57 0, 0, 0, 0, 0, 0, 0, 0, # 50 - 57
0,0,0,0,0,0,0,0, # 58 - 5f 0, 0, 0, 0, 0, 0, 0, 0, # 58 - 5f
0,0,0,0,0,0,0,0, # 60 - 67 0, 0, 0, 0, 0, 0, 0, 0, # 60 - 67
0,0,0,0,0,0,0,0, # 68 - 6f 0, 0, 0, 0, 0, 0, 0, 0, # 68 - 6f
0,0,0,0,0,0,0,0, # 70 - 77 0, 0, 0, 0, 0, 0, 0, 0, # 70 - 77
0,0,0,0,0,0,0,0, # 78 - 7f 0, 0, 0, 0, 0, 0, 0, 0, # 78 - 7f
2,2,2,2,2,2,2,2, # 80 - 87 2, 2, 2, 2, 2, 2, 2, 2, # 80 - 87
2,2,2,2,2,2,2,2, # 88 - 8f 2, 2, 2, 2, 2, 2, 2, 2, # 88 - 8f
2,2,2,2,2,2,2,2, # 90 - 97 2, 2, 2, 2, 2, 2, 2, 2, # 90 - 97
2,2,2,2,2,2,2,2, # 98 - 9f 2, 2, 2, 2, 2, 2, 2, 2, # 98 - 9f
2,2,2,2,2,2,2,2, # a0 - a7 2, 2, 2, 2, 2, 2, 2, 2, # a0 - a7
2,2,2,2,2,2,2,2, # a8 - af 2, 2, 2, 2, 2, 2, 2, 2, # a8 - af
2,2,2,2,2,2,2,2, # b0 - b7 2, 2, 2, 2, 2, 2, 2, 2, # b0 - b7
2,2,2,2,2,2,2,2, # b8 - bf 2, 2, 2, 2, 2, 2, 2, 2, # b8 - bf
2,2,2,2,2,2,2,2, # c0 - c7 2, 2, 2, 2, 2, 2, 2, 2, # c0 - c7
2,2,2,2,2,2,2,2, # c8 - cf 2, 2, 2, 2, 2, 2, 2, 2, # c8 - cf
2,2,2,2,2,2,2,2, # d0 - d7 2, 2, 2, 2, 2, 2, 2, 2, # d0 - d7
2,2,2,2,2,2,2,2, # d8 - df 2, 2, 2, 2, 2, 2, 2, 2, # d8 - df
2,2,2,2,2,2,2,2, # e0 - e7 2, 2, 2, 2, 2, 2, 2, 2, # e0 - e7
2,2,2,2,2,2,2,2, # e8 - ef 2, 2, 2, 2, 2, 2, 2, 2, # e8 - ef
2,2,2,2,2,2,2,2, # f0 - f7 2, 2, 2, 2, 2, 2, 2, 2, # f0 - f7
2,2,2,2,2,2,2,2, # f8 - ff 2, 2, 2, 2, 2, 2, 2, 2, # f8 - ff
) )
ISO2022JP_ST = ( ISO2022JP_ST = (
MachineState.START, 3,MachineState.ERROR,MachineState.START,MachineState.START,MachineState.START,MachineState.START,MachineState.START,# 00-07 MachineState.START, 3, MachineState.ERROR, MachineState.START, MachineState.START, MachineState.START, MachineState.START, MachineState.START, # 00-07
MachineState.START,MachineState.START,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,# 08-0f MachineState.START, MachineState.START, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, # 08-0f
MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,# 10-17 MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ITS_ME, MachineState.ITS_ME, MachineState.ITS_ME, MachineState.ITS_ME, # 10-17
MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ERROR,MachineState.ERROR,# 18-1f MachineState.ITS_ME, MachineState.ITS_ME, MachineState.ITS_ME, MachineState.ITS_ME, MachineState.ITS_ME, MachineState.ITS_ME, MachineState.ERROR, MachineState.ERROR, # 18-1f
MachineState.ERROR, 5,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR, 4,MachineState.ERROR,MachineState.ERROR,# 20-27 MachineState.ERROR, 5, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, 4, MachineState.ERROR, MachineState.ERROR, # 20-27
MachineState.ERROR,MachineState.ERROR,MachineState.ERROR, 6,MachineState.ITS_ME,MachineState.ERROR,MachineState.ITS_ME,MachineState.ERROR,# 28-2f MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, 6, MachineState.ITS_ME, MachineState.ERROR, MachineState.ITS_ME, MachineState.ERROR, # 28-2f
MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ITS_ME,MachineState.ITS_ME,# 30-37 MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ITS_ME, MachineState.ITS_ME, # 30-37
MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ITS_ME,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,# 38-3f MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ITS_ME, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, # 38-3f
MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ITS_ME,MachineState.ERROR,MachineState.START,MachineState.START,# 40-47 MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ITS_ME, MachineState.ERROR, MachineState.START, MachineState.START, # 40-47
) )
# fmt: on
ISO2022JP_CHAR_LEN_TABLE = (0, 0, 0, 0, 0, 0, 0, 0, 0, 0) ISO2022JP_CHAR_LEN_TABLE = (0, 0, 0, 0, 0, 0, 0, 0, 0, 0)
ISO2022JP_SM_MODEL = {'class_table': ISO2022JP_CLS, ISO2022JP_SM_MODEL: CodingStateMachineDict = {
'class_factor': 10, "class_table": ISO2022JP_CLS,
'state_table': ISO2022JP_ST, "class_factor": 10,
'char_len_table': ISO2022JP_CHAR_LEN_TABLE, "state_table": ISO2022JP_ST,
'name': "ISO-2022-JP", "char_len_table": ISO2022JP_CHAR_LEN_TABLE,
'language': 'Japanese'} "name": "ISO-2022-JP",
"language": "Japanese",
}
# fmt: off
ISO2022KR_CLS = ( ISO2022KR_CLS = (
2,0,0,0,0,0,0,0, # 00 - 07 2, 0, 0, 0, 0, 0, 0, 0, # 00 - 07
0,0,0,0,0,0,0,0, # 08 - 0f 0, 0, 0, 0, 0, 0, 0, 0, # 08 - 0f
0,0,0,0,0,0,0,0, # 10 - 17 0, 0, 0, 0, 0, 0, 0, 0, # 10 - 17
0,0,0,1,0,0,0,0, # 18 - 1f 0, 0, 0, 1, 0, 0, 0, 0, # 18 - 1f
0,0,0,0,3,0,0,0, # 20 - 27 0, 0, 0, 0, 3, 0, 0, 0, # 20 - 27
0,4,0,0,0,0,0,0, # 28 - 2f 0, 4, 0, 0, 0, 0, 0, 0, # 28 - 2f
0,0,0,0,0,0,0,0, # 30 - 37 0, 0, 0, 0, 0, 0, 0, 0, # 30 - 37
0,0,0,0,0,0,0,0, # 38 - 3f 0, 0, 0, 0, 0, 0, 0, 0, # 38 - 3f
0,0,0,5,0,0,0,0, # 40 - 47 0, 0, 0, 5, 0, 0, 0, 0, # 40 - 47
0,0,0,0,0,0,0,0, # 48 - 4f 0, 0, 0, 0, 0, 0, 0, 0, # 48 - 4f
0,0,0,0,0,0,0,0, # 50 - 57 0, 0, 0, 0, 0, 0, 0, 0, # 50 - 57
0,0,0,0,0,0,0,0, # 58 - 5f 0, 0, 0, 0, 0, 0, 0, 0, # 58 - 5f
0,0,0,0,0,0,0,0, # 60 - 67 0, 0, 0, 0, 0, 0, 0, 0, # 60 - 67
0,0,0,0,0,0,0,0, # 68 - 6f 0, 0, 0, 0, 0, 0, 0, 0, # 68 - 6f
0,0,0,0,0,0,0,0, # 70 - 77 0, 0, 0, 0, 0, 0, 0, 0, # 70 - 77
0,0,0,0,0,0,0,0, # 78 - 7f 0, 0, 0, 0, 0, 0, 0, 0, # 78 - 7f
2,2,2,2,2,2,2,2, # 80 - 87 2, 2, 2, 2, 2, 2, 2, 2, # 80 - 87
2,2,2,2,2,2,2,2, # 88 - 8f 2, 2, 2, 2, 2, 2, 2, 2, # 88 - 8f
2,2,2,2,2,2,2,2, # 90 - 97 2, 2, 2, 2, 2, 2, 2, 2, # 90 - 97
2,2,2,2,2,2,2,2, # 98 - 9f 2, 2, 2, 2, 2, 2, 2, 2, # 98 - 9f
2,2,2,2,2,2,2,2, # a0 - a7 2, 2, 2, 2, 2, 2, 2, 2, # a0 - a7
2,2,2,2,2,2,2,2, # a8 - af 2, 2, 2, 2, 2, 2, 2, 2, # a8 - af
2,2,2,2,2,2,2,2, # b0 - b7 2, 2, 2, 2, 2, 2, 2, 2, # b0 - b7
2,2,2,2,2,2,2,2, # b8 - bf 2, 2, 2, 2, 2, 2, 2, 2, # b8 - bf
2,2,2,2,2,2,2,2, # c0 - c7 2, 2, 2, 2, 2, 2, 2, 2, # c0 - c7
2,2,2,2,2,2,2,2, # c8 - cf 2, 2, 2, 2, 2, 2, 2, 2, # c8 - cf
2,2,2,2,2,2,2,2, # d0 - d7 2, 2, 2, 2, 2, 2, 2, 2, # d0 - d7
2,2,2,2,2,2,2,2, # d8 - df 2, 2, 2, 2, 2, 2, 2, 2, # d8 - df
2,2,2,2,2,2,2,2, # e0 - e7 2, 2, 2, 2, 2, 2, 2, 2, # e0 - e7
2,2,2,2,2,2,2,2, # e8 - ef 2, 2, 2, 2, 2, 2, 2, 2, # e8 - ef
2,2,2,2,2,2,2,2, # f0 - f7 2, 2, 2, 2, 2, 2, 2, 2, # f0 - f7
2,2,2,2,2,2,2,2, # f8 - ff 2, 2, 2, 2, 2, 2, 2, 2, # f8 - ff
) )
ISO2022KR_ST = ( ISO2022KR_ST = (
MachineState.START, 3,MachineState.ERROR,MachineState.START,MachineState.START,MachineState.START,MachineState.ERROR,MachineState.ERROR,# 00-07 MachineState.START, 3, MachineState.ERROR, MachineState.START, MachineState.START, MachineState.START, MachineState.ERROR, MachineState.ERROR, # 00-07
MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,# 08-0f MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ITS_ME, MachineState.ITS_ME, MachineState.ITS_ME, MachineState.ITS_ME, # 08-0f
MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR, 4,MachineState.ERROR,MachineState.ERROR,# 10-17 MachineState.ITS_ME, MachineState.ITS_ME, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, 4, MachineState.ERROR, MachineState.ERROR, # 10-17
MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR, 5,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,# 18-1f MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, 5, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, # 18-1f
MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ITS_ME,MachineState.START,MachineState.START,MachineState.START,MachineState.START,# 20-27 MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ITS_ME, MachineState.START, MachineState.START, MachineState.START, MachineState.START, # 20-27
) )
# fmt: on
ISO2022KR_CHAR_LEN_TABLE = (0, 0, 0, 0, 0, 0) ISO2022KR_CHAR_LEN_TABLE = (0, 0, 0, 0, 0, 0)
ISO2022KR_SM_MODEL = {'class_table': ISO2022KR_CLS, ISO2022KR_SM_MODEL: CodingStateMachineDict = {
'class_factor': 6, "class_table": ISO2022KR_CLS,
'state_table': ISO2022KR_ST, "class_factor": 6,
'char_len_table': ISO2022KR_CHAR_LEN_TABLE, "state_table": ISO2022KR_ST,
'name': "ISO-2022-KR", "char_len_table": ISO2022KR_CHAR_LEN_TABLE,
'language': 'Korean'} "name": "ISO-2022-KR",
"language": "Korean",
}
+35 -25
View File
@@ -25,68 +25,78 @@
# 02110-1301 USA # 02110-1301 USA
######################### END LICENSE BLOCK ######################### ######################### END LICENSE BLOCK #########################
from .enums import ProbingState, MachineState from typing import Union
from .mbcharsetprober import MultiByteCharSetProber
from .codingstatemachine import CodingStateMachine
from .chardistribution import EUCJPDistributionAnalysis from .chardistribution import EUCJPDistributionAnalysis
from .codingstatemachine import CodingStateMachine
from .enums import MachineState, ProbingState
from .jpcntx import EUCJPContextAnalysis from .jpcntx import EUCJPContextAnalysis
from .mbcharsetprober import MultiByteCharSetProber
from .mbcssm import EUCJP_SM_MODEL from .mbcssm import EUCJP_SM_MODEL
class EUCJPProber(MultiByteCharSetProber): class EUCJPProber(MultiByteCharSetProber):
def __init__(self): def __init__(self) -> None:
super(EUCJPProber, self).__init__() super().__init__()
self.coding_sm = CodingStateMachine(EUCJP_SM_MODEL) self.coding_sm = CodingStateMachine(EUCJP_SM_MODEL)
self.distribution_analyzer = EUCJPDistributionAnalysis() self.distribution_analyzer = EUCJPDistributionAnalysis()
self.context_analyzer = EUCJPContextAnalysis() self.context_analyzer = EUCJPContextAnalysis()
self.reset() self.reset()
def reset(self): def reset(self) -> None:
super(EUCJPProber, self).reset() super().reset()
self.context_analyzer.reset() self.context_analyzer.reset()
@property @property
def charset_name(self): def charset_name(self) -> str:
return "EUC-JP" return "EUC-JP"
@property @property
def language(self): def language(self) -> str:
return "Japanese" return "Japanese"
def feed(self, byte_str): def feed(self, byte_str: Union[bytes, bytearray]) -> ProbingState:
for i in range(len(byte_str)): assert self.coding_sm is not None
# PY3K: byte_str is a byte array, so byte_str[i] is an int, not a byte assert self.distribution_analyzer is not None
coding_state = self.coding_sm.next_state(byte_str[i])
for i, byte in enumerate(byte_str):
# PY3K: byte_str is a byte array, so byte is an int, not a byte
coding_state = self.coding_sm.next_state(byte)
if coding_state == MachineState.ERROR: if coding_state == MachineState.ERROR:
self.logger.debug('%s %s prober hit error at byte %s', self.logger.debug(
self.charset_name, self.language, i) "%s %s prober hit error at byte %s",
self.charset_name,
self.language,
i,
)
self._state = ProbingState.NOT_ME self._state = ProbingState.NOT_ME
break break
elif coding_state == MachineState.ITS_ME: if coding_state == MachineState.ITS_ME:
self._state = ProbingState.FOUND_IT self._state = ProbingState.FOUND_IT
break break
elif coding_state == MachineState.START: if coding_state == MachineState.START:
char_len = self.coding_sm.get_current_charlen() char_len = self.coding_sm.get_current_charlen()
if i == 0: if i == 0:
self._last_char[1] = byte_str[0] self._last_char[1] = byte
self.context_analyzer.feed(self._last_char, char_len) self.context_analyzer.feed(self._last_char, char_len)
self.distribution_analyzer.feed(self._last_char, char_len) self.distribution_analyzer.feed(self._last_char, char_len)
else: else:
self.context_analyzer.feed(byte_str[i - 1:i + 1], self.context_analyzer.feed(byte_str[i - 1 : i + 1], char_len)
char_len) self.distribution_analyzer.feed(byte_str[i - 1 : i + 1], char_len)
self.distribution_analyzer.feed(byte_str[i - 1:i + 1],
char_len)
self._last_char[0] = byte_str[-1] self._last_char[0] = byte_str[-1]
if self.state == ProbingState.DETECTING: if self.state == ProbingState.DETECTING:
if (self.context_analyzer.got_enough_data() and if self.context_analyzer.got_enough_data() and (
(self.get_confidence() > self.SHORTCUT_THRESHOLD)): self.get_confidence() > self.SHORTCUT_THRESHOLD
):
self._state = ProbingState.FOUND_IT self._state = ProbingState.FOUND_IT
return self.state return self.state
def get_confidence(self): def get_confidence(self) -> float:
assert self.distribution_analyzer is not None
context_conf = self.context_analyzer.get_confidence() context_conf = self.context_analyzer.get_confidence()
distrib_conf = self.distribution_analyzer.get_confidence() distrib_conf = self.distribution_analyzer.get_confidence()
return max(context_conf, distrib_conf) return max(context_conf, distrib_conf)
+2 -1
View File
@@ -43,6 +43,7 @@ EUCKR_TYPICAL_DISTRIBUTION_RATIO = 6.0
EUCKR_TABLE_SIZE = 2352 EUCKR_TABLE_SIZE = 2352
# Char to FreqOrder table , # Char to FreqOrder table ,
# fmt: off
EUCKR_CHAR_TO_FREQ_ORDER = ( EUCKR_CHAR_TO_FREQ_ORDER = (
13, 130, 120,1396, 481,1719,1720, 328, 609, 212,1721, 707, 400, 299,1722, 87, 13, 130, 120,1396, 481,1719,1720, 328, 609, 212,1721, 707, 400, 299,1722, 87,
1397,1723, 104, 536,1117,1203,1724,1267, 685,1268, 508,1725,1726,1727,1728,1398, 1397,1723, 104, 536,1117,1203,1724,1267, 685,1268, 508,1725,1726,1727,1728,1398,
@@ -192,4 +193,4 @@ EUCKR_CHAR_TO_FREQ_ORDER = (
2629,2630,2631, 924, 648, 863, 603,2632,2633, 934,1540, 864, 865,2634, 642,1042, 2629,2630,2631, 924, 648, 863, 603,2632,2633, 934,1540, 864, 865,2634, 642,1042,
670,1190,2635,2636,2637,2638, 168,2639, 652, 873, 542,1054,1541,2640,2641,2642, # 512, 256 670,1190,2635,2636,2637,2638, 168,2639, 652, 873, 542,1054,1541,2640,2641,2642, # 512, 256
) )
# fmt: on
+6 -6
View File
@@ -25,23 +25,23 @@
# 02110-1301 USA # 02110-1301 USA
######################### END LICENSE BLOCK ######################### ######################### END LICENSE BLOCK #########################
from .mbcharsetprober import MultiByteCharSetProber
from .codingstatemachine import CodingStateMachine
from .chardistribution import EUCKRDistributionAnalysis from .chardistribution import EUCKRDistributionAnalysis
from .codingstatemachine import CodingStateMachine
from .mbcharsetprober import MultiByteCharSetProber
from .mbcssm import EUCKR_SM_MODEL from .mbcssm import EUCKR_SM_MODEL
class EUCKRProber(MultiByteCharSetProber): class EUCKRProber(MultiByteCharSetProber):
def __init__(self): def __init__(self) -> None:
super(EUCKRProber, self).__init__() super().__init__()
self.coding_sm = CodingStateMachine(EUCKR_SM_MODEL) self.coding_sm = CodingStateMachine(EUCKR_SM_MODEL)
self.distribution_analyzer = EUCKRDistributionAnalysis() self.distribution_analyzer = EUCKRDistributionAnalysis()
self.reset() self.reset()
@property @property
def charset_name(self): def charset_name(self) -> str:
return "EUC-KR" return "EUC-KR"
@property @property
def language(self): def language(self) -> str:
return "Korean" return "Korean"
+339 -338
View File
@@ -43,345 +43,346 @@
EUCTW_TYPICAL_DISTRIBUTION_RATIO = 0.75 EUCTW_TYPICAL_DISTRIBUTION_RATIO = 0.75
# Char to FreqOrder table , # Char to FreqOrder table
EUCTW_TABLE_SIZE = 5376 EUCTW_TABLE_SIZE = 5376
# fmt: off
EUCTW_CHAR_TO_FREQ_ORDER = ( EUCTW_CHAR_TO_FREQ_ORDER = (
1,1800,1506, 255,1431, 198, 9, 82, 6,7310, 177, 202,3615,1256,2808, 110, # 2742 1, 1800, 1506, 255, 1431, 198, 9, 82, 6, 7310, 177, 202, 3615, 1256, 2808, 110, # 2742
3735, 33,3241, 261, 76, 44,2113, 16,2931,2184,1176, 659,3868, 26,3404,2643, # 2758 3735, 33, 3241, 261, 76, 44, 2113, 16, 2931, 2184, 1176, 659, 3868, 26, 3404, 2643, # 2758
1198,3869,3313,4060, 410,2211, 302, 590, 361,1963, 8, 204, 58,4296,7311,1931, # 2774 1198, 3869, 3313, 4060, 410, 2211, 302, 590, 361, 1963, 8, 204, 58, 4296, 7311, 1931, # 2774
63,7312,7313, 317,1614, 75, 222, 159,4061,2412,1480,7314,3500,3068, 224,2809, # 2790 63, 7312, 7313, 317, 1614, 75, 222, 159, 4061, 2412, 1480, 7314, 3500, 3068, 224, 2809, # 2790
3616, 3, 10,3870,1471, 29,2774,1135,2852,1939, 873, 130,3242,1123, 312,7315, # 2806 3616, 3, 10, 3870, 1471, 29, 2774, 1135, 2852, 1939, 873, 130, 3242, 1123, 312, 7315, # 2806
4297,2051, 507, 252, 682,7316, 142,1914, 124, 206,2932, 34,3501,3173, 64, 604, # 2822 4297, 2051, 507, 252, 682, 7316, 142, 1914, 124, 206, 2932, 34, 3501, 3173, 64, 604, # 2822
7317,2494,1976,1977, 155,1990, 645, 641,1606,7318,3405, 337, 72, 406,7319, 80, # 2838 7317, 2494, 1976, 1977, 155, 1990, 645, 641, 1606, 7318, 3405, 337, 72, 406, 7319, 80, # 2838
630, 238,3174,1509, 263, 939,1092,2644, 756,1440,1094,3406, 449, 69,2969, 591, # 2854 630, 238, 3174, 1509, 263, 939, 1092, 2644, 756, 1440, 1094, 3406, 449, 69, 2969, 591, # 2854
179,2095, 471, 115,2034,1843, 60, 50,2970, 134, 806,1868, 734,2035,3407, 180, # 2870 179, 2095, 471, 115, 2034, 1843, 60, 50, 2970, 134, 806, 1868, 734, 2035, 3407, 180, # 2870
995,1607, 156, 537,2893, 688,7320, 319,1305, 779,2144, 514,2374, 298,4298, 359, # 2886 995, 1607, 156, 537, 2893, 688, 7320, 319, 1305, 779, 2144, 514, 2374, 298, 4298, 359, # 2886
2495, 90,2707,1338, 663, 11, 906,1099,2545, 20,2436, 182, 532,1716,7321, 732, # 2902 2495, 90, 2707, 1338, 663, 11, 906, 1099, 2545, 20, 2436, 182, 532, 1716, 7321, 732, # 2902
1376,4062,1311,1420,3175, 25,2312,1056, 113, 399, 382,1949, 242,3408,2467, 529, # 2918 1376, 4062, 1311, 1420, 3175, 25, 2312, 1056, 113, 399, 382, 1949, 242, 3408, 2467, 529, # 2918
3243, 475,1447,3617,7322, 117, 21, 656, 810,1297,2295,2329,3502,7323, 126,4063, # 2934 3243, 475, 1447, 3617, 7322, 117, 21, 656, 810, 1297, 2295, 2329, 3502, 7323, 126, 4063, # 2934
706, 456, 150, 613,4299, 71,1118,2036,4064, 145,3069, 85, 835, 486,2114,1246, # 2950 706, 456, 150, 613, 4299, 71, 1118, 2036, 4064, 145, 3069, 85, 835, 486, 2114, 1246, # 2950
1426, 428, 727,1285,1015, 800, 106, 623, 303,1281,7324,2127,2354, 347,3736, 221, # 2966 1426, 428, 727, 1285, 1015, 800, 106, 623, 303, 1281, 7324, 2127, 2354, 347, 3736, 221, # 2966
3503,3110,7325,1955,1153,4065, 83, 296,1199,3070, 192, 624, 93,7326, 822,1897, # 2982 3503, 3110, 7325, 1955, 1153, 4065, 83, 296, 1199, 3070, 192, 624, 93, 7326, 822, 1897, # 2982
2810,3111, 795,2064, 991,1554,1542,1592, 27, 43,2853, 859, 139,1456, 860,4300, # 2998 2810, 3111, 795, 2064, 991, 1554, 1542, 1592, 27, 43, 2853, 859, 139, 1456, 860, 4300, # 2998
437, 712,3871, 164,2392,3112, 695, 211,3017,2096, 195,3872,1608,3504,3505,3618, # 3014 437, 712, 3871, 164, 2392, 3112, 695, 211, 3017, 2096, 195, 3872, 1608, 3504, 3505, 3618, # 3014
3873, 234, 811,2971,2097,3874,2229,1441,3506,1615,2375, 668,2076,1638, 305, 228, # 3030 3873, 234, 811, 2971, 2097, 3874, 2229, 1441, 3506, 1615, 2375, 668, 2076, 1638, 305, 228, # 3030
1664,4301, 467, 415,7327, 262,2098,1593, 239, 108, 300, 200,1033, 512,1247,2077, # 3046 1664, 4301, 467, 415, 7327, 262, 2098, 1593, 239, 108, 300, 200, 1033, 512, 1247, 2077, # 3046
7328,7329,2173,3176,3619,2673, 593, 845,1062,3244, 88,1723,2037,3875,1950, 212, # 3062 7328, 7329, 2173, 3176, 3619, 2673, 593, 845, 1062, 3244, 88, 1723, 2037, 3875, 1950, 212, # 3062
266, 152, 149, 468,1898,4066,4302, 77, 187,7330,3018, 37, 5,2972,7331,3876, # 3078 266, 152, 149, 468, 1898, 4066, 4302, 77, 187, 7330, 3018, 37, 5, 2972, 7331, 3876, # 3078
7332,7333, 39,2517,4303,2894,3177,2078, 55, 148, 74,4304, 545, 483,1474,1029, # 3094 7332, 7333, 39, 2517, 4303, 2894, 3177, 2078, 55, 148, 74, 4304, 545, 483, 1474, 1029, # 3094
1665, 217,1869,1531,3113,1104,2645,4067, 24, 172,3507, 900,3877,3508,3509,4305, # 3110 1665, 217, 1869, 1531, 3113, 1104, 2645, 4067, 24, 172, 3507, 900, 3877, 3508, 3509, 4305, # 3110
32,1408,2811,1312, 329, 487,2355,2247,2708, 784,2674, 4,3019,3314,1427,1788, # 3126 32, 1408, 2811, 1312, 329, 487, 2355, 2247, 2708, 784, 2674, 4, 3019, 3314, 1427, 1788, # 3126
188, 109, 499,7334,3620,1717,1789, 888,1217,3020,4306,7335,3510,7336,3315,1520, # 3142 188, 109, 499, 7334, 3620, 1717, 1789, 888, 1217, 3020, 4306, 7335, 3510, 7336, 3315, 1520, # 3142
3621,3878, 196,1034, 775,7337,7338, 929,1815, 249, 439, 38,7339,1063,7340, 794, # 3158 3621, 3878, 196, 1034, 775, 7337, 7338, 929, 1815, 249, 439, 38, 7339, 1063, 7340, 794, # 3158
3879,1435,2296, 46, 178,3245,2065,7341,2376,7342, 214,1709,4307, 804, 35, 707, # 3174 3879, 1435, 2296, 46, 178, 3245, 2065, 7341, 2376, 7342, 214, 1709, 4307, 804, 35, 707, # 3174
324,3622,1601,2546, 140, 459,4068,7343,7344,1365, 839, 272, 978,2257,2572,3409, # 3190 324, 3622, 1601, 2546, 140, 459, 4068, 7343, 7344, 1365, 839, 272, 978, 2257, 2572, 3409, # 3190
2128,1363,3623,1423, 697, 100,3071, 48, 70,1231, 495,3114,2193,7345,1294,7346, # 3206 2128, 1363, 3623, 1423, 697, 100, 3071, 48, 70, 1231, 495, 3114, 2193, 7345, 1294, 7346, # 3206
2079, 462, 586,1042,3246, 853, 256, 988, 185,2377,3410,1698, 434,1084,7347,3411, # 3222 2079, 462, 586, 1042, 3246, 853, 256, 988, 185, 2377, 3410, 1698, 434, 1084, 7347, 3411, # 3222
314,2615,2775,4308,2330,2331, 569,2280, 637,1816,2518, 757,1162,1878,1616,3412, # 3238 314, 2615, 2775, 4308, 2330, 2331, 569, 2280, 637, 1816, 2518, 757, 1162, 1878, 1616, 3412, # 3238
287,1577,2115, 768,4309,1671,2854,3511,2519,1321,3737, 909,2413,7348,4069, 933, # 3254 287, 1577, 2115, 768, 4309, 1671, 2854, 3511, 2519, 1321, 3737, 909, 2413, 7348, 4069, 933, # 3254
3738,7349,2052,2356,1222,4310, 765,2414,1322, 786,4311,7350,1919,1462,1677,2895, # 3270 3738, 7349, 2052, 2356, 1222, 4310, 765, 2414, 1322, 786, 4311, 7350, 1919, 1462, 1677, 2895, # 3270
1699,7351,4312,1424,2437,3115,3624,2590,3316,1774,1940,3413,3880,4070, 309,1369, # 3286 1699, 7351, 4312, 1424, 2437, 3115, 3624, 2590, 3316, 1774, 1940, 3413, 3880, 4070, 309, 1369, # 3286
1130,2812, 364,2230,1653,1299,3881,3512,3882,3883,2646, 525,1085,3021, 902,2000, # 3302 1130, 2812, 364, 2230, 1653, 1299, 3881, 3512, 3882, 3883, 2646, 525, 1085, 3021, 902, 2000, # 3302
1475, 964,4313, 421,1844,1415,1057,2281, 940,1364,3116, 376,4314,4315,1381, 7, # 3318 1475, 964, 4313, 421, 1844, 1415, 1057, 2281, 940, 1364, 3116, 376, 4314, 4315, 1381, 7, # 3318
2520, 983,2378, 336,1710,2675,1845, 321,3414, 559,1131,3022,2742,1808,1132,1313, # 3334 2520, 983, 2378, 336, 1710, 2675, 1845, 321, 3414, 559, 1131, 3022, 2742, 1808, 1132, 1313, # 3334
265,1481,1857,7352, 352,1203,2813,3247, 167,1089, 420,2814, 776, 792,1724,3513, # 3350 265, 1481, 1857, 7352, 352, 1203, 2813, 3247, 167, 1089, 420, 2814, 776, 792, 1724, 3513, # 3350
4071,2438,3248,7353,4072,7354, 446, 229, 333,2743, 901,3739,1200,1557,4316,2647, # 3366 4071, 2438, 3248, 7353, 4072, 7354, 446, 229, 333, 2743, 901, 3739, 1200, 1557, 4316, 2647, # 3366
1920, 395,2744,2676,3740,4073,1835, 125, 916,3178,2616,4317,7355,7356,3741,7357, # 3382 1920, 395, 2744, 2676, 3740, 4073, 1835, 125, 916, 3178, 2616, 4317, 7355, 7356, 3741, 7357, # 3382
7358,7359,4318,3117,3625,1133,2547,1757,3415,1510,2313,1409,3514,7360,2145, 438, # 3398 7358, 7359, 4318, 3117, 3625, 1133, 2547, 1757, 3415, 1510, 2313, 1409, 3514, 7360, 2145, 438, # 3398
2591,2896,2379,3317,1068, 958,3023, 461, 311,2855,2677,4074,1915,3179,4075,1978, # 3414 2591, 2896, 2379, 3317, 1068, 958, 3023, 461, 311, 2855, 2677, 4074, 1915, 3179, 4075, 1978, # 3414
383, 750,2745,2617,4076, 274, 539, 385,1278,1442,7361,1154,1964, 384, 561, 210, # 3430 383, 750, 2745, 2617, 4076, 274, 539, 385, 1278, 1442, 7361, 1154, 1964, 384, 561, 210, # 3430
98,1295,2548,3515,7362,1711,2415,1482,3416,3884,2897,1257, 129,7363,3742, 642, # 3446 98, 1295, 2548, 3515, 7362, 1711, 2415, 1482, 3416, 3884, 2897, 1257, 129, 7363, 3742, 642, # 3446
523,2776,2777,2648,7364, 141,2231,1333, 68, 176, 441, 876, 907,4077, 603,2592, # 3462 523, 2776, 2777, 2648, 7364, 141, 2231, 1333, 68, 176, 441, 876, 907, 4077, 603, 2592, # 3462
710, 171,3417, 404, 549, 18,3118,2393,1410,3626,1666,7365,3516,4319,2898,4320, # 3478 710, 171, 3417, 404, 549, 18, 3118, 2393, 1410, 3626, 1666, 7365, 3516, 4319, 2898, 4320, # 3478
7366,2973, 368,7367, 146, 366, 99, 871,3627,1543, 748, 807,1586,1185, 22,2258, # 3494 7366, 2973, 368, 7367, 146, 366, 99, 871, 3627, 1543, 748, 807, 1586, 1185, 22, 2258, # 3494
379,3743,3180,7368,3181, 505,1941,2618,1991,1382,2314,7369, 380,2357, 218, 702, # 3510 379, 3743, 3180, 7368, 3181, 505, 1941, 2618, 1991, 1382, 2314, 7369, 380, 2357, 218, 702, # 3510
1817,1248,3418,3024,3517,3318,3249,7370,2974,3628, 930,3250,3744,7371, 59,7372, # 3526 1817, 1248, 3418, 3024, 3517, 3318, 3249, 7370, 2974, 3628, 930, 3250, 3744, 7371, 59, 7372, # 3526
585, 601,4078, 497,3419,1112,1314,4321,1801,7373,1223,1472,2174,7374, 749,1836, # 3542 585, 601, 4078, 497, 3419, 1112, 1314, 4321, 1801, 7373, 1223, 1472, 2174, 7374, 749, 1836, # 3542
690,1899,3745,1772,3885,1476, 429,1043,1790,2232,2116, 917,4079, 447,1086,1629, # 3558 690, 1899, 3745, 1772, 3885, 1476, 429, 1043, 1790, 2232, 2116, 917, 4079, 447, 1086, 1629, # 3558
7375, 556,7376,7377,2020,1654, 844,1090, 105, 550, 966,1758,2815,1008,1782, 686, # 3574 7375, 556, 7376, 7377, 2020, 1654, 844, 1090, 105, 550, 966, 1758, 2815, 1008, 1782, 686, # 3574
1095,7378,2282, 793,1602,7379,3518,2593,4322,4080,2933,2297,4323,3746, 980,2496, # 3590 1095, 7378, 2282, 793, 1602, 7379, 3518, 2593, 4322, 4080, 2933, 2297, 4323, 3746, 980, 2496, # 3590
544, 353, 527,4324, 908,2678,2899,7380, 381,2619,1942,1348,7381,1341,1252, 560, # 3606 544, 353, 527, 4324, 908, 2678, 2899, 7380, 381, 2619, 1942, 1348, 7381, 1341, 1252, 560, # 3606
3072,7382,3420,2856,7383,2053, 973, 886,2080, 143,4325,7384,7385, 157,3886, 496, # 3622 3072, 7382, 3420, 2856, 7383, 2053, 973, 886, 2080, 143, 4325, 7384, 7385, 157, 3886, 496, # 3622
4081, 57, 840, 540,2038,4326,4327,3421,2117,1445, 970,2259,1748,1965,2081,4082, # 3638 4081, 57, 840, 540, 2038, 4326, 4327, 3421, 2117, 1445, 970, 2259, 1748, 1965, 2081, 4082, # 3638
3119,1234,1775,3251,2816,3629, 773,1206,2129,1066,2039,1326,3887,1738,1725,4083, # 3654 3119, 1234, 1775, 3251, 2816, 3629, 773, 1206, 2129, 1066, 2039, 1326, 3887, 1738, 1725, 4083, # 3654
279,3120, 51,1544,2594, 423,1578,2130,2066, 173,4328,1879,7386,7387,1583, 264, # 3670 279, 3120, 51, 1544, 2594, 423, 1578, 2130, 2066, 173, 4328, 1879, 7386, 7387, 1583, 264, # 3670
610,3630,4329,2439, 280, 154,7388,7389,7390,1739, 338,1282,3073, 693,2857,1411, # 3686 610, 3630, 4329, 2439, 280, 154, 7388, 7389, 7390, 1739, 338, 1282, 3073, 693, 2857, 1411, # 3686
1074,3747,2440,7391,4330,7392,7393,1240, 952,2394,7394,2900,1538,2679, 685,1483, # 3702 1074, 3747, 2440, 7391, 4330, 7392, 7393, 1240, 952, 2394, 7394, 2900, 1538, 2679, 685, 1483, # 3702
4084,2468,1436, 953,4085,2054,4331, 671,2395, 79,4086,2441,3252, 608, 567,2680, # 3718 4084, 2468, 1436, 953, 4085, 2054, 4331, 671, 2395, 79, 4086, 2441, 3252, 608, 567, 2680, # 3718
3422,4087,4088,1691, 393,1261,1791,2396,7395,4332,7396,7397,7398,7399,1383,1672, # 3734 3422, 4087, 4088, 1691, 393, 1261, 1791, 2396, 7395, 4332, 7396, 7397, 7398, 7399, 1383, 1672, # 3734
3748,3182,1464, 522,1119, 661,1150, 216, 675,4333,3888,1432,3519, 609,4334,2681, # 3750 3748, 3182, 1464, 522, 1119, 661, 1150, 216, 675, 4333, 3888, 1432, 3519, 609, 4334, 2681, # 3750
2397,7400,7401,7402,4089,3025, 0,7403,2469, 315, 231,2442, 301,3319,4335,2380, # 3766 2397, 7400, 7401, 7402, 4089, 3025, 0, 7403, 2469, 315, 231, 2442, 301, 3319, 4335, 2380, # 3766
7404, 233,4090,3631,1818,4336,4337,7405, 96,1776,1315,2082,7406, 257,7407,1809, # 3782 7404, 233, 4090, 3631, 1818, 4336, 4337, 7405, 96, 1776, 1315, 2082, 7406, 257, 7407, 1809, # 3782
3632,2709,1139,1819,4091,2021,1124,2163,2778,1777,2649,7408,3074, 363,1655,3183, # 3798 3632, 2709, 1139, 1819, 4091, 2021, 1124, 2163, 2778, 1777, 2649, 7408, 3074, 363, 1655, 3183, # 3798
7409,2975,7410,7411,7412,3889,1567,3890, 718, 103,3184, 849,1443, 341,3320,2934, # 3814 7409, 2975, 7410, 7411, 7412, 3889, 1567, 3890, 718, 103, 3184, 849, 1443, 341, 3320, 2934, # 3814
1484,7413,1712, 127, 67, 339,4092,2398, 679,1412, 821,7414,7415, 834, 738, 351, # 3830 1484, 7413, 1712, 127, 67, 339, 4092, 2398, 679, 1412, 821, 7414, 7415, 834, 738, 351, # 3830
2976,2146, 846, 235,1497,1880, 418,1992,3749,2710, 186,1100,2147,2746,3520,1545, # 3846 2976, 2146, 846, 235, 1497, 1880, 418, 1992, 3749, 2710, 186, 1100, 2147, 2746, 3520, 1545, # 3846
1355,2935,2858,1377, 583,3891,4093,2573,2977,7416,1298,3633,1078,2549,3634,2358, # 3862 1355, 2935, 2858, 1377, 583, 3891, 4093, 2573, 2977, 7416, 1298, 3633, 1078, 2549, 3634, 2358, # 3862
78,3750,3751, 267,1289,2099,2001,1594,4094, 348, 369,1274,2194,2175,1837,4338, # 3878 78, 3750, 3751, 267, 1289, 2099, 2001, 1594, 4094, 348, 369, 1274, 2194, 2175, 1837, 4338, # 3878
1820,2817,3635,2747,2283,2002,4339,2936,2748, 144,3321, 882,4340,3892,2749,3423, # 3894 1820, 2817, 3635, 2747, 2283, 2002, 4339, 2936, 2748, 144, 3321, 882, 4340, 3892, 2749, 3423, # 3894
4341,2901,7417,4095,1726, 320,7418,3893,3026, 788,2978,7419,2818,1773,1327,2859, # 3910 4341, 2901, 7417, 4095, 1726, 320, 7418, 3893, 3026, 788, 2978, 7419, 2818, 1773, 1327, 2859, # 3910
3894,2819,7420,1306,4342,2003,1700,3752,3521,2359,2650, 787,2022, 506, 824,3636, # 3926 3894, 2819, 7420, 1306, 4342, 2003, 1700, 3752, 3521, 2359, 2650, 787, 2022, 506, 824, 3636, # 3926
534, 323,4343,1044,3322,2023,1900, 946,3424,7421,1778,1500,1678,7422,1881,4344, # 3942 534, 323, 4343, 1044, 3322, 2023, 1900, 946, 3424, 7421, 1778, 1500, 1678, 7422, 1881, 4344, # 3942
165, 243,4345,3637,2521, 123, 683,4096, 764,4346, 36,3895,1792, 589,2902, 816, # 3958 165, 243, 4345, 3637, 2521, 123, 683, 4096, 764, 4346, 36, 3895, 1792, 589, 2902, 816, # 3958
626,1667,3027,2233,1639,1555,1622,3753,3896,7423,3897,2860,1370,1228,1932, 891, # 3974 626, 1667, 3027, 2233, 1639, 1555, 1622, 3753, 3896, 7423, 3897, 2860, 1370, 1228, 1932, 891, # 3974
2083,2903, 304,4097,7424, 292,2979,2711,3522, 691,2100,4098,1115,4347, 118, 662, # 3990 2083, 2903, 304, 4097, 7424, 292, 2979, 2711, 3522, 691, 2100, 4098, 1115, 4347, 118, 662, # 3990
7425, 611,1156, 854,2381,1316,2861, 2, 386, 515,2904,7426,7427,3253, 868,2234, # 4006 7425, 611, 1156, 854, 2381, 1316, 2861, 2, 386, 515, 2904, 7426, 7427, 3253, 868, 2234, # 4006
1486, 855,2651, 785,2212,3028,7428,1040,3185,3523,7429,3121, 448,7430,1525,7431, # 4022 1486, 855, 2651, 785, 2212, 3028, 7428, 1040, 3185, 3523, 7429, 3121, 448, 7430, 1525, 7431, # 4022
2164,4348,7432,3754,7433,4099,2820,3524,3122, 503, 818,3898,3123,1568, 814, 676, # 4038 2164, 4348, 7432, 3754, 7433, 4099, 2820, 3524, 3122, 503, 818, 3898, 3123, 1568, 814, 676, # 4038
1444, 306,1749,7434,3755,1416,1030, 197,1428, 805,2821,1501,4349,7435,7436,7437, # 4054 1444, 306, 1749, 7434, 3755, 1416, 1030, 197, 1428, 805, 2821, 1501, 4349, 7435, 7436, 7437, # 4054
1993,7438,4350,7439,7440,2195, 13,2779,3638,2980,3124,1229,1916,7441,3756,2131, # 4070 1993, 7438, 4350, 7439, 7440, 2195, 13, 2779, 3638, 2980, 3124, 1229, 1916, 7441, 3756, 2131, # 4070
7442,4100,4351,2399,3525,7443,2213,1511,1727,1120,7444,7445, 646,3757,2443, 307, # 4086 7442, 4100, 4351, 2399, 3525, 7443, 2213, 1511, 1727, 1120, 7444, 7445, 646, 3757, 2443, 307, # 4086
7446,7447,1595,3186,7448,7449,7450,3639,1113,1356,3899,1465,2522,2523,7451, 519, # 4102 7446, 7447, 1595, 3186, 7448, 7449, 7450, 3639, 1113, 1356, 3899, 1465, 2522, 2523, 7451, 519, # 4102
7452, 128,2132, 92,2284,1979,7453,3900,1512, 342,3125,2196,7454,2780,2214,1980, # 4118 7452, 128, 2132, 92, 2284, 1979, 7453, 3900, 1512, 342, 3125, 2196, 7454, 2780, 2214, 1980, # 4118
3323,7455, 290,1656,1317, 789, 827,2360,7456,3758,4352, 562, 581,3901,7457, 401, # 4134 3323, 7455, 290, 1656, 1317, 789, 827, 2360, 7456, 3758, 4352, 562, 581, 3901, 7457, 401, # 4134
4353,2248, 94,4354,1399,2781,7458,1463,2024,4355,3187,1943,7459, 828,1105,4101, # 4150 4353, 2248, 94, 4354, 1399, 2781, 7458, 1463, 2024, 4355, 3187, 1943, 7459, 828, 1105, 4101, # 4150
1262,1394,7460,4102, 605,4356,7461,1783,2862,7462,2822, 819,2101, 578,2197,2937, # 4166 1262, 1394, 7460, 4102, 605, 4356, 7461, 1783, 2862, 7462, 2822, 819, 2101, 578, 2197, 2937, # 4166
7463,1502, 436,3254,4103,3255,2823,3902,2905,3425,3426,7464,2712,2315,7465,7466, # 4182 7463, 1502, 436, 3254, 4103, 3255, 2823, 3902, 2905, 3425, 3426, 7464, 2712, 2315, 7465, 7466, # 4182
2332,2067, 23,4357, 193, 826,3759,2102, 699,1630,4104,3075, 390,1793,1064,3526, # 4198 2332, 2067, 23, 4357, 193, 826, 3759, 2102, 699, 1630, 4104, 3075, 390, 1793, 1064, 3526, # 4198
7467,1579,3076,3077,1400,7468,4105,1838,1640,2863,7469,4358,4359, 137,4106, 598, # 4214 7467, 1579, 3076, 3077, 1400, 7468, 4105, 1838, 1640, 2863, 7469, 4358, 4359, 137, 4106, 598, # 4214
3078,1966, 780, 104, 974,2938,7470, 278, 899, 253, 402, 572, 504, 493,1339,7471, # 4230 3078, 1966, 780, 104, 974, 2938, 7470, 278, 899, 253, 402, 572, 504, 493, 1339, 7471, # 4230
3903,1275,4360,2574,2550,7472,3640,3029,3079,2249, 565,1334,2713, 863, 41,7473, # 4246 3903, 1275, 4360, 2574, 2550, 7472, 3640, 3029, 3079, 2249, 565, 1334, 2713, 863, 41, 7473, # 4246
7474,4361,7475,1657,2333, 19, 463,2750,4107, 606,7476,2981,3256,1087,2084,1323, # 4262 7474, 4361, 7475, 1657, 2333, 19, 463, 2750, 4107, 606, 7476, 2981, 3256, 1087, 2084, 1323, # 4262
2652,2982,7477,1631,1623,1750,4108,2682,7478,2864, 791,2714,2653,2334, 232,2416, # 4278 2652, 2982, 7477, 1631, 1623, 1750, 4108, 2682, 7478, 2864, 791, 2714, 2653, 2334, 232, 2416, # 4278
7479,2983,1498,7480,2654,2620, 755,1366,3641,3257,3126,2025,1609, 119,1917,3427, # 4294 7479, 2983, 1498, 7480, 2654, 2620, 755, 1366, 3641, 3257, 3126, 2025, 1609, 119, 1917, 3427, # 4294
862,1026,4109,7481,3904,3760,4362,3905,4363,2260,1951,2470,7482,1125, 817,4110, # 4310 862, 1026, 4109, 7481, 3904, 3760, 4362, 3905, 4363, 2260, 1951, 2470, 7482, 1125, 817, 4110, # 4310
4111,3906,1513,1766,2040,1487,4112,3030,3258,2824,3761,3127,7483,7484,1507,7485, # 4326 4111, 3906, 1513, 1766, 2040, 1487, 4112, 3030, 3258, 2824, 3761, 3127, 7483, 7484, 1507, 7485, # 4326
2683, 733, 40,1632,1106,2865, 345,4113, 841,2524, 230,4364,2984,1846,3259,3428, # 4342 2683, 733, 40, 1632, 1106, 2865, 345, 4113, 841, 2524, 230, 4364, 2984, 1846, 3259, 3428, # 4342
7486,1263, 986,3429,7487, 735, 879, 254,1137, 857, 622,1300,1180,1388,1562,3907, # 4358 7486, 1263, 986, 3429, 7487, 735, 879, 254, 1137, 857, 622, 1300, 1180, 1388, 1562, 3907, # 4358
3908,2939, 967,2751,2655,1349, 592,2133,1692,3324,2985,1994,4114,1679,3909,1901, # 4374 3908, 2939, 967, 2751, 2655, 1349, 592, 2133, 1692, 3324, 2985, 1994, 4114, 1679, 3909, 1901, # 4374
2185,7488, 739,3642,2715,1296,1290,7489,4115,2198,2199,1921,1563,2595,2551,1870, # 4390 2185, 7488, 739, 3642, 2715, 1296, 1290, 7489, 4115, 2198, 2199, 1921, 1563, 2595, 2551, 1870, # 4390
2752,2986,7490, 435,7491, 343,1108, 596, 17,1751,4365,2235,3430,3643,7492,4366, # 4406 2752, 2986, 7490, 435, 7491, 343, 1108, 596, 17, 1751, 4365, 2235, 3430, 3643, 7492, 4366, # 4406
294,3527,2940,1693, 477, 979, 281,2041,3528, 643,2042,3644,2621,2782,2261,1031, # 4422 294, 3527, 2940, 1693, 477, 979, 281, 2041, 3528, 643, 2042, 3644, 2621, 2782, 2261, 1031, # 4422
2335,2134,2298,3529,4367, 367,1249,2552,7493,3530,7494,4368,1283,3325,2004, 240, # 4438 2335, 2134, 2298, 3529, 4367, 367, 1249, 2552, 7493, 3530, 7494, 4368, 1283, 3325, 2004, 240, # 4438
1762,3326,4369,4370, 836,1069,3128, 474,7495,2148,2525, 268,3531,7496,3188,1521, # 4454 1762, 3326, 4369, 4370, 836, 1069, 3128, 474, 7495, 2148, 2525, 268, 3531, 7496, 3188, 1521, # 4454
1284,7497,1658,1546,4116,7498,3532,3533,7499,4117,3327,2684,1685,4118, 961,1673, # 4470 1284, 7497, 1658, 1546, 4116, 7498, 3532, 3533, 7499, 4117, 3327, 2684, 1685, 4118, 961, 1673, # 4470
2622, 190,2005,2200,3762,4371,4372,7500, 570,2497,3645,1490,7501,4373,2623,3260, # 4486 2622, 190, 2005, 2200, 3762, 4371, 4372, 7500, 570, 2497, 3645, 1490, 7501, 4373, 2623, 3260, # 4486
1956,4374, 584,1514, 396,1045,1944,7502,4375,1967,2444,7503,7504,4376,3910, 619, # 4502 1956, 4374, 584, 1514, 396, 1045, 1944, 7502, 4375, 1967, 2444, 7503, 7504, 4376, 3910, 619, # 4502
7505,3129,3261, 215,2006,2783,2553,3189,4377,3190,4378, 763,4119,3763,4379,7506, # 4518 7505, 3129, 3261, 215, 2006, 2783, 2553, 3189, 4377, 3190, 4378, 763, 4119, 3763, 4379, 7506, # 4518
7507,1957,1767,2941,3328,3646,1174, 452,1477,4380,3329,3130,7508,2825,1253,2382, # 4534 7507, 1957, 1767, 2941, 3328, 3646, 1174, 452, 1477, 4380, 3329, 3130, 7508, 2825, 1253, 2382, # 4534
2186,1091,2285,4120, 492,7509, 638,1169,1824,2135,1752,3911, 648, 926,1021,1324, # 4550 2186, 1091, 2285, 4120, 492, 7509, 638, 1169, 1824, 2135, 1752, 3911, 648, 926, 1021, 1324, # 4550
4381, 520,4382, 997, 847,1007, 892,4383,3764,2262,1871,3647,7510,2400,1784,4384, # 4566 4381, 520, 4382, 997, 847, 1007, 892, 4383, 3764, 2262, 1871, 3647, 7510, 2400, 1784, 4384, # 4566
1952,2942,3080,3191,1728,4121,2043,3648,4385,2007,1701,3131,1551, 30,2263,4122, # 4582 1952, 2942, 3080, 3191, 1728, 4121, 2043, 3648, 4385, 2007, 1701, 3131, 1551, 30, 2263, 4122, # 4582
7511,2026,4386,3534,7512, 501,7513,4123, 594,3431,2165,1821,3535,3432,3536,3192, # 4598 7511, 2026, 4386, 3534, 7512, 501, 7513, 4123, 594, 3431, 2165, 1821, 3535, 3432, 3536, 3192, # 4598
829,2826,4124,7514,1680,3132,1225,4125,7515,3262,4387,4126,3133,2336,7516,4388, # 4614 829, 2826, 4124, 7514, 1680, 3132, 1225, 4125, 7515, 3262, 4387, 4126, 3133, 2336, 7516, 4388, # 4614
4127,7517,3912,3913,7518,1847,2383,2596,3330,7519,4389, 374,3914, 652,4128,4129, # 4630 4127, 7517, 3912, 3913, 7518, 1847, 2383, 2596, 3330, 7519, 4389, 374, 3914, 652, 4128, 4129, # 4630
375,1140, 798,7520,7521,7522,2361,4390,2264, 546,1659, 138,3031,2445,4391,7523, # 4646 375, 1140, 798, 7520, 7521, 7522, 2361, 4390, 2264, 546, 1659, 138, 3031, 2445, 4391, 7523, # 4646
2250, 612,1848, 910, 796,3765,1740,1371, 825,3766,3767,7524,2906,2554,7525, 692, # 4662 2250, 612, 1848, 910, 796, 3765, 1740, 1371, 825, 3766, 3767, 7524, 2906, 2554, 7525, 692, # 4662
444,3032,2624, 801,4392,4130,7526,1491, 244,1053,3033,4131,4132, 340,7527,3915, # 4678 444, 3032, 2624, 801, 4392, 4130, 7526, 1491, 244, 1053, 3033, 4131, 4132, 340, 7527, 3915, # 4678
1041,2987, 293,1168, 87,1357,7528,1539, 959,7529,2236, 721, 694,4133,3768, 219, # 4694 1041, 2987, 293, 1168, 87, 1357, 7528, 1539, 959, 7529, 2236, 721, 694, 4133, 3768, 219, # 4694
1478, 644,1417,3331,2656,1413,1401,1335,1389,3916,7530,7531,2988,2362,3134,1825, # 4710 1478, 644, 1417, 3331, 2656, 1413, 1401, 1335, 1389, 3916, 7530, 7531, 2988, 2362, 3134, 1825, # 4710
730,1515, 184,2827, 66,4393,7532,1660,2943, 246,3332, 378,1457, 226,3433, 975, # 4726 730, 1515, 184, 2827, 66, 4393, 7532, 1660, 2943, 246, 3332, 378, 1457, 226, 3433, 975, # 4726
3917,2944,1264,3537, 674, 696,7533, 163,7534,1141,2417,2166, 713,3538,3333,4394, # 4742 3917, 2944, 1264, 3537, 674, 696, 7533, 163, 7534, 1141, 2417, 2166, 713, 3538, 3333, 4394, # 4742
3918,7535,7536,1186, 15,7537,1079,1070,7538,1522,3193,3539, 276,1050,2716, 758, # 4758 3918, 7535, 7536, 1186, 15, 7537, 1079, 1070, 7538, 1522, 3193, 3539, 276, 1050, 2716, 758, # 4758
1126, 653,2945,3263,7539,2337, 889,3540,3919,3081,2989, 903,1250,4395,3920,3434, # 4774 1126, 653, 2945, 3263, 7539, 2337, 889, 3540, 3919, 3081, 2989, 903, 1250, 4395, 3920, 3434, # 4774
3541,1342,1681,1718, 766,3264, 286, 89,2946,3649,7540,1713,7541,2597,3334,2990, # 4790 3541, 1342, 1681, 1718, 766, 3264, 286, 89, 2946, 3649, 7540, 1713, 7541, 2597, 3334, 2990, # 4790
7542,2947,2215,3194,2866,7543,4396,2498,2526, 181, 387,1075,3921, 731,2187,3335, # 4806 7542, 2947, 2215, 3194, 2866, 7543, 4396, 2498, 2526, 181, 387, 1075, 3921, 731, 2187, 3335, # 4806
7544,3265, 310, 313,3435,2299, 770,4134, 54,3034, 189,4397,3082,3769,3922,7545, # 4822 7544, 3265, 310, 313, 3435, 2299, 770, 4134, 54, 3034, 189, 4397, 3082, 3769, 3922, 7545, # 4822
1230,1617,1849, 355,3542,4135,4398,3336, 111,4136,3650,1350,3135,3436,3035,4137, # 4838 1230, 1617, 1849, 355, 3542, 4135, 4398, 3336, 111, 4136, 3650, 1350, 3135, 3436, 3035, 4137, # 4838
2149,3266,3543,7546,2784,3923,3924,2991, 722,2008,7547,1071, 247,1207,2338,2471, # 4854 2149, 3266, 3543, 7546, 2784, 3923, 3924, 2991, 722, 2008, 7547, 1071, 247, 1207, 2338, 2471, # 4854
1378,4399,2009, 864,1437,1214,4400, 373,3770,1142,2216, 667,4401, 442,2753,2555, # 4870 1378, 4399, 2009, 864, 1437, 1214, 4400, 373, 3770, 1142, 2216, 667, 4401, 442, 2753, 2555, # 4870
3771,3925,1968,4138,3267,1839, 837, 170,1107, 934,1336,1882,7548,7549,2118,4139, # 4886 3771, 3925, 1968, 4138, 3267, 1839, 837, 170, 1107, 934, 1336, 1882, 7548, 7549, 2118, 4139, # 4886
2828, 743,1569,7550,4402,4140, 582,2384,1418,3437,7551,1802,7552, 357,1395,1729, # 4902 2828, 743, 1569, 7550, 4402, 4140, 582, 2384, 1418, 3437, 7551, 1802, 7552, 357, 1395, 1729, # 4902
3651,3268,2418,1564,2237,7553,3083,3772,1633,4403,1114,2085,4141,1532,7554, 482, # 4918 3651, 3268, 2418, 1564, 2237, 7553, 3083, 3772, 1633, 4403, 1114, 2085, 4141, 1532, 7554, 482, # 4918
2446,4404,7555,7556,1492, 833,1466,7557,2717,3544,1641,2829,7558,1526,1272,3652, # 4934 2446, 4404, 7555, 7556, 1492, 833, 1466, 7557, 2717, 3544, 1641, 2829, 7558, 1526, 1272, 3652, # 4934
4142,1686,1794, 416,2556,1902,1953,1803,7559,3773,2785,3774,1159,2316,7560,2867, # 4950 4142, 1686, 1794, 416, 2556, 1902, 1953, 1803, 7559, 3773, 2785, 3774, 1159, 2316, 7560, 2867, # 4950
4405,1610,1584,3036,2419,2754, 443,3269,1163,3136,7561,7562,3926,7563,4143,2499, # 4966 4405, 1610, 1584, 3036, 2419, 2754, 443, 3269, 1163, 3136, 7561, 7562, 3926, 7563, 4143, 2499, # 4966
3037,4406,3927,3137,2103,1647,3545,2010,1872,4144,7564,4145, 431,3438,7565, 250, # 4982 3037, 4406, 3927, 3137, 2103, 1647, 3545, 2010, 1872, 4144, 7564, 4145, 431, 3438, 7565, 250, # 4982
97, 81,4146,7566,1648,1850,1558, 160, 848,7567, 866, 740,1694,7568,2201,2830, # 4998 97, 81, 4146, 7566, 1648, 1850, 1558, 160, 848, 7567, 866, 740, 1694, 7568, 2201, 2830, # 4998
3195,4147,4407,3653,1687, 950,2472, 426, 469,3196,3654,3655,3928,7569,7570,1188, # 5014 3195, 4147, 4407, 3653, 1687, 950, 2472, 426, 469, 3196, 3654, 3655, 3928, 7569, 7570, 1188, # 5014
424,1995, 861,3546,4148,3775,2202,2685, 168,1235,3547,4149,7571,2086,1674,4408, # 5030 424, 1995, 861, 3546, 4148, 3775, 2202, 2685, 168, 1235, 3547, 4149, 7571, 2086, 1674, 4408, # 5030
3337,3270, 220,2557,1009,7572,3776, 670,2992, 332,1208, 717,7573,7574,3548,2447, # 5046 3337, 3270, 220, 2557, 1009, 7572, 3776, 670, 2992, 332, 1208, 717, 7573, 7574, 3548, 2447, # 5046
3929,3338,7575, 513,7576,1209,2868,3339,3138,4409,1080,7577,7578,7579,7580,2527, # 5062 3929, 3338, 7575, 513, 7576, 1209, 2868, 3339, 3138, 4409, 1080, 7577, 7578, 7579, 7580, 2527, # 5062
3656,3549, 815,1587,3930,3931,7581,3550,3439,3777,1254,4410,1328,3038,1390,3932, # 5078 3656, 3549, 815, 1587, 3930, 3931, 7581, 3550, 3439, 3777, 1254, 4410, 1328, 3038, 1390, 3932, # 5078
1741,3933,3778,3934,7582, 236,3779,2448,3271,7583,7584,3657,3780,1273,3781,4411, # 5094 1741, 3933, 3778, 3934, 7582, 236, 3779, 2448, 3271, 7583, 7584, 3657, 3780, 1273, 3781, 4411, # 5094
7585, 308,7586,4412, 245,4413,1851,2473,1307,2575, 430, 715,2136,2449,7587, 270, # 5110 7585, 308, 7586, 4412, 245, 4413, 1851, 2473, 1307, 2575, 430, 715, 2136, 2449, 7587, 270, # 5110
199,2869,3935,7588,3551,2718,1753, 761,1754, 725,1661,1840,4414,3440,3658,7589, # 5126 199, 2869, 3935, 7588, 3551, 2718, 1753, 761, 1754, 725, 1661, 1840, 4414, 3440, 3658, 7589, # 5126
7590, 587, 14,3272, 227,2598, 326, 480,2265, 943,2755,3552, 291, 650,1883,7591, # 5142 7590, 587, 14, 3272, 227, 2598, 326, 480, 2265, 943, 2755, 3552, 291, 650, 1883, 7591, # 5142
1702,1226, 102,1547, 62,3441, 904,4415,3442,1164,4150,7592,7593,1224,1548,2756, # 5158 1702, 1226, 102, 1547, 62, 3441, 904, 4415, 3442, 1164, 4150, 7592, 7593, 1224, 1548, 2756, # 5158
391, 498,1493,7594,1386,1419,7595,2055,1177,4416, 813, 880,1081,2363, 566,1145, # 5174 391, 498, 1493, 7594, 1386, 1419, 7595, 2055, 1177, 4416, 813, 880, 1081, 2363, 566, 1145, # 5174
4417,2286,1001,1035,2558,2599,2238, 394,1286,7596,7597,2068,7598, 86,1494,1730, # 5190 4417, 2286, 1001, 1035, 2558, 2599, 2238, 394, 1286, 7596, 7597, 2068, 7598, 86, 1494, 1730, # 5190
3936, 491,1588, 745, 897,2948, 843,3340,3937,2757,2870,3273,1768, 998,2217,2069, # 5206 3936, 491, 1588, 745, 897, 2948, 843, 3340, 3937, 2757, 2870, 3273, 1768, 998, 2217, 2069, # 5206
397,1826,1195,1969,3659,2993,3341, 284,7599,3782,2500,2137,2119,1903,7600,3938, # 5222 397, 1826, 1195, 1969, 3659, 2993, 3341, 284, 7599, 3782, 2500, 2137, 2119, 1903, 7600, 3938, # 5222
2150,3939,4151,1036,3443,1904, 114,2559,4152, 209,1527,7601,7602,2949,2831,2625, # 5238 2150, 3939, 4151, 1036, 3443, 1904, 114, 2559, 4152, 209, 1527, 7601, 7602, 2949, 2831, 2625, # 5238
2385,2719,3139, 812,2560,7603,3274,7604,1559, 737,1884,3660,1210, 885, 28,2686, # 5254 2385, 2719, 3139, 812, 2560, 7603, 3274, 7604, 1559, 737, 1884, 3660, 1210, 885, 28, 2686, # 5254
3553,3783,7605,4153,1004,1779,4418,7606, 346,1981,2218,2687,4419,3784,1742, 797, # 5270 3553, 3783, 7605, 4153, 1004, 1779, 4418, 7606, 346, 1981, 2218, 2687, 4419, 3784, 1742, 797, # 5270
1642,3940,1933,1072,1384,2151, 896,3941,3275,3661,3197,2871,3554,7607,2561,1958, # 5286 1642, 3940, 1933, 1072, 1384, 2151, 896, 3941, 3275, 3661, 3197, 2871, 3554, 7607, 2561, 1958, # 5286
4420,2450,1785,7608,7609,7610,3942,4154,1005,1308,3662,4155,2720,4421,4422,1528, # 5302 4420, 2450, 1785, 7608, 7609, 7610, 3942, 4154, 1005, 1308, 3662, 4155, 2720, 4421, 4422, 1528, # 5302
2600, 161,1178,4156,1982, 987,4423,1101,4157, 631,3943,1157,3198,2420,1343,1241, # 5318 2600, 161, 1178, 4156, 1982, 987, 4423, 1101, 4157, 631, 3943, 1157, 3198, 2420, 1343, 1241, # 5318
1016,2239,2562, 372, 877,2339,2501,1160, 555,1934, 911,3944,7611, 466,1170, 169, # 5334 1016, 2239, 2562, 372, 877, 2339, 2501, 1160, 555, 1934, 911, 3944, 7611, 466, 1170, 169, # 5334
1051,2907,2688,3663,2474,2994,1182,2011,2563,1251,2626,7612, 992,2340,3444,1540, # 5350 1051, 2907, 2688, 3663, 2474, 2994, 1182, 2011, 2563, 1251, 2626, 7612, 992, 2340, 3444, 1540, # 5350
2721,1201,2070,2401,1996,2475,7613,4424, 528,1922,2188,1503,1873,1570,2364,3342, # 5366 2721, 1201, 2070, 2401, 1996, 2475, 7613, 4424, 528, 1922, 2188, 1503, 1873, 1570, 2364, 3342, # 5366
3276,7614, 557,1073,7615,1827,3445,2087,2266,3140,3039,3084, 767,3085,2786,4425, # 5382 3276, 7614, 557, 1073, 7615, 1827, 3445, 2087, 2266, 3140, 3039, 3084, 767, 3085, 2786, 4425, # 5382
1006,4158,4426,2341,1267,2176,3664,3199, 778,3945,3200,2722,1597,2657,7616,4427, # 5398 1006, 4158, 4426, 2341, 1267, 2176, 3664, 3199, 778, 3945, 3200, 2722, 1597, 2657, 7616, 4427, # 5398
7617,3446,7618,7619,7620,3277,2689,1433,3278, 131, 95,1504,3946, 723,4159,3141, # 5414 7617, 3446, 7618, 7619, 7620, 3277, 2689, 1433, 3278, 131, 95, 1504, 3946, 723, 4159, 3141, # 5414
1841,3555,2758,2189,3947,2027,2104,3665,7621,2995,3948,1218,7622,3343,3201,3949, # 5430 1841, 3555, 2758, 2189, 3947, 2027, 2104, 3665, 7621, 2995, 3948, 1218, 7622, 3343, 3201, 3949, # 5430
4160,2576, 248,1634,3785, 912,7623,2832,3666,3040,3786, 654, 53,7624,2996,7625, # 5446 4160, 2576, 248, 1634, 3785, 912, 7623, 2832, 3666, 3040, 3786, 654, 53, 7624, 2996, 7625, # 5446
1688,4428, 777,3447,1032,3950,1425,7626, 191, 820,2120,2833, 971,4429, 931,3202, # 5462 1688, 4428, 777, 3447, 1032, 3950, 1425, 7626, 191, 820, 2120, 2833, 971, 4429, 931, 3202, # 5462
135, 664, 783,3787,1997, 772,2908,1935,3951,3788,4430,2909,3203, 282,2723, 640, # 5478 135, 664, 783, 3787, 1997, 772, 2908, 1935, 3951, 3788, 4430, 2909, 3203, 282, 2723, 640, # 5478
1372,3448,1127, 922, 325,3344,7627,7628, 711,2044,7629,7630,3952,2219,2787,1936, # 5494 1372, 3448, 1127, 922, 325, 3344, 7627, 7628, 711, 2044, 7629, 7630, 3952, 2219, 2787, 1936, # 5494
3953,3345,2220,2251,3789,2300,7631,4431,3790,1258,3279,3954,3204,2138,2950,3955, # 5510 3953, 3345, 2220, 2251, 3789, 2300, 7631, 4431, 3790, 1258, 3279, 3954, 3204, 2138, 2950, 3955, # 5510
3956,7632,2221, 258,3205,4432, 101,1227,7633,3280,1755,7634,1391,3281,7635,2910, # 5526 3956, 7632, 2221, 258, 3205, 4432, 101, 1227, 7633, 3280, 1755, 7634, 1391, 3281, 7635, 2910, # 5526
2056, 893,7636,7637,7638,1402,4161,2342,7639,7640,3206,3556,7641,7642, 878,1325, # 5542 2056, 893, 7636, 7637, 7638, 1402, 4161, 2342, 7639, 7640, 3206, 3556, 7641, 7642, 878, 1325, # 5542
1780,2788,4433, 259,1385,2577, 744,1183,2267,4434,7643,3957,2502,7644, 684,1024, # 5558 1780, 2788, 4433, 259, 1385, 2577, 744, 1183, 2267, 4434, 7643, 3957, 2502, 7644, 684, 1024, # 5558
4162,7645, 472,3557,3449,1165,3282,3958,3959, 322,2152, 881, 455,1695,1152,1340, # 5574 4162, 7645, 472, 3557, 3449, 1165, 3282, 3958, 3959, 322, 2152, 881, 455, 1695, 1152, 1340, # 5574
660, 554,2153,4435,1058,4436,4163, 830,1065,3346,3960,4437,1923,7646,1703,1918, # 5590 660, 554, 2153, 4435, 1058, 4436, 4163, 830, 1065, 3346, 3960, 4437, 1923, 7646, 1703, 1918, # 5590
7647, 932,2268, 122,7648,4438, 947, 677,7649,3791,2627, 297,1905,1924,2269,4439, # 5606 7647, 932, 2268, 122, 7648, 4438, 947, 677, 7649, 3791, 2627, 297, 1905, 1924, 2269, 4439, # 5606
2317,3283,7650,7651,4164,7652,4165, 84,4166, 112, 989,7653, 547,1059,3961, 701, # 5622 2317, 3283, 7650, 7651, 4164, 7652, 4165, 84, 4166, 112, 989, 7653, 547, 1059, 3961, 701, # 5622
3558,1019,7654,4167,7655,3450, 942, 639, 457,2301,2451, 993,2951, 407, 851, 494, # 5638 3558, 1019, 7654, 4167, 7655, 3450, 942, 639, 457, 2301, 2451, 993, 2951, 407, 851, 494, # 5638
4440,3347, 927,7656,1237,7657,2421,3348, 573,4168, 680, 921,2911,1279,1874, 285, # 5654 4440, 3347, 927, 7656, 1237, 7657, 2421, 3348, 573, 4168, 680, 921, 2911, 1279, 1874, 285, # 5654
790,1448,1983, 719,2167,7658,7659,4441,3962,3963,1649,7660,1541, 563,7661,1077, # 5670 790, 1448, 1983, 719, 2167, 7658, 7659, 4441, 3962, 3963, 1649, 7660, 1541, 563, 7661, 1077, # 5670
7662,3349,3041,3451, 511,2997,3964,3965,3667,3966,1268,2564,3350,3207,4442,4443, # 5686 7662, 3349, 3041, 3451, 511, 2997, 3964, 3965, 3667, 3966, 1268, 2564, 3350, 3207, 4442, 4443, # 5686
7663, 535,1048,1276,1189,2912,2028,3142,1438,1373,2834,2952,1134,2012,7664,4169, # 5702 7663, 535, 1048, 1276, 1189, 2912, 2028, 3142, 1438, 1373, 2834, 2952, 1134, 2012, 7664, 4169, # 5702
1238,2578,3086,1259,7665, 700,7666,2953,3143,3668,4170,7667,4171,1146,1875,1906, # 5718 1238, 2578, 3086, 1259, 7665, 700, 7666, 2953, 3143, 3668, 4170, 7667, 4171, 1146, 1875, 1906, # 5718
4444,2601,3967, 781,2422, 132,1589, 203, 147, 273,2789,2402, 898,1786,2154,3968, # 5734 4444, 2601, 3967, 781, 2422, 132, 1589, 203, 147, 273, 2789, 2402, 898, 1786, 2154, 3968, # 5734
3969,7668,3792,2790,7669,7670,4445,4446,7671,3208,7672,1635,3793, 965,7673,1804, # 5750 3969, 7668, 3792, 2790, 7669, 7670, 4445, 4446, 7671, 3208, 7672, 1635, 3793, 965, 7673, 1804, # 5750
2690,1516,3559,1121,1082,1329,3284,3970,1449,3794, 65,1128,2835,2913,2759,1590, # 5766 2690, 1516, 3559, 1121, 1082, 1329, 3284, 3970, 1449, 3794, 65, 1128, 2835, 2913, 2759, 1590, # 5766
3795,7674,7675, 12,2658, 45, 976,2579,3144,4447, 517,2528,1013,1037,3209,7676, # 5782 3795, 7674, 7675, 12, 2658, 45, 976, 2579, 3144, 4447, 517, 2528, 1013, 1037, 3209, 7676, # 5782
3796,2836,7677,3797,7678,3452,7679,2602, 614,1998,2318,3798,3087,2724,2628,7680, # 5798 3796, 2836, 7677, 3797, 7678, 3452, 7679, 2602, 614, 1998, 2318, 3798, 3087, 2724, 2628, 7680, # 5798
2580,4172, 599,1269,7681,1810,3669,7682,2691,3088, 759,1060, 489,1805,3351,3285, # 5814 2580, 4172, 599, 1269, 7681, 1810, 3669, 7682, 2691, 3088, 759, 1060, 489, 1805, 3351, 3285, # 5814
1358,7683,7684,2386,1387,1215,2629,2252, 490,7685,7686,4173,1759,2387,2343,7687, # 5830 1358, 7683, 7684, 2386, 1387, 1215, 2629, 2252, 490, 7685, 7686, 4173, 1759, 2387, 2343, 7687, # 5830
4448,3799,1907,3971,2630,1806,3210,4449,3453,3286,2760,2344, 874,7688,7689,3454, # 5846 4448, 3799, 1907, 3971, 2630, 1806, 3210, 4449, 3453, 3286, 2760, 2344, 874, 7688, 7689, 3454, # 5846
3670,1858, 91,2914,3671,3042,3800,4450,7690,3145,3972,2659,7691,3455,1202,1403, # 5862 3670, 1858, 91, 2914, 3671, 3042, 3800, 4450, 7690, 3145, 3972, 2659, 7691, 3455, 1202, 1403, # 5862
3801,2954,2529,1517,2503,4451,3456,2504,7692,4452,7693,2692,1885,1495,1731,3973, # 5878 3801, 2954, 2529, 1517, 2503, 4451, 3456, 2504, 7692, 4452, 7693, 2692, 1885, 1495, 1731, 3973, # 5878
2365,4453,7694,2029,7695,7696,3974,2693,1216, 237,2581,4174,2319,3975,3802,4454, # 5894 2365, 4453, 7694, 2029, 7695, 7696, 3974, 2693, 1216, 237, 2581, 4174, 2319, 3975, 3802, 4454, # 5894
4455,2694,3560,3457, 445,4456,7697,7698,7699,7700,2761, 61,3976,3672,1822,3977, # 5910 4455, 2694, 3560, 3457, 445, 4456, 7697, 7698, 7699, 7700, 2761, 61, 3976, 3672, 1822, 3977, # 5910
7701, 687,2045, 935, 925, 405,2660, 703,1096,1859,2725,4457,3978,1876,1367,2695, # 5926 7701, 687, 2045, 935, 925, 405, 2660, 703, 1096, 1859, 2725, 4457, 3978, 1876, 1367, 2695, # 5926
3352, 918,2105,1781,2476, 334,3287,1611,1093,4458, 564,3146,3458,3673,3353, 945, # 5942 3352, 918, 2105, 1781, 2476, 334, 3287, 1611, 1093, 4458, 564, 3146, 3458, 3673, 3353, 945, # 5942
2631,2057,4459,7702,1925, 872,4175,7703,3459,2696,3089, 349,4176,3674,3979,4460, # 5958 2631, 2057, 4459, 7702, 1925, 872, 4175, 7703, 3459, 2696, 3089, 349, 4176, 3674, 3979, 4460, # 5958
3803,4177,3675,2155,3980,4461,4462,4178,4463,2403,2046, 782,3981, 400, 251,4179, # 5974 3803, 4177, 3675, 2155, 3980, 4461, 4462, 4178, 4463, 2403, 2046, 782, 3981, 400, 251, 4179, # 5974
1624,7704,7705, 277,3676, 299,1265, 476,1191,3804,2121,4180,4181,1109, 205,7706, # 5990 1624, 7704, 7705, 277, 3676, 299, 1265, 476, 1191, 3804, 2121, 4180, 4181, 1109, 205, 7706, # 5990
2582,1000,2156,3561,1860,7707,7708,7709,4464,7710,4465,2565, 107,2477,2157,3982, # 6006 2582, 1000, 2156, 3561, 1860, 7707, 7708, 7709, 4464, 7710, 4465, 2565, 107, 2477, 2157, 3982, # 6006
3460,3147,7711,1533, 541,1301, 158, 753,4182,2872,3562,7712,1696, 370,1088,4183, # 6022 3460, 3147, 7711, 1533, 541, 1301, 158, 753, 4182, 2872, 3562, 7712, 1696, 370, 1088, 4183, # 6022
4466,3563, 579, 327, 440, 162,2240, 269,1937,1374,3461, 968,3043, 56,1396,3090, # 6038 4466, 3563, 579, 327, 440, 162, 2240, 269, 1937, 1374, 3461, 968, 3043, 56, 1396, 3090, # 6038
2106,3288,3354,7713,1926,2158,4467,2998,7714,3564,7715,7716,3677,4468,2478,7717, # 6054 2106, 3288, 3354, 7713, 1926, 2158, 4467, 2998, 7714, 3564, 7715, 7716, 3677, 4468, 2478, 7717, # 6054
2791,7718,1650,4469,7719,2603,7720,7721,3983,2661,3355,1149,3356,3984,3805,3985, # 6070 2791, 7718, 1650, 4469, 7719, 2603, 7720, 7721, 3983, 2661, 3355, 1149, 3356, 3984, 3805, 3985, # 6070
7722,1076, 49,7723, 951,3211,3289,3290, 450,2837, 920,7724,1811,2792,2366,4184, # 6086 7722, 1076, 49, 7723, 951, 3211, 3289, 3290, 450, 2837, 920, 7724, 1811, 2792, 2366, 4184, # 6086
1908,1138,2367,3806,3462,7725,3212,4470,1909,1147,1518,2423,4471,3807,7726,4472, # 6102 1908, 1138, 2367, 3806, 3462, 7725, 3212, 4470, 1909, 1147, 1518, 2423, 4471, 3807, 7726, 4472, # 6102
2388,2604, 260,1795,3213,7727,7728,3808,3291, 708,7729,3565,1704,7730,3566,1351, # 6118 2388, 2604, 260, 1795, 3213, 7727, 7728, 3808, 3291, 708, 7729, 3565, 1704, 7730, 3566, 1351, # 6118
1618,3357,2999,1886, 944,4185,3358,4186,3044,3359,4187,7731,3678, 422, 413,1714, # 6134 1618, 3357, 2999, 1886, 944, 4185, 3358, 4186, 3044, 3359, 4187, 7731, 3678, 422, 413, 1714, # 6134
3292, 500,2058,2345,4188,2479,7732,1344,1910, 954,7733,1668,7734,7735,3986,2404, # 6150 3292, 500, 2058, 2345, 4188, 2479, 7732, 1344, 1910, 954, 7733, 1668, 7734, 7735, 3986, 2404, # 6150
4189,3567,3809,4190,7736,2302,1318,2505,3091, 133,3092,2873,4473, 629, 31,2838, # 6166 4189, 3567, 3809, 4190, 7736, 2302, 1318, 2505, 3091, 133, 3092, 2873, 4473, 629, 31, 2838, # 6166
2697,3810,4474, 850, 949,4475,3987,2955,1732,2088,4191,1496,1852,7737,3988, 620, # 6182 2697, 3810, 4474, 850, 949, 4475, 3987, 2955, 1732, 2088, 4191, 1496, 1852, 7737, 3988, 620, # 6182
3214, 981,1242,3679,3360,1619,3680,1643,3293,2139,2452,1970,1719,3463,2168,7738, # 6198 3214, 981, 1242, 3679, 3360, 1619, 3680, 1643, 3293, 2139, 2452, 1970, 1719, 3463, 2168, 7738, # 6198
3215,7739,7740,3361,1828,7741,1277,4476,1565,2047,7742,1636,3568,3093,7743, 869, # 6214 3215, 7739, 7740, 3361, 1828, 7741, 1277, 4476, 1565, 2047, 7742, 1636, 3568, 3093, 7743, 869, # 6214
2839, 655,3811,3812,3094,3989,3000,3813,1310,3569,4477,7744,7745,7746,1733, 558, # 6230 2839, 655, 3811, 3812, 3094, 3989, 3000, 3813, 1310, 3569, 4477, 7744, 7745, 7746, 1733, 558, # 6230
4478,3681, 335,1549,3045,1756,4192,3682,1945,3464,1829,1291,1192, 470,2726,2107, # 6246 4478, 3681, 335, 1549, 3045, 1756, 4192, 3682, 1945, 3464, 1829, 1291, 1192, 470, 2726, 2107, # 6246
2793, 913,1054,3990,7747,1027,7748,3046,3991,4479, 982,2662,3362,3148,3465,3216, # 6262 2793, 913, 1054, 3990, 7747, 1027, 7748, 3046, 3991, 4479, 982, 2662, 3362, 3148, 3465, 3216, # 6262
3217,1946,2794,7749, 571,4480,7750,1830,7751,3570,2583,1523,2424,7752,2089, 984, # 6278 3217, 1946, 2794, 7749, 571, 4480, 7750, 1830, 7751, 3570, 2583, 1523, 2424, 7752, 2089, 984, # 6278
4481,3683,1959,7753,3684, 852, 923,2795,3466,3685, 969,1519, 999,2048,2320,1705, # 6294 4481, 3683, 1959, 7753, 3684, 852, 923, 2795, 3466, 3685, 969, 1519, 999, 2048, 2320, 1705, # 6294
7754,3095, 615,1662, 151, 597,3992,2405,2321,1049, 275,4482,3686,4193, 568,3687, # 6310 7754, 3095, 615, 1662, 151, 597, 3992, 2405, 2321, 1049, 275, 4482, 3686, 4193, 568, 3687, # 6310
3571,2480,4194,3688,7755,2425,2270, 409,3218,7756,1566,2874,3467,1002, 769,2840, # 6326 3571, 2480, 4194, 3688, 7755, 2425, 2270, 409, 3218, 7756, 1566, 2874, 3467, 1002, 769, 2840, # 6326
194,2090,3149,3689,2222,3294,4195, 628,1505,7757,7758,1763,2177,3001,3993, 521, # 6342 194, 2090, 3149, 3689, 2222, 3294, 4195, 628, 1505, 7757, 7758, 1763, 2177, 3001, 3993, 521, # 6342
1161,2584,1787,2203,2406,4483,3994,1625,4196,4197, 412, 42,3096, 464,7759,2632, # 6358 1161, 2584, 1787, 2203, 2406, 4483, 3994, 1625, 4196, 4197, 412, 42, 3096, 464, 7759, 2632, # 6358
4484,3363,1760,1571,2875,3468,2530,1219,2204,3814,2633,2140,2368,4485,4486,3295, # 6374 4484, 3363, 1760, 1571, 2875, 3468, 2530, 1219, 2204, 3814, 2633, 2140, 2368, 4485, 4486, 3295, # 6374
1651,3364,3572,7760,7761,3573,2481,3469,7762,3690,7763,7764,2271,2091, 460,7765, # 6390 1651, 3364, 3572, 7760, 7761, 3573, 2481, 3469, 7762, 3690, 7763, 7764, 2271, 2091, 460, 7765, # 6390
4487,7766,3002, 962, 588,3574, 289,3219,2634,1116, 52,7767,3047,1796,7768,7769, # 6406 4487, 7766, 3002, 962, 588, 3574, 289, 3219, 2634, 1116, 52, 7767, 3047, 1796, 7768, 7769, # 6406
7770,1467,7771,1598,1143,3691,4198,1984,1734,1067,4488,1280,3365, 465,4489,1572, # 6422 7770, 1467, 7771, 1598, 1143, 3691, 4198, 1984, 1734, 1067, 4488, 1280, 3365, 465, 4489, 1572, # 6422
510,7772,1927,2241,1812,1644,3575,7773,4490,3692,7774,7775,2663,1573,1534,7776, # 6438 510, 7772, 1927, 2241, 1812, 1644, 3575, 7773, 4490, 3692, 7774, 7775, 2663, 1573, 1534, 7776, # 6438
7777,4199, 536,1807,1761,3470,3815,3150,2635,7778,7779,7780,4491,3471,2915,1911, # 6454 7777, 4199, 536, 1807, 1761, 3470, 3815, 3150, 2635, 7778, 7779, 7780, 4491, 3471, 2915, 1911, # 6454
2796,7781,3296,1122, 377,3220,7782, 360,7783,7784,4200,1529, 551,7785,2059,3693, # 6470 2796, 7781, 3296, 1122, 377, 3220, 7782, 360, 7783, 7784, 4200, 1529, 551, 7785, 2059, 3693, # 6470
1769,2426,7786,2916,4201,3297,3097,2322,2108,2030,4492,1404, 136,1468,1479, 672, # 6486 1769, 2426, 7786, 2916, 4201, 3297, 3097, 2322, 2108, 2030, 4492, 1404, 136, 1468, 1479, 672, # 6486
1171,3221,2303, 271,3151,7787,2762,7788,2049, 678,2727, 865,1947,4493,7789,2013, # 6502 1171, 3221, 2303, 271, 3151, 7787, 2762, 7788, 2049, 678, 2727, 865, 1947, 4493, 7789, 2013, # 6502
3995,2956,7790,2728,2223,1397,3048,3694,4494,4495,1735,2917,3366,3576,7791,3816, # 6518 3995, 2956, 7790, 2728, 2223, 1397, 3048, 3694, 4494, 4495, 1735, 2917, 3366, 3576, 7791, 3816, # 6518
509,2841,2453,2876,3817,7792,7793,3152,3153,4496,4202,2531,4497,2304,1166,1010, # 6534 509, 2841, 2453, 2876, 3817, 7792, 7793, 3152, 3153, 4496, 4202, 2531, 4497, 2304, 1166, 1010, # 6534
552, 681,1887,7794,7795,2957,2958,3996,1287,1596,1861,3154, 358, 453, 736, 175, # 6550 552, 681, 1887, 7794, 7795, 2957, 2958, 3996, 1287, 1596, 1861, 3154, 358, 453, 736, 175, # 6550
478,1117, 905,1167,1097,7796,1853,1530,7797,1706,7798,2178,3472,2287,3695,3473, # 6566 478, 1117, 905, 1167, 1097, 7796, 1853, 1530, 7797, 1706, 7798, 2178, 3472, 2287, 3695, 3473, # 6566
3577,4203,2092,4204,7799,3367,1193,2482,4205,1458,2190,2205,1862,1888,1421,3298, # 6582 3577, 4203, 2092, 4204, 7799, 3367, 1193, 2482, 4205, 1458, 2190, 2205, 1862, 1888, 1421, 3298, # 6582
2918,3049,2179,3474, 595,2122,7800,3997,7801,7802,4206,1707,2636, 223,3696,1359, # 6598 2918, 3049, 2179, 3474, 595, 2122, 7800, 3997, 7801, 7802, 4206, 1707, 2636, 223, 3696, 1359, # 6598
751,3098, 183,3475,7803,2797,3003, 419,2369, 633, 704,3818,2389, 241,7804,7805, # 6614 751, 3098, 183, 3475, 7803, 2797, 3003, 419, 2369, 633, 704, 3818, 2389, 241, 7804, 7805, # 6614
7806, 838,3004,3697,2272,2763,2454,3819,1938,2050,3998,1309,3099,2242,1181,7807, # 6630 7806, 838, 3004, 3697, 2272, 2763, 2454, 3819, 1938, 2050, 3998, 1309, 3099, 2242, 1181, 7807, # 6630
1136,2206,3820,2370,1446,4207,2305,4498,7808,7809,4208,1055,2605, 484,3698,7810, # 6646 1136, 2206, 3820, 2370, 1446, 4207, 2305, 4498, 7808, 7809, 4208, 1055, 2605, 484, 3698, 7810, # 6646
3999, 625,4209,2273,3368,1499,4210,4000,7811,4001,4211,3222,2274,2275,3476,7812, # 6662 3999, 625, 4209, 2273, 3368, 1499, 4210, 4000, 7811, 4001, 4211, 3222, 2274, 2275, 3476, 7812, # 6662
7813,2764, 808,2606,3699,3369,4002,4212,3100,2532, 526,3370,3821,4213, 955,7814, # 6678 7813, 2764, 808, 2606, 3699, 3369, 4002, 4212, 3100, 2532, 526, 3370, 3821, 4213, 955, 7814, # 6678
1620,4214,2637,2427,7815,1429,3700,1669,1831, 994, 928,7816,3578,1260,7817,7818, # 6694 1620, 4214, 2637, 2427, 7815, 1429, 3700, 1669, 1831, 994, 928, 7816, 3578, 1260, 7817, 7818, # 6694
7819,1948,2288, 741,2919,1626,4215,2729,2455, 867,1184, 362,3371,1392,7820,7821, # 6710 7819, 1948, 2288, 741, 2919, 1626, 4215, 2729, 2455, 867, 1184, 362, 3371, 1392, 7820, 7821, # 6710
4003,4216,1770,1736,3223,2920,4499,4500,1928,2698,1459,1158,7822,3050,3372,2877, # 6726 4003, 4216, 1770, 1736, 3223, 2920, 4499, 4500, 1928, 2698, 1459, 1158, 7822, 3050, 3372, 2877, # 6726
1292,1929,2506,2842,3701,1985,1187,2071,2014,2607,4217,7823,2566,2507,2169,3702, # 6742 1292, 1929, 2506, 2842, 3701, 1985, 1187, 2071, 2014, 2607, 4217, 7823, 2566, 2507, 2169, 3702, # 6742
2483,3299,7824,3703,4501,7825,7826, 666,1003,3005,1022,3579,4218,7827,4502,1813, # 6758 2483, 3299, 7824, 3703, 4501, 7825, 7826, 666, 1003, 3005, 1022, 3579, 4218, 7827, 4502, 1813, # 6758
2253, 574,3822,1603, 295,1535, 705,3823,4219, 283, 858, 417,7828,7829,3224,4503, # 6774 2253, 574, 3822, 1603, 295, 1535, 705, 3823, 4219, 283, 858, 417, 7828, 7829, 3224, 4503, # 6774
4504,3051,1220,1889,1046,2276,2456,4004,1393,1599, 689,2567, 388,4220,7830,2484, # 6790 4504, 3051, 1220, 1889, 1046, 2276, 2456, 4004, 1393, 1599, 689, 2567, 388, 4220, 7830, 2484, # 6790
802,7831,2798,3824,2060,1405,2254,7832,4505,3825,2109,1052,1345,3225,1585,7833, # 6806 802, 7831, 2798, 3824, 2060, 1405, 2254, 7832, 4505, 3825, 2109, 1052, 1345, 3225, 1585, 7833, # 6806
809,7834,7835,7836, 575,2730,3477, 956,1552,1469,1144,2323,7837,2324,1560,2457, # 6822 809, 7834, 7835, 7836, 575, 2730, 3477, 956, 1552, 1469, 1144, 2323, 7837, 2324, 1560, 2457, # 6822
3580,3226,4005, 616,2207,3155,2180,2289,7838,1832,7839,3478,4506,7840,1319,3704, # 6838 3580, 3226, 4005, 616, 2207, 3155, 2180, 2289, 7838, 1832, 7839, 3478, 4506, 7840, 1319, 3704, # 6838
3705,1211,3581,1023,3227,1293,2799,7841,7842,7843,3826, 607,2306,3827, 762,2878, # 6854 3705, 1211, 3581, 1023, 3227, 1293, 2799, 7841, 7842, 7843, 3826, 607, 2306, 3827, 762, 2878, # 6854
1439,4221,1360,7844,1485,3052,7845,4507,1038,4222,1450,2061,2638,4223,1379,4508, # 6870 1439, 4221, 1360, 7844, 1485, 3052, 7845, 4507, 1038, 4222, 1450, 2061, 2638, 4223, 1379, 4508, # 6870
2585,7846,7847,4224,1352,1414,2325,2921,1172,7848,7849,3828,3829,7850,1797,1451, # 6886 2585, 7846, 7847, 4224, 1352, 1414, 2325, 2921, 1172, 7848, 7849, 3828, 3829, 7850, 1797, 1451, # 6886
7851,7852,7853,7854,2922,4006,4007,2485,2346, 411,4008,4009,3582,3300,3101,4509, # 6902 7851, 7852, 7853, 7854, 2922, 4006, 4007, 2485, 2346, 411, 4008, 4009, 3582, 3300, 3101, 4509, # 6902
1561,2664,1452,4010,1375,7855,7856, 47,2959, 316,7857,1406,1591,2923,3156,7858, # 6918 1561, 2664, 1452, 4010, 1375, 7855, 7856, 47, 2959, 316, 7857, 1406, 1591, 2923, 3156, 7858, # 6918
1025,2141,3102,3157, 354,2731, 884,2224,4225,2407, 508,3706, 726,3583, 996,2428, # 6934 1025, 2141, 3102, 3157, 354, 2731, 884, 2224, 4225, 2407, 508, 3706, 726, 3583, 996, 2428, # 6934
3584, 729,7859, 392,2191,1453,4011,4510,3707,7860,7861,2458,3585,2608,1675,2800, # 6950 3584, 729, 7859, 392, 2191, 1453, 4011, 4510, 3707, 7860, 7861, 2458, 3585, 2608, 1675, 2800, # 6950
919,2347,2960,2348,1270,4511,4012, 73,7862,7863, 647,7864,3228,2843,2255,1550, # 6966 919, 2347, 2960, 2348, 1270, 4511, 4012, 73, 7862, 7863, 647, 7864, 3228, 2843, 2255, 1550, # 6966
1346,3006,7865,1332, 883,3479,7866,7867,7868,7869,3301,2765,7870,1212, 831,1347, # 6982 1346, 3006, 7865, 1332, 883, 3479, 7866, 7867, 7868, 7869, 3301, 2765, 7870, 1212, 831, 1347, # 6982
4226,4512,2326,3830,1863,3053, 720,3831,4513,4514,3832,7871,4227,7872,7873,4515, # 6998 4226, 4512, 2326, 3830, 1863, 3053, 720, 3831, 4513, 4514, 3832, 7871, 4227, 7872, 7873, 4515, # 6998
7874,7875,1798,4516,3708,2609,4517,3586,1645,2371,7876,7877,2924, 669,2208,2665, # 7014 7874, 7875, 1798, 4516, 3708, 2609, 4517, 3586, 1645, 2371, 7876, 7877, 2924, 669, 2208, 2665, # 7014
2429,7878,2879,7879,7880,1028,3229,7881,4228,2408,7882,2256,1353,7883,7884,4518, # 7030 2429, 7878, 2879, 7879, 7880, 1028, 3229, 7881, 4228, 2408, 7882, 2256, 1353, 7883, 7884, 4518, # 7030
3158, 518,7885,4013,7886,4229,1960,7887,2142,4230,7888,7889,3007,2349,2350,3833, # 7046 3158, 518, 7885, 4013, 7886, 4229, 1960, 7887, 2142, 4230, 7888, 7889, 3007, 2349, 2350, 3833, # 7046
516,1833,1454,4014,2699,4231,4519,2225,2610,1971,1129,3587,7890,2766,7891,2961, # 7062 516, 1833, 1454, 4014, 2699, 4231, 4519, 2225, 2610, 1971, 1129, 3587, 7890, 2766, 7891, 2961, # 7062
1422, 577,1470,3008,1524,3373,7892,7893, 432,4232,3054,3480,7894,2586,1455,2508, # 7078 1422, 577, 1470, 3008, 1524, 3373, 7892, 7893, 432, 4232, 3054, 3480, 7894, 2586, 1455, 2508, # 7078
2226,1972,1175,7895,1020,2732,4015,3481,4520,7896,2733,7897,1743,1361,3055,3482, # 7094 2226, 1972, 1175, 7895, 1020, 2732, 4015, 3481, 4520, 7896, 2733, 7897, 1743, 1361, 3055, 3482, # 7094
2639,4016,4233,4521,2290, 895, 924,4234,2170, 331,2243,3056, 166,1627,3057,1098, # 7110 2639, 4016, 4233, 4521, 2290, 895, 924, 4234, 2170, 331, 2243, 3056, 166, 1627, 3057, 1098, # 7110
7898,1232,2880,2227,3374,4522, 657, 403,1196,2372, 542,3709,3375,1600,4235,3483, # 7126 7898, 1232, 2880, 2227, 3374, 4522, 657, 403, 1196, 2372, 542, 3709, 3375, 1600, 4235, 3483, # 7126
7899,4523,2767,3230, 576, 530,1362,7900,4524,2533,2666,3710,4017,7901, 842,3834, # 7142 7899, 4523, 2767, 3230, 576, 530, 1362, 7900, 4524, 2533, 2666, 3710, 4017, 7901, 842, 3834, # 7142
7902,2801,2031,1014,4018, 213,2700,3376, 665, 621,4236,7903,3711,2925,2430,7904, # 7158 7902, 2801, 2031, 1014, 4018, 213, 2700, 3376, 665, 621, 4236, 7903, 3711, 2925, 2430, 7904, # 7158
2431,3302,3588,3377,7905,4237,2534,4238,4525,3589,1682,4239,3484,1380,7906, 724, # 7174 2431, 3302, 3588, 3377, 7905, 4237, 2534, 4238, 4525, 3589, 1682, 4239, 3484, 1380, 7906, 724, # 7174
2277, 600,1670,7907,1337,1233,4526,3103,2244,7908,1621,4527,7909, 651,4240,7910, # 7190 2277, 600, 1670, 7907, 1337, 1233, 4526, 3103, 2244, 7908, 1621, 4527, 7909, 651, 4240, 7910, # 7190
1612,4241,2611,7911,2844,7912,2734,2307,3058,7913, 716,2459,3059, 174,1255,2701, # 7206 1612, 4241, 2611, 7911, 2844, 7912, 2734, 2307, 3058, 7913, 716, 2459, 3059, 174, 1255, 2701, # 7206
4019,3590, 548,1320,1398, 728,4020,1574,7914,1890,1197,3060,4021,7915,3061,3062, # 7222 4019, 3590, 548, 1320, 1398, 728, 4020, 1574, 7914, 1890, 1197, 3060, 4021, 7915, 3061, 3062, # 7222
3712,3591,3713, 747,7916, 635,4242,4528,7917,7918,7919,4243,7920,7921,4529,7922, # 7238 3712, 3591, 3713, 747, 7916, 635, 4242, 4528, 7917, 7918, 7919, 4243, 7920, 7921, 4529, 7922, # 7238
3378,4530,2432, 451,7923,3714,2535,2072,4244,2735,4245,4022,7924,1764,4531,7925, # 7254 3378, 4530, 2432, 451, 7923, 3714, 2535, 2072, 4244, 2735, 4245, 4022, 7924, 1764, 4531, 7925, # 7254
4246, 350,7926,2278,2390,2486,7927,4247,4023,2245,1434,4024, 488,4532, 458,4248, # 7270 4246, 350, 7926, 2278, 2390, 2486, 7927, 4247, 4023, 2245, 1434, 4024, 488, 4532, 458, 4248, # 7270
4025,3715, 771,1330,2391,3835,2568,3159,2159,2409,1553,2667,3160,4249,7928,2487, # 7286 4025, 3715, 771, 1330, 2391, 3835, 2568, 3159, 2159, 2409, 1553, 2667, 3160, 4249, 7928, 2487, # 7286
2881,2612,1720,2702,4250,3379,4533,7929,2536,4251,7930,3231,4252,2768,7931,2015, # 7302 2881, 2612, 1720, 2702, 4250, 3379, 4533, 7929, 2536, 4251, 7930, 3231, 4252, 2768, 7931, 2015, # 7302
2736,7932,1155,1017,3716,3836,7933,3303,2308, 201,1864,4253,1430,7934,4026,7935, # 7318 2736, 7932, 1155, 1017, 3716, 3836, 7933, 3303, 2308, 201, 1864, 4253, 1430, 7934, 4026, 7935, # 7318
7936,7937,7938,7939,4254,1604,7940, 414,1865, 371,2587,4534,4535,3485,2016,3104, # 7334 7936, 7937, 7938, 7939, 4254, 1604, 7940, 414, 1865, 371, 2587, 4534, 4535, 3485, 2016, 3104, # 7334
4536,1708, 960,4255, 887, 389,2171,1536,1663,1721,7941,2228,4027,2351,2926,1580, # 7350 4536, 1708, 960, 4255, 887, 389, 2171, 1536, 1663, 1721, 7941, 2228, 4027, 2351, 2926, 1580, # 7350
7942,7943,7944,1744,7945,2537,4537,4538,7946,4539,7947,2073,7948,7949,3592,3380, # 7366 7942, 7943, 7944, 1744, 7945, 2537, 4537, 4538, 7946, 4539, 7947, 2073, 7948, 7949, 3592, 3380, # 7366
2882,4256,7950,4257,2640,3381,2802, 673,2703,2460, 709,3486,4028,3593,4258,7951, # 7382 2882, 4256, 7950, 4257, 2640, 3381, 2802, 673, 2703, 2460, 709, 3486, 4028, 3593, 4258, 7951, # 7382
1148, 502, 634,7952,7953,1204,4540,3594,1575,4541,2613,3717,7954,3718,3105, 948, # 7398 1148, 502, 634, 7952, 7953, 1204, 4540, 3594, 1575, 4541, 2613, 3717, 7954, 3718, 3105, 948, # 7398
3232, 121,1745,3837,1110,7955,4259,3063,2509,3009,4029,3719,1151,1771,3838,1488, # 7414 3232, 121, 1745, 3837, 1110, 7955, 4259, 3063, 2509, 3009, 4029, 3719, 1151, 1771, 3838, 1488, # 7414
4030,1986,7956,2433,3487,7957,7958,2093,7959,4260,3839,1213,1407,2803, 531,2737, # 7430 4030, 1986, 7956, 2433, 3487, 7957, 7958, 2093, 7959, 4260, 3839, 1213, 1407, 2803, 531, 2737, # 7430
2538,3233,1011,1537,7960,2769,4261,3106,1061,7961,3720,3721,1866,2883,7962,2017, # 7446 2538, 3233, 1011, 1537, 7960, 2769, 4261, 3106, 1061, 7961, 3720, 3721, 1866, 2883, 7962, 2017, # 7446
120,4262,4263,2062,3595,3234,2309,3840,2668,3382,1954,4542,7963,7964,3488,1047, # 7462 120, 4262, 4263, 2062, 3595, 3234, 2309, 3840, 2668, 3382, 1954, 4542, 7963, 7964, 3488, 1047, # 7462
2704,1266,7965,1368,4543,2845, 649,3383,3841,2539,2738,1102,2846,2669,7966,7967, # 7478 2704, 1266, 7965, 1368, 4543, 2845, 649, 3383, 3841, 2539, 2738, 1102, 2846, 2669, 7966, 7967, # 7478
1999,7968,1111,3596,2962,7969,2488,3842,3597,2804,1854,3384,3722,7970,7971,3385, # 7494 1999, 7968, 1111, 3596, 2962, 7969, 2488, 3842, 3597, 2804, 1854, 3384, 3722, 7970, 7971, 3385, # 7494
2410,2884,3304,3235,3598,7972,2569,7973,3599,2805,4031,1460, 856,7974,3600,7975, # 7510 2410, 2884, 3304, 3235, 3598, 7972, 2569, 7973, 3599, 2805, 4031, 1460, 856, 7974, 3600, 7975, # 7510
2885,2963,7976,2886,3843,7977,4264, 632,2510, 875,3844,1697,3845,2291,7978,7979, # 7526 2885, 2963, 7976, 2886, 3843, 7977, 4264, 632, 2510, 875, 3844, 1697, 3845, 2291, 7978, 7979, # 7526
4544,3010,1239, 580,4545,4265,7980, 914, 936,2074,1190,4032,1039,2123,7981,7982, # 7542 4544, 3010, 1239, 580, 4545, 4265, 7980, 914, 936, 2074, 1190, 4032, 1039, 2123, 7981, 7982, # 7542
7983,3386,1473,7984,1354,4266,3846,7985,2172,3064,4033, 915,3305,4267,4268,3306, # 7558 7983, 3386, 1473, 7984, 1354, 4266, 3846, 7985, 2172, 3064, 4033, 915, 3305, 4267, 4268, 3306, # 7558
1605,1834,7986,2739, 398,3601,4269,3847,4034, 328,1912,2847,4035,3848,1331,4270, # 7574 1605, 1834, 7986, 2739, 398, 3601, 4269, 3847, 4034, 328, 1912, 2847, 4035, 3848, 1331, 4270, # 7574
3011, 937,4271,7987,3602,4036,4037,3387,2160,4546,3388, 524, 742, 538,3065,1012, # 7590 3011, 937, 4271, 7987, 3602, 4036, 4037, 3387, 2160, 4546, 3388, 524, 742, 538, 3065, 1012, # 7590
7988,7989,3849,2461,7990, 658,1103, 225,3850,7991,7992,4547,7993,4548,7994,3236, # 7606 7988, 7989, 3849, 2461, 7990, 658, 1103, 225, 3850, 7991, 7992, 4547, 7993, 4548, 7994, 3236, # 7606
1243,7995,4038, 963,2246,4549,7996,2705,3603,3161,7997,7998,2588,2327,7999,4550, # 7622 1243, 7995, 4038, 963, 2246, 4549, 7996, 2705, 3603, 3161, 7997, 7998, 2588, 2327, 7999, 4550, # 7622
8000,8001,8002,3489,3307, 957,3389,2540,2032,1930,2927,2462, 870,2018,3604,1746, # 7638 8000, 8001, 8002, 3489, 3307, 957, 3389, 2540, 2032, 1930, 2927, 2462, 870, 2018, 3604, 1746, # 7638
2770,2771,2434,2463,8003,3851,8004,3723,3107,3724,3490,3390,3725,8005,1179,3066, # 7654 2770, 2771, 2434, 2463, 8003, 3851, 8004, 3723, 3107, 3724, 3490, 3390, 3725, 8005, 1179, 3066, # 7654
8006,3162,2373,4272,3726,2541,3163,3108,2740,4039,8007,3391,1556,2542,2292, 977, # 7670 8006, 3162, 2373, 4272, 3726, 2541, 3163, 3108, 2740, 4039, 8007, 3391, 1556, 2542, 2292, 977, # 7670
2887,2033,4040,1205,3392,8008,1765,3393,3164,2124,1271,1689, 714,4551,3491,8009, # 7686 2887, 2033, 4040, 1205, 3392, 8008, 1765, 3393, 3164, 2124, 1271, 1689, 714, 4551, 3491, 8009, # 7686
2328,3852, 533,4273,3605,2181, 617,8010,2464,3308,3492,2310,8011,8012,3165,8013, # 7702 2328, 3852, 533, 4273, 3605, 2181, 617, 8010, 2464, 3308, 3492, 2310, 8011, 8012, 3165, 8013, # 7702
8014,3853,1987, 618, 427,2641,3493,3394,8015,8016,1244,1690,8017,2806,4274,4552, # 7718 8014, 3853, 1987, 618, 427, 2641, 3493, 3394, 8015, 8016, 1244, 1690, 8017, 2806, 4274, 4552, # 7718
8018,3494,8019,8020,2279,1576, 473,3606,4275,3395, 972,8021,3607,8022,3067,8023, # 7734 8018, 3494, 8019, 8020, 2279, 1576, 473, 3606, 4275, 3395, 972, 8021, 3607, 8022, 3067, 8023, # 7734
8024,4553,4554,8025,3727,4041,4042,8026, 153,4555, 356,8027,1891,2888,4276,2143, # 7750 8024, 4553, 4554, 8025, 3727, 4041, 4042, 8026, 153, 4555, 356, 8027, 1891, 2888, 4276, 2143, # 7750
408, 803,2352,8028,3854,8029,4277,1646,2570,2511,4556,4557,3855,8030,3856,4278, # 7766 408, 803, 2352, 8028, 3854, 8029, 4277, 1646, 2570, 2511, 4556, 4557, 3855, 8030, 3856, 4278, # 7766
8031,2411,3396, 752,8032,8033,1961,2964,8034, 746,3012,2465,8035,4279,3728, 698, # 7782 8031, 2411, 3396, 752, 8032, 8033, 1961, 2964, 8034, 746, 3012, 2465, 8035, 4279, 3728, 698, # 7782
4558,1892,4280,3608,2543,4559,3609,3857,8036,3166,3397,8037,1823,1302,4043,2706, # 7798 4558, 1892, 4280, 3608, 2543, 4559, 3609, 3857, 8036, 3166, 3397, 8037, 1823, 1302, 4043, 2706, # 7798
3858,1973,4281,8038,4282,3167, 823,1303,1288,1236,2848,3495,4044,3398, 774,3859, # 7814 3858, 1973, 4281, 8038, 4282, 3167, 823, 1303, 1288, 1236, 2848, 3495, 4044, 3398, 774, 3859, # 7814
8039,1581,4560,1304,2849,3860,4561,8040,2435,2161,1083,3237,4283,4045,4284, 344, # 7830 8039, 1581, 4560, 1304, 2849, 3860, 4561, 8040, 2435, 2161, 1083, 3237, 4283, 4045, 4284, 344, # 7830
1173, 288,2311, 454,1683,8041,8042,1461,4562,4046,2589,8043,8044,4563, 985, 894, # 7846 1173, 288, 2311, 454, 1683, 8041, 8042, 1461, 4562, 4046, 2589, 8043, 8044, 4563, 985, 894, # 7846
8045,3399,3168,8046,1913,2928,3729,1988,8047,2110,1974,8048,4047,8049,2571,1194, # 7862 8045, 3399, 3168, 8046, 1913, 2928, 3729, 1988, 8047, 2110, 1974, 8048, 4047, 8049, 2571, 1194, # 7862
425,8050,4564,3169,1245,3730,4285,8051,8052,2850,8053, 636,4565,1855,3861, 760, # 7878 425, 8050, 4564, 3169, 1245, 3730, 4285, 8051, 8052, 2850, 8053, 636, 4565, 1855, 3861, 760, # 7878
1799,8054,4286,2209,1508,4566,4048,1893,1684,2293,8055,8056,8057,4287,4288,2210, # 7894 1799, 8054, 4286, 2209, 1508, 4566, 4048, 1893, 1684, 2293, 8055, 8056, 8057, 4287, 4288, 2210, # 7894
479,8058,8059, 832,8060,4049,2489,8061,2965,2490,3731, 990,3109, 627,1814,2642, # 7910 479, 8058, 8059, 832, 8060, 4049, 2489, 8061, 2965, 2490, 3731, 990, 3109, 627, 1814, 2642, # 7910
4289,1582,4290,2125,2111,3496,4567,8062, 799,4291,3170,8063,4568,2112,1737,3013, # 7926 4289, 1582, 4290, 2125, 2111, 3496, 4567, 8062, 799, 4291, 3170, 8063, 4568, 2112, 1737, 3013, # 7926
1018, 543, 754,4292,3309,1676,4569,4570,4050,8064,1489,8065,3497,8066,2614,2889, # 7942 1018, 543, 754, 4292, 3309, 1676, 4569, 4570, 4050, 8064, 1489, 8065, 3497, 8066, 2614, 2889, # 7942
4051,8067,8068,2966,8069,8070,8071,8072,3171,4571,4572,2182,1722,8073,3238,3239, # 7958 4051, 8067, 8068, 2966, 8069, 8070, 8071, 8072, 3171, 4571, 4572, 2182, 1722, 8073, 3238, 3239, # 7958
1842,3610,1715, 481, 365,1975,1856,8074,8075,1962,2491,4573,8076,2126,3611,3240, # 7974 1842, 3610, 1715, 481, 365, 1975, 1856, 8074, 8075, 1962, 2491, 4573, 8076, 2126, 3611, 3240, # 7974
433,1894,2063,2075,8077, 602,2741,8078,8079,8080,8081,8082,3014,1628,3400,8083, # 7990 433, 1894, 2063, 2075, 8077, 602, 2741, 8078, 8079, 8080, 8081, 8082, 3014, 1628, 3400, 8083, # 7990
3172,4574,4052,2890,4575,2512,8084,2544,2772,8085,8086,8087,3310,4576,2891,8088, # 8006 3172, 4574, 4052, 2890, 4575, 2512, 8084, 2544, 2772, 8085, 8086, 8087, 3310, 4576, 2891, 8088, # 8006
4577,8089,2851,4578,4579,1221,2967,4053,2513,8090,8091,8092,1867,1989,8093,8094, # 8022 4577, 8089, 2851, 4578, 4579, 1221, 2967, 4053, 2513, 8090, 8091, 8092, 1867, 1989, 8093, 8094, # 8022
8095,1895,8096,8097,4580,1896,4054, 318,8098,2094,4055,4293,8099,8100, 485,8101, # 8038 8095, 1895, 8096, 8097, 4580, 1896, 4054, 318, 8098, 2094, 4055, 4293, 8099, 8100, 485, 8101, # 8038
938,3862, 553,2670, 116,8102,3863,3612,8103,3498,2671,2773,3401,3311,2807,8104, # 8054 938, 3862, 553, 2670, 116, 8102, 3863, 3612, 8103, 3498, 2671, 2773, 3401, 3311, 2807, 8104, # 8054
3613,2929,4056,1747,2930,2968,8105,8106, 207,8107,8108,2672,4581,2514,8109,3015, # 8070 3613, 2929, 4056, 1747, 2930, 2968, 8105, 8106, 207, 8107, 8108, 2672, 4581, 2514, 8109, 3015, # 8070
890,3614,3864,8110,1877,3732,3402,8111,2183,2353,3403,1652,8112,8113,8114, 941, # 8086 890, 3614, 3864, 8110, 1877, 3732, 3402, 8111, 2183, 2353, 3403, 1652, 8112, 8113, 8114, 941, # 8086
2294, 208,3499,4057,2019, 330,4294,3865,2892,2492,3733,4295,8115,8116,8117,8118, # 8102 2294, 208, 3499, 4057, 2019, 330, 4294, 3865, 2892, 2492, 3733, 4295, 8115, 8116, 8117, 8118, # 8102
) )
# fmt: on
+7 -6
View File
@@ -25,22 +25,23 @@
# 02110-1301 USA # 02110-1301 USA
######################### END LICENSE BLOCK ######################### ######################### END LICENSE BLOCK #########################
from .mbcharsetprober import MultiByteCharSetProber
from .codingstatemachine import CodingStateMachine
from .chardistribution import EUCTWDistributionAnalysis from .chardistribution import EUCTWDistributionAnalysis
from .codingstatemachine import CodingStateMachine
from .mbcharsetprober import MultiByteCharSetProber
from .mbcssm import EUCTW_SM_MODEL from .mbcssm import EUCTW_SM_MODEL
class EUCTWProber(MultiByteCharSetProber): class EUCTWProber(MultiByteCharSetProber):
def __init__(self): def __init__(self) -> None:
super(EUCTWProber, self).__init__() super().__init__()
self.coding_sm = CodingStateMachine(EUCTW_SM_MODEL) self.coding_sm = CodingStateMachine(EUCTW_SM_MODEL)
self.distribution_analyzer = EUCTWDistributionAnalysis() self.distribution_analyzer = EUCTWDistributionAnalysis()
self.reset() self.reset()
@property @property
def charset_name(self): def charset_name(self) -> str:
return "EUC-TW" return "EUC-TW"
@property @property
def language(self): def language(self) -> str:
return "Taiwan" return "Taiwan"
+2 -1
View File
@@ -43,6 +43,7 @@ GB2312_TYPICAL_DISTRIBUTION_RATIO = 0.9
GB2312_TABLE_SIZE = 3760 GB2312_TABLE_SIZE = 3760
# fmt: off
GB2312_CHAR_TO_FREQ_ORDER = ( GB2312_CHAR_TO_FREQ_ORDER = (
1671, 749,1443,2364,3924,3807,2330,3921,1704,3463,2691,1511,1515, 572,3191,2205, 1671, 749,1443,2364,3924,3807,2330,3921,1704,3463,2691,1511,1515, 572,3191,2205,
2361, 224,2558, 479,1711, 963,3162, 440,4060,1905,2966,2947,3580,2647,3961,3842, 2361, 224,2558, 479,1711, 963,3162, 440,4060,1905,2966,2947,3580,2647,3961,3842,
@@ -280,4 +281,4 @@ GB2312_CHAR_TO_FREQ_ORDER = (
381,1638,4592,1020, 516,3214, 458, 947,4575,1432, 211,1514,2926,1865,2142, 189, 381,1638,4592,1020, 516,3214, 458, 947,4575,1432, 211,1514,2926,1865,2142, 189,
852,1221,1400,1486, 882,2299,4036, 351, 28,1122, 700,6479,6480,6481,6482,6483, #last 512 852,1221,1400,1486, 882,2299,4036, 351, 28,1122, 700,6479,6480,6481,6482,6483, #last 512
) )
# fmt: on
+7 -6
View File
@@ -25,22 +25,23 @@
# 02110-1301 USA # 02110-1301 USA
######################### END LICENSE BLOCK ######################### ######################### END LICENSE BLOCK #########################
from .mbcharsetprober import MultiByteCharSetProber
from .codingstatemachine import CodingStateMachine
from .chardistribution import GB2312DistributionAnalysis from .chardistribution import GB2312DistributionAnalysis
from .codingstatemachine import CodingStateMachine
from .mbcharsetprober import MultiByteCharSetProber
from .mbcssm import GB2312_SM_MODEL from .mbcssm import GB2312_SM_MODEL
class GB2312Prober(MultiByteCharSetProber): class GB2312Prober(MultiByteCharSetProber):
def __init__(self): def __init__(self) -> None:
super(GB2312Prober, self).__init__() super().__init__()
self.coding_sm = CodingStateMachine(GB2312_SM_MODEL) self.coding_sm = CodingStateMachine(GB2312_SM_MODEL)
self.distribution_analyzer = GB2312DistributionAnalysis() self.distribution_analyzer = GB2312DistributionAnalysis()
self.reset() self.reset()
@property @property
def charset_name(self): def charset_name(self) -> str:
return "GB2312" return "GB2312"
@property @property
def language(self): def language(self) -> str:
return "Chinese" return "Chinese"
+67 -43
View File
@@ -25,8 +25,11 @@
# 02110-1301 USA # 02110-1301 USA
######################### END LICENSE BLOCK ######################### ######################### END LICENSE BLOCK #########################
from typing import Optional, Union
from .charsetprober import CharSetProber from .charsetprober import CharSetProber
from .enums import ProbingState from .enums import ProbingState
from .sbcharsetprober import SingleByteCharSetProber
# This prober doesn't actually recognize a language or a charset. # This prober doesn't actually recognize a language or a charset.
# It is a helper prober for the use of the Hebrew model probers # It is a helper prober for the use of the Hebrew model probers
@@ -125,18 +128,20 @@ from .enums import ProbingState
# model probers scores. The answer is returned in the form of the name of the # model probers scores. The answer is returned in the form of the name of the
# charset identified, either "windows-1255" or "ISO-8859-8". # charset identified, either "windows-1255" or "ISO-8859-8".
class HebrewProber(CharSetProber): class HebrewProber(CharSetProber):
SPACE = 0x20
# windows-1255 / ISO-8859-8 code points of interest # windows-1255 / ISO-8859-8 code points of interest
FINAL_KAF = 0xea FINAL_KAF = 0xEA
NORMAL_KAF = 0xeb NORMAL_KAF = 0xEB
FINAL_MEM = 0xed FINAL_MEM = 0xED
NORMAL_MEM = 0xee NORMAL_MEM = 0xEE
FINAL_NUN = 0xef FINAL_NUN = 0xEF
NORMAL_NUN = 0xf0 NORMAL_NUN = 0xF0
FINAL_PE = 0xf3 FINAL_PE = 0xF3
NORMAL_PE = 0xf4 NORMAL_PE = 0xF4
FINAL_TSADI = 0xf5 FINAL_TSADI = 0xF5
NORMAL_TSADI = 0xf6 NORMAL_TSADI = 0xF6
# Minimum Visual vs Logical final letter score difference. # Minimum Visual vs Logical final letter score difference.
# If the difference is below this, don't rely solely on the final letter score # If the difference is below this, don't rely solely on the final letter score
@@ -151,35 +156,44 @@ class HebrewProber(CharSetProber):
VISUAL_HEBREW_NAME = "ISO-8859-8" VISUAL_HEBREW_NAME = "ISO-8859-8"
LOGICAL_HEBREW_NAME = "windows-1255" LOGICAL_HEBREW_NAME = "windows-1255"
def __init__(self): def __init__(self) -> None:
super(HebrewProber, self).__init__() super().__init__()
self._final_char_logical_score = None self._final_char_logical_score = 0
self._final_char_visual_score = None self._final_char_visual_score = 0
self._prev = None self._prev = self.SPACE
self._before_prev = None self._before_prev = self.SPACE
self._logical_prober = None self._logical_prober: Optional[SingleByteCharSetProber] = None
self._visual_prober = None self._visual_prober: Optional[SingleByteCharSetProber] = None
self.reset() self.reset()
def reset(self): def reset(self) -> None:
self._final_char_logical_score = 0 self._final_char_logical_score = 0
self._final_char_visual_score = 0 self._final_char_visual_score = 0
# The two last characters seen in the previous buffer, # The two last characters seen in the previous buffer,
# mPrev and mBeforePrev are initialized to space in order to simulate # mPrev and mBeforePrev are initialized to space in order to simulate
# a word delimiter at the beginning of the data # a word delimiter at the beginning of the data
self._prev = ' ' self._prev = self.SPACE
self._before_prev = ' ' self._before_prev = self.SPACE
# These probers are owned by the group prober. # These probers are owned by the group prober.
def set_model_probers(self, logicalProber, visualProber): def set_model_probers(
self._logical_prober = logicalProber self,
self._visual_prober = visualProber logical_prober: SingleByteCharSetProber,
visual_prober: SingleByteCharSetProber,
) -> None:
self._logical_prober = logical_prober
self._visual_prober = visual_prober
def is_final(self, c): def is_final(self, c: int) -> bool:
return c in [self.FINAL_KAF, self.FINAL_MEM, self.FINAL_NUN, return c in [
self.FINAL_PE, self.FINAL_TSADI] self.FINAL_KAF,
self.FINAL_MEM,
self.FINAL_NUN,
self.FINAL_PE,
self.FINAL_TSADI,
]
def is_non_final(self, c): def is_non_final(self, c: int) -> bool:
# The normal Tsadi is not a good Non-Final letter due to words like # The normal Tsadi is not a good Non-Final letter due to words like
# 'lechotet' (to chat) containing an apostrophe after the tsadi. This # 'lechotet' (to chat) containing an apostrophe after the tsadi. This
# apostrophe is converted to a space in FilterWithoutEnglishLetters # apostrophe is converted to a space in FilterWithoutEnglishLetters
@@ -190,10 +204,9 @@ class HebrewProber(CharSetProber):
# for example legally end with a Non-Final Pe or Kaf. However, the # for example legally end with a Non-Final Pe or Kaf. However, the
# benefit of these letters as Non-Final letters outweighs the damage # benefit of these letters as Non-Final letters outweighs the damage
# since these words are quite rare. # since these words are quite rare.
return c in [self.NORMAL_KAF, self.NORMAL_MEM, return c in [self.NORMAL_KAF, self.NORMAL_MEM, self.NORMAL_NUN, self.NORMAL_PE]
self.NORMAL_NUN, self.NORMAL_PE]
def feed(self, byte_str): def feed(self, byte_str: Union[bytes, bytearray]) -> ProbingState:
# Final letter analysis for logical-visual decision. # Final letter analysis for logical-visual decision.
# Look for evidence that the received buffer is either logical Hebrew # Look for evidence that the received buffer is either logical Hebrew
# or visual Hebrew. # or visual Hebrew.
@@ -227,9 +240,9 @@ class HebrewProber(CharSetProber):
byte_str = self.filter_high_byte_only(byte_str) byte_str = self.filter_high_byte_only(byte_str)
for cur in byte_str: for cur in byte_str:
if cur == ' ': if cur == self.SPACE:
# We stand on a space - a word just ended # We stand on a space - a word just ended
if self._before_prev != ' ': if self._before_prev != self.SPACE:
# next-to-last char was not a space so self._prev is not a # next-to-last char was not a space so self._prev is not a
# 1 letter word # 1 letter word
if self.is_final(self._prev): if self.is_final(self._prev):
@@ -241,8 +254,11 @@ class HebrewProber(CharSetProber):
self._final_char_visual_score += 1 self._final_char_visual_score += 1
else: else:
# Not standing on a space # Not standing on a space
if ((self._before_prev == ' ') and if (
(self.is_final(self._prev)) and (cur != ' ')): (self._before_prev == self.SPACE)
and (self.is_final(self._prev))
and (cur != self.SPACE)
):
# case (3) [-2:space][-1:final letter][cur:not space] # case (3) [-2:space][-1:final letter][cur:not space]
self._final_char_visual_score += 1 self._final_char_visual_score += 1
self._before_prev = self._prev self._before_prev = self._prev
@@ -253,7 +269,10 @@ class HebrewProber(CharSetProber):
return ProbingState.DETECTING return ProbingState.DETECTING
@property @property
def charset_name(self): def charset_name(self) -> str:
assert self._logical_prober is not None
assert self._visual_prober is not None
# Make the decision: is it Logical or Visual? # Make the decision: is it Logical or Visual?
# If the final letter score distance is dominant enough, rely on it. # If the final letter score distance is dominant enough, rely on it.
finalsub = self._final_char_logical_score - self._final_char_visual_score finalsub = self._final_char_logical_score - self._final_char_visual_score
@@ -263,8 +282,9 @@ class HebrewProber(CharSetProber):
return self.VISUAL_HEBREW_NAME return self.VISUAL_HEBREW_NAME
# It's not dominant enough, try to rely on the model scores instead. # It's not dominant enough, try to rely on the model scores instead.
modelsub = (self._logical_prober.get_confidence() modelsub = (
- self._visual_prober.get_confidence()) self._logical_prober.get_confidence() - self._visual_prober.get_confidence()
)
if modelsub > self.MIN_MODEL_DISTANCE: if modelsub > self.MIN_MODEL_DISTANCE:
return self.LOGICAL_HEBREW_NAME return self.LOGICAL_HEBREW_NAME
if modelsub < -self.MIN_MODEL_DISTANCE: if modelsub < -self.MIN_MODEL_DISTANCE:
@@ -280,13 +300,17 @@ class HebrewProber(CharSetProber):
return self.LOGICAL_HEBREW_NAME return self.LOGICAL_HEBREW_NAME
@property @property
def language(self): def language(self) -> str:
return 'Hebrew' return "Hebrew"
@property @property
def state(self): def state(self) -> ProbingState:
assert self._logical_prober is not None
assert self._visual_prober is not None
# Remain active as long as any of the model probers are active. # Remain active as long as any of the model probers are active.
if (self._logical_prober.state == ProbingState.NOT_ME) and \ if (self._logical_prober.state == ProbingState.NOT_ME) and (
(self._visual_prober.state == ProbingState.NOT_ME): self._visual_prober.state == ProbingState.NOT_ME
):
return ProbingState.NOT_ME return ProbingState.NOT_ME
return ProbingState.DETECTING return ProbingState.DETECTING
+2 -2
View File
@@ -46,6 +46,7 @@ JIS_TYPICAL_DISTRIBUTION_RATIO = 3.0
# Char to FreqOrder table , # Char to FreqOrder table ,
JIS_TABLE_SIZE = 4368 JIS_TABLE_SIZE = 4368
# fmt: off
JIS_CHAR_TO_FREQ_ORDER = ( JIS_CHAR_TO_FREQ_ORDER = (
40, 1, 6, 182, 152, 180, 295,2127, 285, 381,3295,4304,3068,4606,3165,3510, # 16 40, 1, 6, 182, 152, 180, 295,2127, 285, 381,3295,4304,3068,4606,3165,3510, # 16
3511,1822,2785,4607,1193,2226,5070,4608, 171,2996,1247, 18, 179,5071, 856,1661, # 32 3511,1822,2785,4607,1193,2226,5070,4608, 171,2996,1247, 18, 179,5071, 856,1661, # 32
@@ -321,5 +322,4 @@ JIS_CHAR_TO_FREQ_ORDER = (
1444,1698,2385,2251,3729,1365,2281,2235,1717,6188, 864,3841,2515, 444, 527,2767, # 4352 1444,1698,2385,2251,3729,1365,2281,2235,1717,6188, 864,3841,2515, 444, 527,2767, # 4352
2922,3625, 544, 461,6189, 566, 209,2437,3398,2098,1065,2068,3331,3626,3257,2137, # 4368 #last 512 2922,3625, 544, 461,6189, 566, 209,2437,3398,2098,1065,2068,3331,3626,3257,2137, # 4368 #last 512
) )
# fmt: on
File diff suppressed because it is too large Load Diff
@@ -1,7 +1,13 @@
######################## BEGIN LICENSE BLOCK ######################## ######################## BEGIN LICENSE BLOCK ########################
# The Original Code is mozilla.org code.
#
# The Initial Developer of the Original Code is
# Netscape Communications Corporation.
# Portions created by the Initial Developer are Copyright (C) 1998
# the Initial Developer. All Rights Reserved.
#
# Contributor(s): # Contributor(s):
# Dan Blanchard # Mark Pilgrim - port to Python
# Ian Cordasco
# #
# This library is free software; you can redistribute it and/or # This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public # modify it under the terms of the GNU Lesser General Public
@@ -19,18 +25,23 @@
# 02110-1301 USA # 02110-1301 USA
######################### END LICENSE BLOCK ######################### ######################### END LICENSE BLOCK #########################
import sys from .chardistribution import JOHABDistributionAnalysis
from .codingstatemachine import CodingStateMachine
from .mbcharsetprober import MultiByteCharSetProber
from .mbcssm import JOHAB_SM_MODEL
if sys.version_info < (3, 0): class JOHABProber(MultiByteCharSetProber):
PY2 = True def __init__(self) -> None:
PY3 = False super().__init__()
string_types = (str, unicode) self.coding_sm = CodingStateMachine(JOHAB_SM_MODEL)
text_type = unicode self.distribution_analyzer = JOHABDistributionAnalysis()
iteritems = dict.iteritems self.reset()
else:
PY2 = False @property
PY3 = True def charset_name(self) -> str:
string_types = (bytes, str) return "Johab"
text_type = str
iteritems = dict.items @property
def language(self) -> str:
return "Korean"
+112 -107
View File
@@ -25,110 +25,114 @@
# 02110-1301 USA # 02110-1301 USA
######################### END LICENSE BLOCK ######################### ######################### END LICENSE BLOCK #########################
from typing import List, Tuple, Union
# This is hiragana 2-char sequence table, the number in each cell represents its frequency category # This is hiragana 2-char sequence table, the number in each cell represents its frequency category
jp2CharContext = ( # fmt: off
(0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1), jp2_char_context = (
(2,4,0,4,0,3,0,4,0,3,4,4,4,2,4,3,3,4,3,2,3,3,4,2,3,3,3,2,4,1,4,3,3,1,5,4,3,4,3,4,3,5,3,0,3,5,4,2,0,3,1,0,3,3,0,3,3,0,1,1,0,4,3,0,3,3,0,4,0,2,0,3,5,5,5,5,4,0,4,1,0,3,4), (0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1),
(0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2), (2, 4, 0, 4, 0, 3, 0, 4, 0, 3, 4, 4, 4, 2, 4, 3, 3, 4, 3, 2, 3, 3, 4, 2, 3, 3, 3, 2, 4, 1, 4, 3, 3, 1, 5, 4, 3, 4, 3, 4, 3, 5, 3, 0, 3, 5, 4, 2, 0, 3, 1, 0, 3, 3, 0, 3, 3, 0, 1, 1, 0, 4, 3, 0, 3, 3, 0, 4, 0, 2, 0, 3, 5, 5, 5, 5, 4, 0, 4, 1, 0, 3, 4),
(0,4,0,5,0,5,0,4,0,4,5,4,4,3,5,3,5,1,5,3,4,3,4,4,3,4,3,3,4,3,5,4,4,3,5,5,3,5,5,5,3,5,5,3,4,5,5,3,1,3,2,0,3,4,0,4,2,0,4,2,1,5,3,2,3,5,0,4,0,2,0,5,4,4,5,4,5,0,4,0,0,4,4), (0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2),
(0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0), (0, 4, 0, 5, 0, 5, 0, 4, 0, 4, 5, 4, 4, 3, 5, 3, 5, 1, 5, 3, 4, 3, 4, 4, 3, 4, 3, 3, 4, 3, 5, 4, 4, 3, 5, 5, 3, 5, 5, 5, 3, 5, 5, 3, 4, 5, 5, 3, 1, 3, 2, 0, 3, 4, 0, 4, 2, 0, 4, 2, 1, 5, 3, 2, 3, 5, 0, 4, 0, 2, 0, 5, 4, 4, 5, 4, 5, 0, 4, 0, 0, 4, 4),
(0,3,0,4,0,3,0,3,0,4,5,4,3,3,3,3,4,3,5,4,4,3,5,4,4,3,4,3,4,4,4,4,5,3,4,4,3,4,5,5,4,5,5,1,4,5,4,3,0,3,3,1,3,3,0,4,4,0,3,3,1,5,3,3,3,5,0,4,0,3,0,4,4,3,4,3,3,0,4,1,1,3,4), (0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0),
(0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0), (0, 3, 0, 4, 0, 3, 0, 3, 0, 4, 5, 4, 3, 3, 3, 3, 4, 3, 5, 4, 4, 3, 5, 4, 4, 3, 4, 3, 4, 4, 4, 4, 5, 3, 4, 4, 3, 4, 5, 5, 4, 5, 5, 1, 4, 5, 4, 3, 0, 3, 3, 1, 3, 3, 0, 4, 4, 0, 3, 3, 1, 5, 3, 3, 3, 5, 0, 4, 0, 3, 0, 4, 4, 3, 4, 3, 3, 0, 4, 1, 1, 3, 4),
(0,4,0,3,0,3,0,4,0,3,4,4,3,2,2,1,2,1,3,1,3,3,3,3,3,4,3,1,3,3,5,3,3,0,4,3,0,5,4,3,3,5,4,4,3,4,4,5,0,1,2,0,1,2,0,2,2,0,1,0,0,5,2,2,1,4,0,3,0,1,0,4,4,3,5,4,3,0,2,1,0,4,3), (0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0),
(0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0), (0, 4, 0, 3, 0, 3, 0, 4, 0, 3, 4, 4, 3, 2, 2, 1, 2, 1, 3, 1, 3, 3, 3, 3, 3, 4, 3, 1, 3, 3, 5, 3, 3, 0, 4, 3, 0, 5, 4, 3, 3, 5, 4, 4, 3, 4, 4, 5, 0, 1, 2, 0, 1, 2, 0, 2, 2, 0, 1, 0, 0, 5, 2, 2, 1, 4, 0, 3, 0, 1, 0, 4, 4, 3, 5, 4, 3, 0, 2, 1, 0, 4, 3),
(0,3,0,5,0,4,0,2,1,4,4,2,4,1,4,2,4,2,4,3,3,3,4,3,3,3,3,1,4,2,3,3,3,1,4,4,1,1,1,4,3,3,2,0,2,4,3,2,0,3,3,0,3,1,1,0,0,0,3,3,0,4,2,2,3,4,0,4,0,3,0,4,4,5,3,4,4,0,3,0,0,1,4), (0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0),
(1,4,0,4,0,4,0,4,0,3,5,4,4,3,4,3,5,4,3,3,4,3,5,4,4,4,4,3,4,2,4,3,3,1,5,4,3,2,4,5,4,5,5,4,4,5,4,4,0,3,2,2,3,3,0,4,3,1,3,2,1,4,3,3,4,5,0,3,0,2,0,4,5,5,4,5,4,0,4,0,0,5,4), (0, 3, 0, 5, 0, 4, 0, 2, 1, 4, 4, 2, 4, 1, 4, 2, 4, 2, 4, 3, 3, 3, 4, 3, 3, 3, 3, 1, 4, 2, 3, 3, 3, 1, 4, 4, 1, 1, 1, 4, 3, 3, 2, 0, 2, 4, 3, 2, 0, 3, 3, 0, 3, 1, 1, 0, 0, 0, 3, 3, 0, 4, 2, 2, 3, 4, 0, 4, 0, 3, 0, 4, 4, 5, 3, 4, 4, 0, 3, 0, 0, 1, 4),
(0,5,0,5,0,4,0,3,0,4,4,3,4,3,3,3,4,0,4,4,4,3,4,3,4,3,3,1,4,2,4,3,4,0,5,4,1,4,5,4,4,5,3,2,4,3,4,3,2,4,1,3,3,3,2,3,2,0,4,3,3,4,3,3,3,4,0,4,0,3,0,4,5,4,4,4,3,0,4,1,0,1,3), (1, 4, 0, 4, 0, 4, 0, 4, 0, 3, 5, 4, 4, 3, 4, 3, 5, 4, 3, 3, 4, 3, 5, 4, 4, 4, 4, 3, 4, 2, 4, 3, 3, 1, 5, 4, 3, 2, 4, 5, 4, 5, 5, 4, 4, 5, 4, 4, 0, 3, 2, 2, 3, 3, 0, 4, 3, 1, 3, 2, 1, 4, 3, 3, 4, 5, 0, 3, 0, 2, 0, 4, 5, 5, 4, 5, 4, 0, 4, 0, 0, 5, 4),
(0,3,1,4,0,3,0,2,0,3,4,4,3,1,4,2,3,3,4,3,4,3,4,3,4,4,3,2,3,1,5,4,4,1,4,4,3,5,4,4,3,5,5,4,3,4,4,3,1,2,3,1,2,2,0,3,2,0,3,1,0,5,3,3,3,4,3,3,3,3,4,4,4,4,5,4,2,0,3,3,2,4,3), (0, 5, 0, 5, 0, 4, 0, 3, 0, 4, 4, 3, 4, 3, 3, 3, 4, 0, 4, 4, 4, 3, 4, 3, 4, 3, 3, 1, 4, 2, 4, 3, 4, 0, 5, 4, 1, 4, 5, 4, 4, 5, 3, 2, 4, 3, 4, 3, 2, 4, 1, 3, 3, 3, 2, 3, 2, 0, 4, 3, 3, 4, 3, 3, 3, 4, 0, 4, 0, 3, 0, 4, 5, 4, 4, 4, 3, 0, 4, 1, 0, 1, 3),
(0,2,0,3,0,1,0,1,0,0,3,2,0,0,2,0,1,0,2,1,3,3,3,1,2,3,1,0,1,0,4,2,1,1,3,3,0,4,3,3,1,4,3,3,0,3,3,2,0,0,0,0,1,0,0,2,0,0,0,0,0,4,1,0,2,3,2,2,2,1,3,3,3,4,4,3,2,0,3,1,0,3,3), (0, 3, 1, 4, 0, 3, 0, 2, 0, 3, 4, 4, 3, 1, 4, 2, 3, 3, 4, 3, 4, 3, 4, 3, 4, 4, 3, 2, 3, 1, 5, 4, 4, 1, 4, 4, 3, 5, 4, 4, 3, 5, 5, 4, 3, 4, 4, 3, 1, 2, 3, 1, 2, 2, 0, 3, 2, 0, 3, 1, 0, 5, 3, 3, 3, 4, 3, 3, 3, 3, 4, 4, 4, 4, 5, 4, 2, 0, 3, 3, 2, 4, 3),
(0,4,0,4,0,3,0,3,0,4,4,4,3,3,3,3,3,3,4,3,4,2,4,3,4,3,3,2,4,3,4,5,4,1,4,5,3,5,4,5,3,5,4,0,3,5,5,3,1,3,3,2,2,3,0,3,4,1,3,3,2,4,3,3,3,4,0,4,0,3,0,4,5,4,4,5,3,0,4,1,0,3,4), (0, 2, 0, 3, 0, 1, 0, 1, 0, 0, 3, 2, 0, 0, 2, 0, 1, 0, 2, 1, 3, 3, 3, 1, 2, 3, 1, 0, 1, 0, 4, 2, 1, 1, 3, 3, 0, 4, 3, 3, 1, 4, 3, 3, 0, 3, 3, 2, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 0, 0, 0, 4, 1, 0, 2, 3, 2, 2, 2, 1, 3, 3, 3, 4, 4, 3, 2, 0, 3, 1, 0, 3, 3),
(0,2,0,3,0,3,0,0,0,2,2,2,1,0,1,0,0,0,3,0,3,0,3,0,1,3,1,0,3,1,3,3,3,1,3,3,3,0,1,3,1,3,4,0,0,3,1,1,0,3,2,0,0,0,0,1,3,0,1,0,0,3,3,2,0,3,0,0,0,0,0,3,4,3,4,3,3,0,3,0,0,2,3), (0, 4, 0, 4, 0, 3, 0, 3, 0, 4, 4, 4, 3, 3, 3, 3, 3, 3, 4, 3, 4, 2, 4, 3, 4, 3, 3, 2, 4, 3, 4, 5, 4, 1, 4, 5, 3, 5, 4, 5, 3, 5, 4, 0, 3, 5, 5, 3, 1, 3, 3, 2, 2, 3, 0, 3, 4, 1, 3, 3, 2, 4, 3, 3, 3, 4, 0, 4, 0, 3, 0, 4, 5, 4, 4, 5, 3, 0, 4, 1, 0, 3, 4),
(2,3,0,3,0,2,0,1,0,3,3,4,3,1,3,1,1,1,3,1,4,3,4,3,3,3,0,0,3,1,5,4,3,1,4,3,2,5,5,4,4,4,4,3,3,4,4,4,0,2,1,1,3,2,0,1,2,0,0,1,0,4,1,3,3,3,0,3,0,1,0,4,4,4,5,5,3,0,2,0,0,4,4), (0, 2, 0, 3, 0, 3, 0, 0, 0, 2, 2, 2, 1, 0, 1, 0, 0, 0, 3, 0, 3, 0, 3, 0, 1, 3, 1, 0, 3, 1, 3, 3, 3, 1, 3, 3, 3, 0, 1, 3, 1, 3, 4, 0, 0, 3, 1, 1, 0, 3, 2, 0, 0, 0, 0, 1, 3, 0, 1, 0, 0, 3, 3, 2, 0, 3, 0, 0, 0, 0, 0, 3, 4, 3, 4, 3, 3, 0, 3, 0, 0, 2, 3),
(0,2,0,1,0,3,1,3,0,2,3,3,3,0,3,1,0,0,3,0,3,2,3,1,3,2,1,1,0,0,4,2,1,0,2,3,1,4,3,2,0,4,4,3,1,3,1,3,0,1,0,0,1,0,0,0,1,0,0,0,0,4,1,1,1,2,0,3,0,0,0,3,4,2,4,3,2,0,1,0,0,3,3), (2, 3, 0, 3, 0, 2, 0, 1, 0, 3, 3, 4, 3, 1, 3, 1, 1, 1, 3, 1, 4, 3, 4, 3, 3, 3, 0, 0, 3, 1, 5, 4, 3, 1, 4, 3, 2, 5, 5, 4, 4, 4, 4, 3, 3, 4, 4, 4, 0, 2, 1, 1, 3, 2, 0, 1, 2, 0, 0, 1, 0, 4, 1, 3, 3, 3, 0, 3, 0, 1, 0, 4, 4, 4, 5, 5, 3, 0, 2, 0, 0, 4, 4),
(0,1,0,4,0,5,0,4,0,2,4,4,2,3,3,2,3,3,5,3,3,3,4,3,4,2,3,0,4,3,3,3,4,1,4,3,2,1,5,5,3,4,5,1,3,5,4,2,0,3,3,0,1,3,0,4,2,0,1,3,1,4,3,3,3,3,0,3,0,1,0,3,4,4,4,5,5,0,3,0,1,4,5), (0, 2, 0, 1, 0, 3, 1, 3, 0, 2, 3, 3, 3, 0, 3, 1, 0, 0, 3, 0, 3, 2, 3, 1, 3, 2, 1, 1, 0, 0, 4, 2, 1, 0, 2, 3, 1, 4, 3, 2, 0, 4, 4, 3, 1, 3, 1, 3, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 4, 1, 1, 1, 2, 0, 3, 0, 0, 0, 3, 4, 2, 4, 3, 2, 0, 1, 0, 0, 3, 3),
(0,2,0,3,0,3,0,0,0,2,3,1,3,0,4,0,1,1,3,0,3,4,3,2,3,1,0,3,3,2,3,1,3,0,2,3,0,2,1,4,1,2,2,0,0,3,3,0,0,2,0,0,0,1,0,0,0,0,2,2,0,3,2,1,3,3,0,2,0,2,0,0,3,3,1,2,4,0,3,0,2,2,3), (0, 1, 0, 4, 0, 5, 0, 4, 0, 2, 4, 4, 2, 3, 3, 2, 3, 3, 5, 3, 3, 3, 4, 3, 4, 2, 3, 0, 4, 3, 3, 3, 4, 1, 4, 3, 2, 1, 5, 5, 3, 4, 5, 1, 3, 5, 4, 2, 0, 3, 3, 0, 1, 3, 0, 4, 2, 0, 1, 3, 1, 4, 3, 3, 3, 3, 0, 3, 0, 1, 0, 3, 4, 4, 4, 5, 5, 0, 3, 0, 1, 4, 5),
(2,4,0,5,0,4,0,4,0,2,4,4,4,3,4,3,3,3,1,2,4,3,4,3,4,4,5,0,3,3,3,3,2,0,4,3,1,4,3,4,1,4,4,3,3,4,4,3,1,2,3,0,4,2,0,4,1,0,3,3,0,4,3,3,3,4,0,4,0,2,0,3,5,3,4,5,2,0,3,0,0,4,5), (0, 2, 0, 3, 0, 3, 0, 0, 0, 2, 3, 1, 3, 0, 4, 0, 1, 1, 3, 0, 3, 4, 3, 2, 3, 1, 0, 3, 3, 2, 3, 1, 3, 0, 2, 3, 0, 2, 1, 4, 1, 2, 2, 0, 0, 3, 3, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 2, 2, 0, 3, 2, 1, 3, 3, 0, 2, 0, 2, 0, 0, 3, 3, 1, 2, 4, 0, 3, 0, 2, 2, 3),
(0,3,0,4,0,1,0,1,0,1,3,2,2,1,3,0,3,0,2,0,2,0,3,0,2,0,0,0,1,0,1,1,0,0,3,1,0,0,0,4,0,3,1,0,2,1,3,0,0,0,0,0,0,3,0,0,0,0,0,0,0,4,2,2,3,1,0,3,0,0,0,1,4,4,4,3,0,0,4,0,0,1,4), (2, 4, 0, 5, 0, 4, 0, 4, 0, 2, 4, 4, 4, 3, 4, 3, 3, 3, 1, 2, 4, 3, 4, 3, 4, 4, 5, 0, 3, 3, 3, 3, 2, 0, 4, 3, 1, 4, 3, 4, 1, 4, 4, 3, 3, 4, 4, 3, 1, 2, 3, 0, 4, 2, 0, 4, 1, 0, 3, 3, 0, 4, 3, 3, 3, 4, 0, 4, 0, 2, 0, 3, 5, 3, 4, 5, 2, 0, 3, 0, 0, 4, 5),
(1,4,1,5,0,3,0,3,0,4,5,4,4,3,5,3,3,4,4,3,4,1,3,3,3,3,2,1,4,1,5,4,3,1,4,4,3,5,4,4,3,5,4,3,3,4,4,4,0,3,3,1,2,3,0,3,1,0,3,3,0,5,4,4,4,4,4,4,3,3,5,4,4,3,3,5,4,0,3,2,0,4,4), (0, 3, 0, 4, 0, 1, 0, 1, 0, 1, 3, 2, 2, 1, 3, 0, 3, 0, 2, 0, 2, 0, 3, 0, 2, 0, 0, 0, 1, 0, 1, 1, 0, 0, 3, 1, 0, 0, 0, 4, 0, 3, 1, 0, 2, 1, 3, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4, 2, 2, 3, 1, 0, 3, 0, 0, 0, 1, 4, 4, 4, 3, 0, 0, 4, 0, 0, 1, 4),
(0,2,0,3,0,1,0,0,0,1,3,3,3,2,4,1,3,0,3,1,3,0,2,2,1,1,0,0,2,0,4,3,1,0,4,3,0,4,4,4,1,4,3,1,1,3,3,1,0,2,0,0,1,3,0,0,0,0,2,0,0,4,3,2,4,3,5,4,3,3,3,4,3,3,4,3,3,0,2,1,0,3,3), (1, 4, 1, 5, 0, 3, 0, 3, 0, 4, 5, 4, 4, 3, 5, 3, 3, 4, 4, 3, 4, 1, 3, 3, 3, 3, 2, 1, 4, 1, 5, 4, 3, 1, 4, 4, 3, 5, 4, 4, 3, 5, 4, 3, 3, 4, 4, 4, 0, 3, 3, 1, 2, 3, 0, 3, 1, 0, 3, 3, 0, 5, 4, 4, 4, 4, 4, 4, 3, 3, 5, 4, 4, 3, 3, 5, 4, 0, 3, 2, 0, 4, 4),
(0,2,0,4,0,3,0,2,0,2,5,5,3,4,4,4,4,1,4,3,3,0,4,3,4,3,1,3,3,2,4,3,0,3,4,3,0,3,4,4,2,4,4,0,4,5,3,3,2,2,1,1,1,2,0,1,5,0,3,3,2,4,3,3,3,4,0,3,0,2,0,4,4,3,5,5,0,0,3,0,2,3,3), (0, 2, 0, 3, 0, 1, 0, 0, 0, 1, 3, 3, 3, 2, 4, 1, 3, 0, 3, 1, 3, 0, 2, 2, 1, 1, 0, 0, 2, 0, 4, 3, 1, 0, 4, 3, 0, 4, 4, 4, 1, 4, 3, 1, 1, 3, 3, 1, 0, 2, 0, 0, 1, 3, 0, 0, 0, 0, 2, 0, 0, 4, 3, 2, 4, 3, 5, 4, 3, 3, 3, 4, 3, 3, 4, 3, 3, 0, 2, 1, 0, 3, 3),
(0,3,0,4,0,3,0,1,0,3,4,3,3,1,3,3,3,0,3,1,3,0,4,3,3,1,1,0,3,0,3,3,0,0,4,4,0,1,5,4,3,3,5,0,3,3,4,3,0,2,0,1,1,1,0,1,3,0,1,2,1,3,3,2,3,3,0,3,0,1,0,1,3,3,4,4,1,0,1,2,2,1,3), (0, 2, 0, 4, 0, 3, 0, 2, 0, 2, 5, 5, 3, 4, 4, 4, 4, 1, 4, 3, 3, 0, 4, 3, 4, 3, 1, 3, 3, 2, 4, 3, 0, 3, 4, 3, 0, 3, 4, 4, 2, 4, 4, 0, 4, 5, 3, 3, 2, 2, 1, 1, 1, 2, 0, 1, 5, 0, 3, 3, 2, 4, 3, 3, 3, 4, 0, 3, 0, 2, 0, 4, 4, 3, 5, 5, 0, 0, 3, 0, 2, 3, 3),
(0,1,0,4,0,4,0,3,0,1,3,3,3,2,3,1,1,0,3,0,3,3,4,3,2,4,2,0,1,0,4,3,2,0,4,3,0,5,3,3,2,4,4,4,3,3,3,4,0,1,3,0,0,1,0,0,1,0,0,0,0,4,2,3,3,3,0,3,0,0,0,4,4,4,5,3,2,0,3,3,0,3,5), (0, 3, 0, 4, 0, 3, 0, 1, 0, 3, 4, 3, 3, 1, 3, 3, 3, 0, 3, 1, 3, 0, 4, 3, 3, 1, 1, 0, 3, 0, 3, 3, 0, 0, 4, 4, 0, 1, 5, 4, 3, 3, 5, 0, 3, 3, 4, 3, 0, 2, 0, 1, 1, 1, 0, 1, 3, 0, 1, 2, 1, 3, 3, 2, 3, 3, 0, 3, 0, 1, 0, 1, 3, 3, 4, 4, 1, 0, 1, 2, 2, 1, 3),
(0,2,0,3,0,0,0,3,0,1,3,0,2,0,0,0,1,0,3,1,1,3,3,0,0,3,0,0,3,0,2,3,1,0,3,1,0,3,3,2,0,4,2,2,0,2,0,0,0,4,0,0,0,0,0,0,0,0,0,0,0,2,1,2,0,1,0,1,0,0,0,1,3,1,2,0,0,0,1,0,0,1,4), (0, 1, 0, 4, 0, 4, 0, 3, 0, 1, 3, 3, 3, 2, 3, 1, 1, 0, 3, 0, 3, 3, 4, 3, 2, 4, 2, 0, 1, 0, 4, 3, 2, 0, 4, 3, 0, 5, 3, 3, 2, 4, 4, 4, 3, 3, 3, 4, 0, 1, 3, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 4, 2, 3, 3, 3, 0, 3, 0, 0, 0, 4, 4, 4, 5, 3, 2, 0, 3, 3, 0, 3, 5),
(0,3,0,3,0,5,0,1,0,2,4,3,1,3,3,2,1,1,5,2,1,0,5,1,2,0,0,0,3,3,2,2,3,2,4,3,0,0,3,3,1,3,3,0,2,5,3,4,0,3,3,0,1,2,0,2,2,0,3,2,0,2,2,3,3,3,0,2,0,1,0,3,4,4,2,5,4,0,3,0,0,3,5), (0, 2, 0, 3, 0, 0, 0, 3, 0, 1, 3, 0, 2, 0, 0, 0, 1, 0, 3, 1, 1, 3, 3, 0, 0, 3, 0, 0, 3, 0, 2, 3, 1, 0, 3, 1, 0, 3, 3, 2, 0, 4, 2, 2, 0, 2, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 2, 0, 1, 0, 1, 0, 0, 0, 1, 3, 1, 2, 0, 0, 0, 1, 0, 0, 1, 4),
(0,3,0,3,0,3,0,1,0,3,3,3,3,0,3,0,2,0,2,1,1,0,2,0,1,0,0,0,2,1,0,0,1,0,3,2,0,0,3,3,1,2,3,1,0,3,3,0,0,1,0,0,0,0,0,2,0,0,0,0,0,2,3,1,2,3,0,3,0,1,0,3,2,1,0,4,3,0,1,1,0,3,3), (0, 3, 0, 3, 0, 5, 0, 1, 0, 2, 4, 3, 1, 3, 3, 2, 1, 1, 5, 2, 1, 0, 5, 1, 2, 0, 0, 0, 3, 3, 2, 2, 3, 2, 4, 3, 0, 0, 3, 3, 1, 3, 3, 0, 2, 5, 3, 4, 0, 3, 3, 0, 1, 2, 0, 2, 2, 0, 3, 2, 0, 2, 2, 3, 3, 3, 0, 2, 0, 1, 0, 3, 4, 4, 2, 5, 4, 0, 3, 0, 0, 3, 5),
(0,4,0,5,0,3,0,3,0,4,5,5,4,3,5,3,4,3,5,3,3,2,5,3,4,4,4,3,4,3,4,5,5,3,4,4,3,4,4,5,4,4,4,3,4,5,5,4,2,3,4,2,3,4,0,3,3,1,4,3,2,4,3,3,5,5,0,3,0,3,0,5,5,5,5,4,4,0,4,0,1,4,4), (0, 3, 0, 3, 0, 3, 0, 1, 0, 3, 3, 3, 3, 0, 3, 0, 2, 0, 2, 1, 1, 0, 2, 0, 1, 0, 0, 0, 2, 1, 0, 0, 1, 0, 3, 2, 0, 0, 3, 3, 1, 2, 3, 1, 0, 3, 3, 0, 0, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 3, 1, 2, 3, 0, 3, 0, 1, 0, 3, 2, 1, 0, 4, 3, 0, 1, 1, 0, 3, 3),
(0,4,0,4,0,3,0,3,0,3,5,4,4,2,3,2,5,1,3,2,5,1,4,2,3,2,3,3,4,3,3,3,3,2,5,4,1,3,3,5,3,4,4,0,4,4,3,1,1,3,1,0,2,3,0,2,3,0,3,0,0,4,3,1,3,4,0,3,0,2,0,4,4,4,3,4,5,0,4,0,0,3,4), (0, 4, 0, 5, 0, 3, 0, 3, 0, 4, 5, 5, 4, 3, 5, 3, 4, 3, 5, 3, 3, 2, 5, 3, 4, 4, 4, 3, 4, 3, 4, 5, 5, 3, 4, 4, 3, 4, 4, 5, 4, 4, 4, 3, 4, 5, 5, 4, 2, 3, 4, 2, 3, 4, 0, 3, 3, 1, 4, 3, 2, 4, 3, 3, 5, 5, 0, 3, 0, 3, 0, 5, 5, 5, 5, 4, 4, 0, 4, 0, 1, 4, 4),
(0,3,0,3,0,3,1,2,0,3,4,4,3,3,3,0,2,2,4,3,3,1,3,3,3,1,1,0,3,1,4,3,2,3,4,4,2,4,4,4,3,4,4,3,2,4,4,3,1,3,3,1,3,3,0,4,1,0,2,2,1,4,3,2,3,3,5,4,3,3,5,4,4,3,3,0,4,0,3,2,2,4,4), (0, 4, 0, 4, 0, 3, 0, 3, 0, 3, 5, 4, 4, 2, 3, 2, 5, 1, 3, 2, 5, 1, 4, 2, 3, 2, 3, 3, 4, 3, 3, 3, 3, 2, 5, 4, 1, 3, 3, 5, 3, 4, 4, 0, 4, 4, 3, 1, 1, 3, 1, 0, 2, 3, 0, 2, 3, 0, 3, 0, 0, 4, 3, 1, 3, 4, 0, 3, 0, 2, 0, 4, 4, 4, 3, 4, 5, 0, 4, 0, 0, 3, 4),
(0,2,0,1,0,0,0,0,0,1,2,1,3,0,0,0,0,0,2,0,1,2,1,0,0,1,0,0,0,0,3,0,0,1,0,1,1,3,1,0,0,0,1,1,0,1,1,0,0,0,0,0,2,0,0,0,0,0,0,0,0,1,1,2,2,0,3,4,0,0,0,1,1,0,0,1,0,0,0,0,0,1,1), (0, 3, 0, 3, 0, 3, 1, 2, 0, 3, 4, 4, 3, 3, 3, 0, 2, 2, 4, 3, 3, 1, 3, 3, 3, 1, 1, 0, 3, 1, 4, 3, 2, 3, 4, 4, 2, 4, 4, 4, 3, 4, 4, 3, 2, 4, 4, 3, 1, 3, 3, 1, 3, 3, 0, 4, 1, 0, 2, 2, 1, 4, 3, 2, 3, 3, 5, 4, 3, 3, 5, 4, 4, 3, 3, 0, 4, 0, 3, 2, 2, 4, 4),
(0,1,0,0,0,1,0,0,0,0,4,0,4,1,4,0,3,0,4,0,3,0,4,0,3,0,3,0,4,1,5,1,4,0,0,3,0,5,0,5,2,0,1,0,0,0,2,1,4,0,1,3,0,0,3,0,0,3,1,1,4,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0), (0, 2, 0, 1, 0, 0, 0, 0, 0, 1, 2, 1, 3, 0, 0, 0, 0, 0, 2, 0, 1, 2, 1, 0, 0, 1, 0, 0, 0, 0, 3, 0, 0, 1, 0, 1, 1, 3, 1, 0, 0, 0, 1, 1, 0, 1, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 0, 3, 4, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 1),
(1,4,0,5,0,3,0,2,0,3,5,4,4,3,4,3,5,3,4,3,3,0,4,3,3,3,3,3,3,2,4,4,3,1,3,4,4,5,4,4,3,4,4,1,3,5,4,3,3,3,1,2,2,3,3,1,3,1,3,3,3,5,3,3,4,5,0,3,0,3,0,3,4,3,4,4,3,0,3,0,2,4,3), (0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 4, 0, 4, 1, 4, 0, 3, 0, 4, 0, 3, 0, 4, 0, 3, 0, 3, 0, 4, 1, 5, 1, 4, 0, 0, 3, 0, 5, 0, 5, 2, 0, 1, 0, 0, 0, 2, 1, 4, 0, 1, 3, 0, 0, 3, 0, 0, 3, 1, 1, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0),
(0,1,0,4,0,0,0,0,0,1,4,0,4,1,4,2,4,0,3,0,1,0,1,0,0,0,0,0,2,0,3,1,1,1,0,3,0,0,0,1,2,1,0,0,1,1,1,1,0,1,0,0,0,1,0,0,3,0,0,0,0,3,2,0,2,2,0,1,0,0,0,2,3,2,3,3,0,0,0,0,2,1,0), (1, 4, 0, 5, 0, 3, 0, 2, 0, 3, 5, 4, 4, 3, 4, 3, 5, 3, 4, 3, 3, 0, 4, 3, 3, 3, 3, 3, 3, 2, 4, 4, 3, 1, 3, 4, 4, 5, 4, 4, 3, 4, 4, 1, 3, 5, 4, 3, 3, 3, 1, 2, 2, 3, 3, 1, 3, 1, 3, 3, 3, 5, 3, 3, 4, 5, 0, 3, 0, 3, 0, 3, 4, 3, 4, 4, 3, 0, 3, 0, 2, 4, 3),
(0,5,1,5,0,3,0,3,0,5,4,4,5,1,5,3,3,0,4,3,4,3,5,3,4,3,3,2,4,3,4,3,3,0,3,3,1,4,4,3,4,4,4,3,4,5,5,3,2,3,1,1,3,3,1,3,1,1,3,3,2,4,5,3,3,5,0,4,0,3,0,4,4,3,5,3,3,0,3,4,0,4,3), (0, 1, 0, 4, 0, 0, 0, 0, 0, 1, 4, 0, 4, 1, 4, 2, 4, 0, 3, 0, 1, 0, 1, 0, 0, 0, 0, 0, 2, 0, 3, 1, 1, 1, 0, 3, 0, 0, 0, 1, 2, 1, 0, 0, 1, 1, 1, 1, 0, 1, 0, 0, 0, 1, 0, 0, 3, 0, 0, 0, 0, 3, 2, 0, 2, 2, 0, 1, 0, 0, 0, 2, 3, 2, 3, 3, 0, 0, 0, 0, 2, 1, 0),
(0,5,0,5,0,3,0,2,0,4,4,3,5,2,4,3,3,3,4,4,4,3,5,3,5,3,3,1,4,0,4,3,3,0,3,3,0,4,4,4,4,5,4,3,3,5,5,3,2,3,1,2,3,2,0,1,0,0,3,2,2,4,4,3,1,5,0,4,0,3,0,4,3,1,3,2,1,0,3,3,0,3,3), (0, 5, 1, 5, 0, 3, 0, 3, 0, 5, 4, 4, 5, 1, 5, 3, 3, 0, 4, 3, 4, 3, 5, 3, 4, 3, 3, 2, 4, 3, 4, 3, 3, 0, 3, 3, 1, 4, 4, 3, 4, 4, 4, 3, 4, 5, 5, 3, 2, 3, 1, 1, 3, 3, 1, 3, 1, 1, 3, 3, 2, 4, 5, 3, 3, 5, 0, 4, 0, 3, 0, 4, 4, 3, 5, 3, 3, 0, 3, 4, 0, 4, 3),
(0,4,0,5,0,5,0,4,0,4,5,5,5,3,4,3,3,2,5,4,4,3,5,3,5,3,4,0,4,3,4,4,3,2,4,4,3,4,5,4,4,5,5,0,3,5,5,4,1,3,3,2,3,3,1,3,1,0,4,3,1,4,4,3,4,5,0,4,0,2,0,4,3,4,4,3,3,0,4,0,0,5,5), (0, 5, 0, 5, 0, 3, 0, 2, 0, 4, 4, 3, 5, 2, 4, 3, 3, 3, 4, 4, 4, 3, 5, 3, 5, 3, 3, 1, 4, 0, 4, 3, 3, 0, 3, 3, 0, 4, 4, 4, 4, 5, 4, 3, 3, 5, 5, 3, 2, 3, 1, 2, 3, 2, 0, 1, 0, 0, 3, 2, 2, 4, 4, 3, 1, 5, 0, 4, 0, 3, 0, 4, 3, 1, 3, 2, 1, 0, 3, 3, 0, 3, 3),
(0,4,0,4,0,5,0,1,1,3,3,4,4,3,4,1,3,0,5,1,3,0,3,1,3,1,1,0,3,0,3,3,4,0,4,3,0,4,4,4,3,4,4,0,3,5,4,1,0,3,0,0,2,3,0,3,1,0,3,1,0,3,2,1,3,5,0,3,0,1,0,3,2,3,3,4,4,0,2,2,0,4,4), (0, 4, 0, 5, 0, 5, 0, 4, 0, 4, 5, 5, 5, 3, 4, 3, 3, 2, 5, 4, 4, 3, 5, 3, 5, 3, 4, 0, 4, 3, 4, 4, 3, 2, 4, 4, 3, 4, 5, 4, 4, 5, 5, 0, 3, 5, 5, 4, 1, 3, 3, 2, 3, 3, 1, 3, 1, 0, 4, 3, 1, 4, 4, 3, 4, 5, 0, 4, 0, 2, 0, 4, 3, 4, 4, 3, 3, 0, 4, 0, 0, 5, 5),
(2,4,0,5,0,4,0,3,0,4,5,5,4,3,5,3,5,3,5,3,5,2,5,3,4,3,3,4,3,4,5,3,2,1,5,4,3,2,3,4,5,3,4,1,2,5,4,3,0,3,3,0,3,2,0,2,3,0,4,1,0,3,4,3,3,5,0,3,0,1,0,4,5,5,5,4,3,0,4,2,0,3,5), (0, 4, 0, 4, 0, 5, 0, 1, 1, 3, 3, 4, 4, 3, 4, 1, 3, 0, 5, 1, 3, 0, 3, 1, 3, 1, 1, 0, 3, 0, 3, 3, 4, 0, 4, 3, 0, 4, 4, 4, 3, 4, 4, 0, 3, 5, 4, 1, 0, 3, 0, 0, 2, 3, 0, 3, 1, 0, 3, 1, 0, 3, 2, 1, 3, 5, 0, 3, 0, 1, 0, 3, 2, 3, 3, 4, 4, 0, 2, 2, 0, 4, 4),
(0,5,0,4,0,4,0,2,0,5,4,3,4,3,4,3,3,3,4,3,4,2,5,3,5,3,4,1,4,3,4,4,4,0,3,5,0,4,4,4,4,5,3,1,3,4,5,3,3,3,3,3,3,3,0,2,2,0,3,3,2,4,3,3,3,5,3,4,1,3,3,5,3,2,0,0,0,0,4,3,1,3,3), (2, 4, 0, 5, 0, 4, 0, 3, 0, 4, 5, 5, 4, 3, 5, 3, 5, 3, 5, 3, 5, 2, 5, 3, 4, 3, 3, 4, 3, 4, 5, 3, 2, 1, 5, 4, 3, 2, 3, 4, 5, 3, 4, 1, 2, 5, 4, 3, 0, 3, 3, 0, 3, 2, 0, 2, 3, 0, 4, 1, 0, 3, 4, 3, 3, 5, 0, 3, 0, 1, 0, 4, 5, 5, 5, 4, 3, 0, 4, 2, 0, 3, 5),
(0,1,0,3,0,3,0,1,0,1,3,3,3,2,3,3,3,0,3,0,0,0,3,1,3,0,0,0,2,2,2,3,0,0,3,2,0,1,2,4,1,3,3,0,0,3,3,3,0,1,0,0,2,1,0,0,3,0,3,1,0,3,0,0,1,3,0,2,0,1,0,3,3,1,3,3,0,0,1,1,0,3,3), (0, 5, 0, 4, 0, 4, 0, 2, 0, 5, 4, 3, 4, 3, 4, 3, 3, 3, 4, 3, 4, 2, 5, 3, 5, 3, 4, 1, 4, 3, 4, 4, 4, 0, 3, 5, 0, 4, 4, 4, 4, 5, 3, 1, 3, 4, 5, 3, 3, 3, 3, 3, 3, 3, 0, 2, 2, 0, 3, 3, 2, 4, 3, 3, 3, 5, 3, 4, 1, 3, 3, 5, 3, 2, 0, 0, 0, 0, 4, 3, 1, 3, 3),
(0,2,0,3,0,2,1,4,0,2,2,3,1,1,3,1,1,0,2,0,3,1,2,3,1,3,0,0,1,0,4,3,2,3,3,3,1,4,2,3,3,3,3,1,0,3,1,4,0,1,1,0,1,2,0,1,1,0,1,1,0,3,1,3,2,2,0,1,0,0,0,2,3,3,3,1,0,0,0,0,0,2,3), (0, 1, 0, 3, 0, 3, 0, 1, 0, 1, 3, 3, 3, 2, 3, 3, 3, 0, 3, 0, 0, 0, 3, 1, 3, 0, 0, 0, 2, 2, 2, 3, 0, 0, 3, 2, 0, 1, 2, 4, 1, 3, 3, 0, 0, 3, 3, 3, 0, 1, 0, 0, 2, 1, 0, 0, 3, 0, 3, 1, 0, 3, 0, 0, 1, 3, 0, 2, 0, 1, 0, 3, 3, 1, 3, 3, 0, 0, 1, 1, 0, 3, 3),
(0,5,0,4,0,5,0,2,0,4,5,5,3,3,4,3,3,1,5,4,4,2,4,4,4,3,4,2,4,3,5,5,4,3,3,4,3,3,5,5,4,5,5,1,3,4,5,3,1,4,3,1,3,3,0,3,3,1,4,3,1,4,5,3,3,5,0,4,0,3,0,5,3,3,1,4,3,0,4,0,1,5,3), (0, 2, 0, 3, 0, 2, 1, 4, 0, 2, 2, 3, 1, 1, 3, 1, 1, 0, 2, 0, 3, 1, 2, 3, 1, 3, 0, 0, 1, 0, 4, 3, 2, 3, 3, 3, 1, 4, 2, 3, 3, 3, 3, 1, 0, 3, 1, 4, 0, 1, 1, 0, 1, 2, 0, 1, 1, 0, 1, 1, 0, 3, 1, 3, 2, 2, 0, 1, 0, 0, 0, 2, 3, 3, 3, 1, 0, 0, 0, 0, 0, 2, 3),
(0,5,0,5,0,4,0,2,0,4,4,3,4,3,3,3,3,3,5,4,4,4,4,4,4,5,3,3,5,2,4,4,4,3,4,4,3,3,4,4,5,5,3,3,4,3,4,3,3,4,3,3,3,3,1,2,2,1,4,3,3,5,4,4,3,4,0,4,0,3,0,4,4,4,4,4,1,0,4,2,0,2,4), (0, 5, 0, 4, 0, 5, 0, 2, 0, 4, 5, 5, 3, 3, 4, 3, 3, 1, 5, 4, 4, 2, 4, 4, 4, 3, 4, 2, 4, 3, 5, 5, 4, 3, 3, 4, 3, 3, 5, 5, 4, 5, 5, 1, 3, 4, 5, 3, 1, 4, 3, 1, 3, 3, 0, 3, 3, 1, 4, 3, 1, 4, 5, 3, 3, 5, 0, 4, 0, 3, 0, 5, 3, 3, 1, 4, 3, 0, 4, 0, 1, 5, 3),
(0,4,0,4,0,3,0,1,0,3,5,2,3,0,3,0,2,1,4,2,3,3,4,1,4,3,3,2,4,1,3,3,3,0,3,3,0,0,3,3,3,5,3,3,3,3,3,2,0,2,0,0,2,0,0,2,0,0,1,0,0,3,1,2,2,3,0,3,0,2,0,4,4,3,3,4,1,0,3,0,0,2,4), (0, 5, 0, 5, 0, 4, 0, 2, 0, 4, 4, 3, 4, 3, 3, 3, 3, 3, 5, 4, 4, 4, 4, 4, 4, 5, 3, 3, 5, 2, 4, 4, 4, 3, 4, 4, 3, 3, 4, 4, 5, 5, 3, 3, 4, 3, 4, 3, 3, 4, 3, 3, 3, 3, 1, 2, 2, 1, 4, 3, 3, 5, 4, 4, 3, 4, 0, 4, 0, 3, 0, 4, 4, 4, 4, 4, 1, 0, 4, 2, 0, 2, 4),
(0,0,0,4,0,0,0,0,0,0,1,0,1,0,2,0,0,0,0,0,1,0,2,0,1,0,0,0,0,0,3,1,3,0,3,2,0,0,0,1,0,3,2,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3,4,0,2,0,0,0,0,0,0,2), (0, 4, 0, 4, 0, 3, 0, 1, 0, 3, 5, 2, 3, 0, 3, 0, 2, 1, 4, 2, 3, 3, 4, 1, 4, 3, 3, 2, 4, 1, 3, 3, 3, 0, 3, 3, 0, 0, 3, 3, 3, 5, 3, 3, 3, 3, 3, 2, 0, 2, 0, 0, 2, 0, 0, 2, 0, 0, 1, 0, 0, 3, 1, 2, 2, 3, 0, 3, 0, 2, 0, 4, 4, 3, 3, 4, 1, 0, 3, 0, 0, 2, 4),
(0,2,1,3,0,2,0,2,0,3,3,3,3,1,3,1,3,3,3,3,3,3,4,2,2,1,2,1,4,0,4,3,1,3,3,3,2,4,3,5,4,3,3,3,3,3,3,3,0,1,3,0,2,0,0,1,0,0,1,0,0,4,2,0,2,3,0,3,3,0,3,3,4,2,3,1,4,0,1,2,0,2,3), (0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 2, 0, 0, 0, 0, 0, 1, 0, 2, 0, 1, 0, 0, 0, 0, 0, 3, 1, 3, 0, 3, 2, 0, 0, 0, 1, 0, 3, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 0, 2, 0, 0, 0, 0, 0, 0, 2),
(0,3,0,3,0,1,0,3,0,2,3,3,3,0,3,1,2,0,3,3,2,3,3,2,3,2,3,1,3,0,4,3,2,0,3,3,1,4,3,3,2,3,4,3,1,3,3,1,1,0,1,1,0,1,0,1,0,1,0,0,0,4,1,1,0,3,0,3,1,0,2,3,3,3,3,3,1,0,0,2,0,3,3), (0, 2, 1, 3, 0, 2, 0, 2, 0, 3, 3, 3, 3, 1, 3, 1, 3, 3, 3, 3, 3, 3, 4, 2, 2, 1, 2, 1, 4, 0, 4, 3, 1, 3, 3, 3, 2, 4, 3, 5, 4, 3, 3, 3, 3, 3, 3, 3, 0, 1, 3, 0, 2, 0, 0, 1, 0, 0, 1, 0, 0, 4, 2, 0, 2, 3, 0, 3, 3, 0, 3, 3, 4, 2, 3, 1, 4, 0, 1, 2, 0, 2, 3),
(0,0,0,0,0,0,0,0,0,0,3,0,2,0,3,0,0,0,0,0,0,0,3,0,0,0,0,0,0,0,3,0,3,0,3,1,0,1,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3,0,2,0,2,3,0,0,0,0,0,0,0,0,3), (0, 3, 0, 3, 0, 1, 0, 3, 0, 2, 3, 3, 3, 0, 3, 1, 2, 0, 3, 3, 2, 3, 3, 2, 3, 2, 3, 1, 3, 0, 4, 3, 2, 0, 3, 3, 1, 4, 3, 3, 2, 3, 4, 3, 1, 3, 3, 1, 1, 0, 1, 1, 0, 1, 0, 1, 0, 1, 0, 0, 0, 4, 1, 1, 0, 3, 0, 3, 1, 0, 2, 3, 3, 3, 3, 3, 1, 0, 0, 2, 0, 3, 3),
(0,2,0,3,1,3,0,3,0,2,3,3,3,1,3,1,3,1,3,1,3,3,3,1,3,0,2,3,1,1,4,3,3,2,3,3,1,2,2,4,1,3,3,0,1,4,2,3,0,1,3,0,3,0,0,1,3,0,2,0,0,3,3,2,1,3,0,3,0,2,0,3,4,4,4,3,1,0,3,0,0,3,3), (0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 0, 3, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 0, 3, 1, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3),
(0,2,0,1,0,2,0,0,0,1,3,2,2,1,3,0,1,1,3,0,3,2,3,1,2,0,2,0,1,1,3,3,3,0,3,3,1,1,2,3,2,3,3,1,2,3,2,0,0,1,0,0,0,0,0,0,3,0,1,0,0,2,1,2,1,3,0,3,0,0,0,3,4,4,4,3,2,0,2,0,0,2,4), (0, 2, 0, 3, 1, 3, 0, 3, 0, 2, 3, 3, 3, 1, 3, 1, 3, 1, 3, 1, 3, 3, 3, 1, 3, 0, 2, 3, 1, 1, 4, 3, 3, 2, 3, 3, 1, 2, 2, 4, 1, 3, 3, 0, 1, 4, 2, 3, 0, 1, 3, 0, 3, 0, 0, 1, 3, 0, 2, 0, 0, 3, 3, 2, 1, 3, 0, 3, 0, 2, 0, 3, 4, 4, 4, 3, 1, 0, 3, 0, 0, 3, 3),
(0,0,0,1,0,1,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,1,1,1,0,0,0,0,0,0,0,0,0,2,2,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,3,1,0,0,0,0,0,0,0,3), (0, 2, 0, 1, 0, 2, 0, 0, 0, 1, 3, 2, 2, 1, 3, 0, 1, 1, 3, 0, 3, 2, 3, 1, 2, 0, 2, 0, 1, 1, 3, 3, 3, 0, 3, 3, 1, 1, 2, 3, 2, 3, 3, 1, 2, 3, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 0, 2, 1, 2, 1, 3, 0, 3, 0, 0, 0, 3, 4, 4, 4, 3, 2, 0, 2, 0, 0, 2, 4),
(0,3,0,3,0,2,0,3,0,3,3,3,2,3,2,2,2,0,3,1,3,3,3,2,3,3,0,0,3,0,3,2,2,0,2,3,1,4,3,4,3,3,2,3,1,5,4,4,0,3,1,2,1,3,0,3,1,1,2,0,2,3,1,3,1,3,0,3,0,1,0,3,3,4,4,2,1,0,2,1,0,2,4), (0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 3, 1, 0, 0, 0, 0, 0, 0, 0, 3),
(0,1,0,3,0,1,0,2,0,1,4,2,5,1,4,0,2,0,2,1,3,1,4,0,2,1,0,0,2,1,4,1,1,0,3,3,0,5,1,3,2,3,3,1,0,3,2,3,0,1,0,0,0,0,0,0,1,0,0,0,0,4,0,1,0,3,0,2,0,1,0,3,3,3,4,3,3,0,0,0,0,2,3), (0, 3, 0, 3, 0, 2, 0, 3, 0, 3, 3, 3, 2, 3, 2, 2, 2, 0, 3, 1, 3, 3, 3, 2, 3, 3, 0, 0, 3, 0, 3, 2, 2, 0, 2, 3, 1, 4, 3, 4, 3, 3, 2, 3, 1, 5, 4, 4, 0, 3, 1, 2, 1, 3, 0, 3, 1, 1, 2, 0, 2, 3, 1, 3, 1, 3, 0, 3, 0, 1, 0, 3, 3, 4, 4, 2, 1, 0, 2, 1, 0, 2, 4),
(0,0,0,1,0,0,0,0,0,0,2,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,3,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,1,0,0,1,0,0,0,0,0,3), (0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 4, 2, 5, 1, 4, 0, 2, 0, 2, 1, 3, 1, 4, 0, 2, 1, 0, 0, 2, 1, 4, 1, 1, 0, 3, 3, 0, 5, 1, 3, 2, 3, 3, 1, 0, 3, 2, 3, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4, 0, 1, 0, 3, 0, 2, 0, 1, 0, 3, 3, 3, 4, 3, 3, 0, 0, 0, 0, 2, 3),
(0,1,0,3,0,4,0,3,0,2,4,3,1,0,3,2,2,1,3,1,2,2,3,1,1,1,2,1,3,0,1,2,0,1,3,2,1,3,0,5,5,1,0,0,1,3,2,1,0,3,0,0,1,0,0,0,0,0,3,4,0,1,1,1,3,2,0,2,0,1,0,2,3,3,1,2,3,0,1,0,1,0,4), (0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 1, 0, 0, 0, 0, 0, 3),
(0,0,0,1,0,3,0,3,0,2,2,1,0,0,4,0,3,0,3,1,3,0,3,0,3,0,1,0,3,0,3,1,3,0,3,3,0,0,1,2,1,1,1,0,1,2,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,2,2,1,2,0,0,2,0,0,0,0,2,3,3,3,3,0,0,0,0,1,4), (0, 1, 0, 3, 0, 4, 0, 3, 0, 2, 4, 3, 1, 0, 3, 2, 2, 1, 3, 1, 2, 2, 3, 1, 1, 1, 2, 1, 3, 0, 1, 2, 0, 1, 3, 2, 1, 3, 0, 5, 5, 1, 0, 0, 1, 3, 2, 1, 0, 3, 0, 0, 1, 0, 0, 0, 0, 0, 3, 4, 0, 1, 1, 1, 3, 2, 0, 2, 0, 1, 0, 2, 3, 3, 1, 2, 3, 0, 1, 0, 1, 0, 4),
(0,0,0,3,0,3,0,0,0,0,3,1,1,0,3,0,1,0,2,0,1,0,0,0,0,0,0,0,1,0,3,0,2,0,2,3,0,0,2,2,3,1,2,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3,0,0,2,0,0,0,0,2,3), (0, 0, 0, 1, 0, 3, 0, 3, 0, 2, 2, 1, 0, 0, 4, 0, 3, 0, 3, 1, 3, 0, 3, 0, 3, 0, 1, 0, 3, 0, 3, 1, 3, 0, 3, 3, 0, 0, 1, 2, 1, 1, 1, 0, 1, 2, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 1, 2, 0, 0, 2, 0, 0, 0, 0, 2, 3, 3, 3, 3, 0, 0, 0, 0, 1, 4),
(2,4,0,5,0,5,0,4,0,3,4,3,3,3,4,3,3,3,4,3,4,4,5,4,5,5,5,2,3,0,5,5,4,1,5,4,3,1,5,4,3,4,4,3,3,4,3,3,0,3,2,0,2,3,0,3,0,0,3,3,0,5,3,2,3,3,0,3,0,3,0,3,4,5,4,5,3,0,4,3,0,3,4), (0, 0, 0, 3, 0, 3, 0, 0, 0, 0, 3, 1, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 2, 0, 2, 3, 0, 0, 2, 2, 3, 1, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 2, 0, 0, 0, 0, 2, 3),
(0,3,0,3,0,3,0,3,0,3,3,4,3,2,3,2,3,0,4,3,3,3,3,3,3,3,3,0,3,2,4,3,3,1,3,4,3,4,4,4,3,4,4,3,2,4,4,1,0,2,0,0,1,1,0,2,0,0,3,1,0,5,3,2,1,3,0,3,0,1,2,4,3,2,4,3,3,0,3,2,0,4,4), (2, 4, 0, 5, 0, 5, 0, 4, 0, 3, 4, 3, 3, 3, 4, 3, 3, 3, 4, 3, 4, 4, 5, 4, 5, 5, 5, 2, 3, 0, 5, 5, 4, 1, 5, 4, 3, 1, 5, 4, 3, 4, 4, 3, 3, 4, 3, 3, 0, 3, 2, 0, 2, 3, 0, 3, 0, 0, 3, 3, 0, 5, 3, 2, 3, 3, 0, 3, 0, 3, 0, 3, 4, 5, 4, 5, 3, 0, 4, 3, 0, 3, 4),
(0,3,0,3,0,1,0,0,0,1,4,3,3,2,3,1,3,1,4,2,3,2,4,2,3,4,3,0,2,2,3,3,3,0,3,3,3,0,3,4,1,3,3,0,3,4,3,3,0,1,1,0,1,0,0,0,4,0,3,0,0,3,1,2,1,3,0,4,0,1,0,4,3,3,4,3,3,0,2,0,0,3,3), (0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 3, 4, 3, 2, 3, 2, 3, 0, 4, 3, 3, 3, 3, 3, 3, 3, 3, 0, 3, 2, 4, 3, 3, 1, 3, 4, 3, 4, 4, 4, 3, 4, 4, 3, 2, 4, 4, 1, 0, 2, 0, 0, 1, 1, 0, 2, 0, 0, 3, 1, 0, 5, 3, 2, 1, 3, 0, 3, 0, 1, 2, 4, 3, 2, 4, 3, 3, 0, 3, 2, 0, 4, 4),
(0,3,0,4,0,1,0,3,0,3,4,3,3,0,3,3,3,1,3,1,3,3,4,3,3,3,0,0,3,1,5,3,3,1,3,3,2,5,4,3,3,4,5,3,2,5,3,4,0,1,0,0,0,0,0,2,0,0,1,1,0,4,2,2,1,3,0,3,0,2,0,4,4,3,5,3,2,0,1,1,0,3,4), (0, 3, 0, 3, 0, 1, 0, 0, 0, 1, 4, 3, 3, 2, 3, 1, 3, 1, 4, 2, 3, 2, 4, 2, 3, 4, 3, 0, 2, 2, 3, 3, 3, 0, 3, 3, 3, 0, 3, 4, 1, 3, 3, 0, 3, 4, 3, 3, 0, 1, 1, 0, 1, 0, 0, 0, 4, 0, 3, 0, 0, 3, 1, 2, 1, 3, 0, 4, 0, 1, 0, 4, 3, 3, 4, 3, 3, 0, 2, 0, 0, 3, 3),
(0,5,0,4,0,5,0,2,0,4,4,3,3,2,3,3,3,1,4,3,4,1,5,3,4,3,4,0,4,2,4,3,4,1,5,4,0,4,4,4,4,5,4,1,3,5,4,2,1,4,1,1,3,2,0,3,1,0,3,2,1,4,3,3,3,4,0,4,0,3,0,4,4,4,3,3,3,0,4,2,0,3,4), (0, 3, 0, 4, 0, 1, 0, 3, 0, 3, 4, 3, 3, 0, 3, 3, 3, 1, 3, 1, 3, 3, 4, 3, 3, 3, 0, 0, 3, 1, 5, 3, 3, 1, 3, 3, 2, 5, 4, 3, 3, 4, 5, 3, 2, 5, 3, 4, 0, 1, 0, 0, 0, 0, 0, 2, 0, 0, 1, 1, 0, 4, 2, 2, 1, 3, 0, 3, 0, 2, 0, 4, 4, 3, 5, 3, 2, 0, 1, 1, 0, 3, 4),
(1,4,0,4,0,3,0,1,0,3,3,3,1,1,3,3,2,2,3,3,1,0,3,2,2,1,2,0,3,1,2,1,2,0,3,2,0,2,2,3,3,4,3,0,3,3,1,2,0,1,1,3,1,2,0,0,3,0,1,1,0,3,2,2,3,3,0,3,0,0,0,2,3,3,4,3,3,0,1,0,0,1,4), (0, 5, 0, 4, 0, 5, 0, 2, 0, 4, 4, 3, 3, 2, 3, 3, 3, 1, 4, 3, 4, 1, 5, 3, 4, 3, 4, 0, 4, 2, 4, 3, 4, 1, 5, 4, 0, 4, 4, 4, 4, 5, 4, 1, 3, 5, 4, 2, 1, 4, 1, 1, 3, 2, 0, 3, 1, 0, 3, 2, 1, 4, 3, 3, 3, 4, 0, 4, 0, 3, 0, 4, 4, 4, 3, 3, 3, 0, 4, 2, 0, 3, 4),
(0,4,0,4,0,4,0,0,0,3,4,4,3,1,4,2,3,2,3,3,3,1,4,3,4,0,3,0,4,2,3,3,2,2,5,4,2,1,3,4,3,4,3,1,3,3,4,2,0,2,1,0,3,3,0,0,2,0,3,1,0,4,4,3,4,3,0,4,0,1,0,2,4,4,4,4,4,0,3,2,0,3,3), (1, 4, 0, 4, 0, 3, 0, 1, 0, 3, 3, 3, 1, 1, 3, 3, 2, 2, 3, 3, 1, 0, 3, 2, 2, 1, 2, 0, 3, 1, 2, 1, 2, 0, 3, 2, 0, 2, 2, 3, 3, 4, 3, 0, 3, 3, 1, 2, 0, 1, 1, 3, 1, 2, 0, 0, 3, 0, 1, 1, 0, 3, 2, 2, 3, 3, 0, 3, 0, 0, 0, 2, 3, 3, 4, 3, 3, 0, 1, 0, 0, 1, 4),
(0,0,0,1,0,4,0,0,0,0,0,0,1,1,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,3,2,0,0,1,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,2), (0, 4, 0, 4, 0, 4, 0, 0, 0, 3, 4, 4, 3, 1, 4, 2, 3, 2, 3, 3, 3, 1, 4, 3, 4, 0, 3, 0, 4, 2, 3, 3, 2, 2, 5, 4, 2, 1, 3, 4, 3, 4, 3, 1, 3, 3, 4, 2, 0, 2, 1, 0, 3, 3, 0, 0, 2, 0, 3, 1, 0, 4, 4, 3, 4, 3, 0, 4, 0, 1, 0, 2, 4, 4, 4, 4, 4, 0, 3, 2, 0, 3, 3),
(0,2,0,3,0,4,0,4,0,1,3,3,3,0,4,0,2,1,2,1,1,1,2,0,3,1,1,0,1,0,3,1,0,0,3,3,2,0,1,1,0,0,0,0,0,1,0,2,0,2,2,0,3,1,0,0,1,0,1,1,0,1,2,0,3,0,0,0,0,1,0,0,3,3,4,3,1,0,1,0,3,0,2), (0, 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 2, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2),
(0,0,0,3,0,5,0,0,0,0,1,0,2,0,3,1,0,1,3,0,0,0,2,0,0,0,1,0,0,0,1,1,0,0,4,0,0,0,2,3,0,1,4,1,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,3,0,0,0,0,0,1,0,0,0,0,0,0,0,2,0,0,3,0,0,0,0,0,3), (0, 2, 0, 3, 0, 4, 0, 4, 0, 1, 3, 3, 3, 0, 4, 0, 2, 1, 2, 1, 1, 1, 2, 0, 3, 1, 1, 0, 1, 0, 3, 1, 0, 0, 3, 3, 2, 0, 1, 1, 0, 0, 0, 0, 0, 1, 0, 2, 0, 2, 2, 0, 3, 1, 0, 0, 1, 0, 1, 1, 0, 1, 2, 0, 3, 0, 0, 0, 0, 1, 0, 0, 3, 3, 4, 3, 1, 0, 1, 0, 3, 0, 2),
(0,2,0,5,0,5,0,1,0,2,4,3,3,2,5,1,3,2,3,3,3,0,4,1,2,0,3,0,4,0,2,2,1,1,5,3,0,0,1,4,2,3,2,0,3,3,3,2,0,2,4,1,1,2,0,1,1,0,3,1,0,1,3,1,2,3,0,2,0,0,0,1,3,5,4,4,4,0,3,0,0,1,3), (0, 0, 0, 3, 0, 5, 0, 0, 0, 0, 1, 0, 2, 0, 3, 1, 0, 1, 3, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 4, 0, 0, 0, 2, 3, 0, 1, 4, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3, 0, 0, 0, 0, 0, 3),
(0,4,0,5,0,4,0,4,0,4,5,4,3,3,4,3,3,3,4,3,4,4,5,3,4,5,4,2,4,2,3,4,3,1,4,4,1,3,5,4,4,5,5,4,4,5,5,5,2,3,3,1,4,3,1,3,3,0,3,3,1,4,3,4,4,4,0,3,0,4,0,3,3,4,4,5,0,0,4,3,0,4,5), (0, 2, 0, 5, 0, 5, 0, 1, 0, 2, 4, 3, 3, 2, 5, 1, 3, 2, 3, 3, 3, 0, 4, 1, 2, 0, 3, 0, 4, 0, 2, 2, 1, 1, 5, 3, 0, 0, 1, 4, 2, 3, 2, 0, 3, 3, 3, 2, 0, 2, 4, 1, 1, 2, 0, 1, 1, 0, 3, 1, 0, 1, 3, 1, 2, 3, 0, 2, 0, 0, 0, 1, 3, 5, 4, 4, 4, 0, 3, 0, 0, 1, 3),
(0,4,0,4,0,3,0,3,0,3,4,4,4,3,3,2,4,3,4,3,4,3,5,3,4,3,2,1,4,2,4,4,3,1,3,4,2,4,5,5,3,4,5,4,1,5,4,3,0,3,2,2,3,2,1,3,1,0,3,3,3,5,3,3,3,5,4,4,2,3,3,4,3,3,3,2,1,0,3,2,1,4,3), (0, 4, 0, 5, 0, 4, 0, 4, 0, 4, 5, 4, 3, 3, 4, 3, 3, 3, 4, 3, 4, 4, 5, 3, 4, 5, 4, 2, 4, 2, 3, 4, 3, 1, 4, 4, 1, 3, 5, 4, 4, 5, 5, 4, 4, 5, 5, 5, 2, 3, 3, 1, 4, 3, 1, 3, 3, 0, 3, 3, 1, 4, 3, 4, 4, 4, 0, 3, 0, 4, 0, 3, 3, 4, 4, 5, 0, 0, 4, 3, 0, 4, 5),
(0,4,0,5,0,4,0,3,0,3,5,5,3,2,4,3,4,0,5,4,4,1,4,4,4,3,3,3,4,3,5,5,2,3,3,4,1,2,5,5,3,5,5,2,3,5,5,4,0,3,2,0,3,3,1,1,5,1,4,1,0,4,3,2,3,5,0,4,0,3,0,5,4,3,4,3,0,0,4,1,0,4,4), (0, 4, 0, 4, 0, 3, 0, 3, 0, 3, 4, 4, 4, 3, 3, 2, 4, 3, 4, 3, 4, 3, 5, 3, 4, 3, 2, 1, 4, 2, 4, 4, 3, 1, 3, 4, 2, 4, 5, 5, 3, 4, 5, 4, 1, 5, 4, 3, 0, 3, 2, 2, 3, 2, 1, 3, 1, 0, 3, 3, 3, 5, 3, 3, 3, 5, 4, 4, 2, 3, 3, 4, 3, 3, 3, 2, 1, 0, 3, 2, 1, 4, 3),
(1,3,0,4,0,2,0,2,0,2,5,5,3,3,3,3,3,0,4,2,3,4,4,4,3,4,0,0,3,4,5,4,3,3,3,3,2,5,5,4,5,5,5,4,3,5,5,5,1,3,1,0,1,0,0,3,2,0,4,2,0,5,2,3,2,4,1,3,0,3,0,4,5,4,5,4,3,0,4,2,0,5,4), (0, 4, 0, 5, 0, 4, 0, 3, 0, 3, 5, 5, 3, 2, 4, 3, 4, 0, 5, 4, 4, 1, 4, 4, 4, 3, 3, 3, 4, 3, 5, 5, 2, 3, 3, 4, 1, 2, 5, 5, 3, 5, 5, 2, 3, 5, 5, 4, 0, 3, 2, 0, 3, 3, 1, 1, 5, 1, 4, 1, 0, 4, 3, 2, 3, 5, 0, 4, 0, 3, 0, 5, 4, 3, 4, 3, 0, 0, 4, 1, 0, 4, 4),
(0,3,0,4,0,5,0,3,0,3,4,4,3,2,3,2,3,3,3,3,3,2,4,3,3,2,2,0,3,3,3,3,3,1,3,3,3,0,4,4,3,4,4,1,1,4,4,2,0,3,1,0,1,1,0,4,1,0,2,3,1,3,3,1,3,4,0,3,0,1,0,3,1,3,0,0,1,0,2,0,0,4,4), (1, 3, 0, 4, 0, 2, 0, 2, 0, 2, 5, 5, 3, 3, 3, 3, 3, 0, 4, 2, 3, 4, 4, 4, 3, 4, 0, 0, 3, 4, 5, 4, 3, 3, 3, 3, 2, 5, 5, 4, 5, 5, 5, 4, 3, 5, 5, 5, 1, 3, 1, 0, 1, 0, 0, 3, 2, 0, 4, 2, 0, 5, 2, 3, 2, 4, 1, 3, 0, 3, 0, 4, 5, 4, 5, 4, 3, 0, 4, 2, 0, 5, 4),
(0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0), (0, 3, 0, 4, 0, 5, 0, 3, 0, 3, 4, 4, 3, 2, 3, 2, 3, 3, 3, 3, 3, 2, 4, 3, 3, 2, 2, 0, 3, 3, 3, 3, 3, 1, 3, 3, 3, 0, 4, 4, 3, 4, 4, 1, 1, 4, 4, 2, 0, 3, 1, 0, 1, 1, 0, 4, 1, 0, 2, 3, 1, 3, 3, 1, 3, 4, 0, 3, 0, 1, 0, 3, 1, 3, 0, 0, 1, 0, 2, 0, 0, 4, 4),
(0,3,0,3,0,2,0,3,0,1,5,4,3,3,3,1,4,2,1,2,3,4,4,2,4,4,5,0,3,1,4,3,4,0,4,3,3,3,2,3,2,5,3,4,3,2,2,3,0,0,3,0,2,1,0,1,2,0,0,0,0,2,1,1,3,1,0,2,0,4,0,3,4,4,4,5,2,0,2,0,0,1,3), (0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0),
(0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,1,1,0,0,1,1,0,0,0,4,2,1,1,0,1,0,3,2,0,0,3,1,1,1,2,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3,0,1,0,0,0,2,0,0,0,1,4,0,4,2,1,0,0,0,0,0,1), (0, 3, 0, 3, 0, 2, 0, 3, 0, 1, 5, 4, 3, 3, 3, 1, 4, 2, 1, 2, 3, 4, 4, 2, 4, 4, 5, 0, 3, 1, 4, 3, 4, 0, 4, 3, 3, 3, 2, 3, 2, 5, 3, 4, 3, 2, 2, 3, 0, 0, 3, 0, 2, 1, 0, 1, 2, 0, 0, 0, 0, 2, 1, 1, 3, 1, 0, 2, 0, 4, 0, 3, 4, 4, 4, 5, 2, 0, 2, 0, 0, 1, 3),
(0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,1,0,0,0,0,0,0,1,0,1,0,0,0,0,3,1,0,0,0,2,0,2,1,0,0,1,2,1,0,1,1,0,0,3,0,0,0,0,0,0,0,0,0,0,0,1,3,1,0,0,0,0,0,1,0,0,2,1,0,0,0,0,0,0,0,0,2), (0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 1, 1, 0, 0, 0, 4, 2, 1, 1, 0, 1, 0, 3, 2, 0, 0, 3, 1, 1, 1, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 0, 0, 2, 0, 0, 0, 1, 4, 0, 4, 2, 1, 0, 0, 0, 0, 0, 1),
(0,4,0,4,0,4,0,3,0,4,4,3,4,2,4,3,2,0,4,4,4,3,5,3,5,3,3,2,4,2,4,3,4,3,1,4,0,2,3,4,4,4,3,3,3,4,4,4,3,4,1,3,4,3,2,1,2,1,3,3,3,4,4,3,3,5,0,4,0,3,0,4,3,3,3,2,1,0,3,0,0,3,3), (0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 3, 1, 0, 0, 0, 2, 0, 2, 1, 0, 0, 1, 2, 1, 0, 1, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 1, 0, 0, 0, 0, 0, 1, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2),
(0,4,0,3,0,3,0,3,0,3,5,5,3,3,3,3,4,3,4,3,3,3,4,4,4,3,3,3,3,4,3,5,3,3,1,3,2,4,5,5,5,5,4,3,4,5,5,3,2,2,3,3,3,3,2,3,3,1,2,3,2,4,3,3,3,4,0,4,0,2,0,4,3,2,2,1,2,0,3,0,0,4,1), (0, 4, 0, 4, 0, 4, 0, 3, 0, 4, 4, 3, 4, 2, 4, 3, 2, 0, 4, 4, 4, 3, 5, 3, 5, 3, 3, 2, 4, 2, 4, 3, 4, 3, 1, 4, 0, 2, 3, 4, 4, 4, 3, 3, 3, 4, 4, 4, 3, 4, 1, 3, 4, 3, 2, 1, 2, 1, 3, 3, 3, 4, 4, 3, 3, 5, 0, 4, 0, 3, 0, 4, 3, 3, 3, 2, 1, 0, 3, 0, 0, 3, 3),
(0, 4, 0, 3, 0, 3, 0, 3, 0, 3, 5, 5, 3, 3, 3, 3, 4, 3, 4, 3, 3, 3, 4, 4, 4, 3, 3, 3, 3, 4, 3, 5, 3, 3, 1, 3, 2, 4, 5, 5, 5, 5, 4, 3, 4, 5, 5, 3, 2, 2, 3, 3, 3, 3, 2, 3, 3, 1, 2, 3, 2, 4, 3, 3, 3, 4, 0, 4, 0, 2, 0, 4, 3, 2, 2, 1, 2, 0, 3, 0, 0, 4, 1),
) )
# fmt: on
class JapaneseContextAnalysis(object):
class JapaneseContextAnalysis:
NUM_OF_CATEGORY = 6 NUM_OF_CATEGORY = 6
DONT_KNOW = -1 DONT_KNOW = -1
ENOUGH_REL_THRESHOLD = 100 ENOUGH_REL_THRESHOLD = 100
MAX_REL_THRESHOLD = 1000 MAX_REL_THRESHOLD = 1000
MINIMUM_DATA_THRESHOLD = 4 MINIMUM_DATA_THRESHOLD = 4
def __init__(self): def __init__(self) -> None:
self._total_rel = None self._total_rel = 0
self._rel_sample = None self._rel_sample: List[int] = []
self._need_to_skip_char_num = None self._need_to_skip_char_num = 0
self._last_char_order = None self._last_char_order = -1
self._done = None self._done = False
self.reset() self.reset()
def reset(self): def reset(self) -> None:
self._total_rel = 0 # total sequence received self._total_rel = 0 # total sequence received
# category counters, each integer counts sequence in its category # category counters, each integer counts sequence in its category
self._rel_sample = [0] * self.NUM_OF_CATEGORY self._rel_sample = [0] * self.NUM_OF_CATEGORY
@@ -140,7 +144,7 @@ class JapaneseContextAnalysis(object):
# been made # been made
self._done = False self._done = False
def feed(self, byte_str, num_bytes): def feed(self, byte_str: Union[bytes, bytearray], num_bytes: int) -> None:
if self._done: if self._done:
return return
@@ -153,7 +157,7 @@ class JapaneseContextAnalysis(object):
# this character will simply our logic and improve performance. # this character will simply our logic and improve performance.
i = self._need_to_skip_char_num i = self._need_to_skip_char_num
while i < num_bytes: while i < num_bytes:
order, char_len = self.get_order(byte_str[i:i + 2]) order, char_len = self.get_order(byte_str[i : i + 2])
i += char_len i += char_len
if i > num_bytes: if i > num_bytes:
self._need_to_skip_char_num = i - num_bytes self._need_to_skip_char_num = i - num_bytes
@@ -164,32 +168,34 @@ class JapaneseContextAnalysis(object):
if self._total_rel > self.MAX_REL_THRESHOLD: if self._total_rel > self.MAX_REL_THRESHOLD:
self._done = True self._done = True
break break
self._rel_sample[jp2CharContext[self._last_char_order][order]] += 1 self._rel_sample[
jp2_char_context[self._last_char_order][order]
] += 1
self._last_char_order = order self._last_char_order = order
def got_enough_data(self): def got_enough_data(self) -> bool:
return self._total_rel > self.ENOUGH_REL_THRESHOLD return self._total_rel > self.ENOUGH_REL_THRESHOLD
def get_confidence(self): def get_confidence(self) -> float:
# This is just one way to calculate confidence. It works well for me. # This is just one way to calculate confidence. It works well for me.
if self._total_rel > self.MINIMUM_DATA_THRESHOLD: if self._total_rel > self.MINIMUM_DATA_THRESHOLD:
return (self._total_rel - self._rel_sample[0]) / self._total_rel return (self._total_rel - self._rel_sample[0]) / self._total_rel
else: return self.DONT_KNOW
return self.DONT_KNOW
def get_order(self, byte_str): def get_order(self, _: Union[bytes, bytearray]) -> Tuple[int, int]:
return -1, 1 return -1, 1
class SJISContextAnalysis(JapaneseContextAnalysis): class SJISContextAnalysis(JapaneseContextAnalysis):
def __init__(self): def __init__(self) -> None:
super(SJISContextAnalysis, self).__init__() super().__init__()
self._charset_name = "SHIFT_JIS" self._charset_name = "SHIFT_JIS"
@property @property
def charset_name(self): def charset_name(self) -> str:
return self._charset_name return self._charset_name
def get_order(self, byte_str): def get_order(self, byte_str: Union[bytes, bytearray]) -> Tuple[int, int]:
if not byte_str: if not byte_str:
return -1, 1 return -1, 1
# find out current char's byte length # find out current char's byte length
@@ -209,8 +215,9 @@ class SJISContextAnalysis(JapaneseContextAnalysis):
return -1, char_len return -1, char_len
class EUCJPContextAnalysis(JapaneseContextAnalysis): class EUCJPContextAnalysis(JapaneseContextAnalysis):
def get_order(self, byte_str): def get_order(self, byte_str: Union[bytes, bytearray]) -> Tuple[int, int]:
if not byte_str: if not byte_str:
return -1, 1 return -1, 1
# find out current char's byte length # find out current char's byte length
@@ -229,5 +236,3 @@ class EUCJPContextAnalysis(JapaneseContextAnalysis):
return second_char - 0xA1, char_len return second_char - 0xA1, char_len
return -1, char_len return -1, char_len
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+265 -268
View File
@@ -1,9 +1,5 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from chardet.sbcharsetprober import SingleByteCharSetModel from chardet.sbcharsetprober import SingleByteCharSetModel
# 3: Positive # 3: Positive
# 2: Likely # 2: Likely
# 1: Unlikely # 1: Unlikely
@@ -4115,269 +4111,270 @@ HEBREW_LANG_MODEL = {
# Character Mapping Table(s): # Character Mapping Table(s):
WINDOWS_1255_HEBREW_CHAR_TO_ORDER = { WINDOWS_1255_HEBREW_CHAR_TO_ORDER = {
0: 255, # '\x00' 0: 255, # '\x00'
1: 255, # '\x01' 1: 255, # '\x01'
2: 255, # '\x02' 2: 255, # '\x02'
3: 255, # '\x03' 3: 255, # '\x03'
4: 255, # '\x04' 4: 255, # '\x04'
5: 255, # '\x05' 5: 255, # '\x05'
6: 255, # '\x06' 6: 255, # '\x06'
7: 255, # '\x07' 7: 255, # '\x07'
8: 255, # '\x08' 8: 255, # '\x08'
9: 255, # '\t' 9: 255, # '\t'
10: 254, # '\n' 10: 254, # '\n'
11: 255, # '\x0b' 11: 255, # '\x0b'
12: 255, # '\x0c' 12: 255, # '\x0c'
13: 254, # '\r' 13: 254, # '\r'
14: 255, # '\x0e' 14: 255, # '\x0e'
15: 255, # '\x0f' 15: 255, # '\x0f'
16: 255, # '\x10' 16: 255, # '\x10'
17: 255, # '\x11' 17: 255, # '\x11'
18: 255, # '\x12' 18: 255, # '\x12'
19: 255, # '\x13' 19: 255, # '\x13'
20: 255, # '\x14' 20: 255, # '\x14'
21: 255, # '\x15' 21: 255, # '\x15'
22: 255, # '\x16' 22: 255, # '\x16'
23: 255, # '\x17' 23: 255, # '\x17'
24: 255, # '\x18' 24: 255, # '\x18'
25: 255, # '\x19' 25: 255, # '\x19'
26: 255, # '\x1a' 26: 255, # '\x1a'
27: 255, # '\x1b' 27: 255, # '\x1b'
28: 255, # '\x1c' 28: 255, # '\x1c'
29: 255, # '\x1d' 29: 255, # '\x1d'
30: 255, # '\x1e' 30: 255, # '\x1e'
31: 255, # '\x1f' 31: 255, # '\x1f'
32: 253, # ' ' 32: 253, # ' '
33: 253, # '!' 33: 253, # '!'
34: 253, # '"' 34: 253, # '"'
35: 253, # '#' 35: 253, # '#'
36: 253, # '$' 36: 253, # '$'
37: 253, # '%' 37: 253, # '%'
38: 253, # '&' 38: 253, # '&'
39: 253, # "'" 39: 253, # "'"
40: 253, # '(' 40: 253, # '('
41: 253, # ')' 41: 253, # ')'
42: 253, # '*' 42: 253, # '*'
43: 253, # '+' 43: 253, # '+'
44: 253, # ',' 44: 253, # ','
45: 253, # '-' 45: 253, # '-'
46: 253, # '.' 46: 253, # '.'
47: 253, # '/' 47: 253, # '/'
48: 252, # '0' 48: 252, # '0'
49: 252, # '1' 49: 252, # '1'
50: 252, # '2' 50: 252, # '2'
51: 252, # '3' 51: 252, # '3'
52: 252, # '4' 52: 252, # '4'
53: 252, # '5' 53: 252, # '5'
54: 252, # '6' 54: 252, # '6'
55: 252, # '7' 55: 252, # '7'
56: 252, # '8' 56: 252, # '8'
57: 252, # '9' 57: 252, # '9'
58: 253, # ':' 58: 253, # ':'
59: 253, # ';' 59: 253, # ';'
60: 253, # '<' 60: 253, # '<'
61: 253, # '=' 61: 253, # '='
62: 253, # '>' 62: 253, # '>'
63: 253, # '?' 63: 253, # '?'
64: 253, # '@' 64: 253, # '@'
65: 69, # 'A' 65: 69, # 'A'
66: 91, # 'B' 66: 91, # 'B'
67: 79, # 'C' 67: 79, # 'C'
68: 80, # 'D' 68: 80, # 'D'
69: 92, # 'E' 69: 92, # 'E'
70: 89, # 'F' 70: 89, # 'F'
71: 97, # 'G' 71: 97, # 'G'
72: 90, # 'H' 72: 90, # 'H'
73: 68, # 'I' 73: 68, # 'I'
74: 111, # 'J' 74: 111, # 'J'
75: 112, # 'K' 75: 112, # 'K'
76: 82, # 'L' 76: 82, # 'L'
77: 73, # 'M' 77: 73, # 'M'
78: 95, # 'N' 78: 95, # 'N'
79: 85, # 'O' 79: 85, # 'O'
80: 78, # 'P' 80: 78, # 'P'
81: 121, # 'Q' 81: 121, # 'Q'
82: 86, # 'R' 82: 86, # 'R'
83: 71, # 'S' 83: 71, # 'S'
84: 67, # 'T' 84: 67, # 'T'
85: 102, # 'U' 85: 102, # 'U'
86: 107, # 'V' 86: 107, # 'V'
87: 84, # 'W' 87: 84, # 'W'
88: 114, # 'X' 88: 114, # 'X'
89: 103, # 'Y' 89: 103, # 'Y'
90: 115, # 'Z' 90: 115, # 'Z'
91: 253, # '[' 91: 253, # '['
92: 253, # '\\' 92: 253, # '\\'
93: 253, # ']' 93: 253, # ']'
94: 253, # '^' 94: 253, # '^'
95: 253, # '_' 95: 253, # '_'
96: 253, # '`' 96: 253, # '`'
97: 50, # 'a' 97: 50, # 'a'
98: 74, # 'b' 98: 74, # 'b'
99: 60, # 'c' 99: 60, # 'c'
100: 61, # 'd' 100: 61, # 'd'
101: 42, # 'e' 101: 42, # 'e'
102: 76, # 'f' 102: 76, # 'f'
103: 70, # 'g' 103: 70, # 'g'
104: 64, # 'h' 104: 64, # 'h'
105: 53, # 'i' 105: 53, # 'i'
106: 105, # 'j' 106: 105, # 'j'
107: 93, # 'k' 107: 93, # 'k'
108: 56, # 'l' 108: 56, # 'l'
109: 65, # 'm' 109: 65, # 'm'
110: 54, # 'n' 110: 54, # 'n'
111: 49, # 'o' 111: 49, # 'o'
112: 66, # 'p' 112: 66, # 'p'
113: 110, # 'q' 113: 110, # 'q'
114: 51, # 'r' 114: 51, # 'r'
115: 43, # 's' 115: 43, # 's'
116: 44, # 't' 116: 44, # 't'
117: 63, # 'u' 117: 63, # 'u'
118: 81, # 'v' 118: 81, # 'v'
119: 77, # 'w' 119: 77, # 'w'
120: 98, # 'x' 120: 98, # 'x'
121: 75, # 'y' 121: 75, # 'y'
122: 108, # 'z' 122: 108, # 'z'
123: 253, # '{' 123: 253, # '{'
124: 253, # '|' 124: 253, # '|'
125: 253, # '}' 125: 253, # '}'
126: 253, # '~' 126: 253, # '~'
127: 253, # '\x7f' 127: 253, # '\x7f'
128: 124, # '€' 128: 124, # '€'
129: 202, # None 129: 202, # None
130: 203, # '' 130: 203, # ''
131: 204, # 'ƒ' 131: 204, # 'ƒ'
132: 205, # '„' 132: 205, # '„'
133: 40, # '…' 133: 40, # '…'
134: 58, # '†' 134: 58, # '†'
135: 206, # '‡' 135: 206, # '‡'
136: 207, # 'ˆ' 136: 207, # 'ˆ'
137: 208, # '‰' 137: 208, # '‰'
138: 209, # None 138: 209, # None
139: 210, # '' 139: 210, # ''
140: 211, # None 140: 211, # None
141: 212, # None 141: 212, # None
142: 213, # None 142: 213, # None
143: 214, # None 143: 214, # None
144: 215, # None 144: 215, # None
145: 83, # '' 145: 83, # ''
146: 52, # '' 146: 52, # ''
147: 47, # '“' 147: 47, # '“'
148: 46, # '”' 148: 46, # '”'
149: 72, # '•' 149: 72, # '•'
150: 32, # '' 150: 32, # ''
151: 94, # '—' 151: 94, # '—'
152: 216, # '˜' 152: 216, # '˜'
153: 113, # '™' 153: 113, # '™'
154: 217, # None 154: 217, # None
155: 109, # '' 155: 109, # ''
156: 218, # None 156: 218, # None
157: 219, # None 157: 219, # None
158: 220, # None 158: 220, # None
159: 221, # None 159: 221, # None
160: 34, # '\xa0' 160: 34, # '\xa0'
161: 116, # '¡' 161: 116, # '¡'
162: 222, # '¢' 162: 222, # '¢'
163: 118, # '£' 163: 118, # '£'
164: 100, # '₪' 164: 100, # '₪'
165: 223, # '¥' 165: 223, # '¥'
166: 224, # '¦' 166: 224, # '¦'
167: 117, # '§' 167: 117, # '§'
168: 119, # '¨' 168: 119, # '¨'
169: 104, # '©' 169: 104, # '©'
170: 125, # '×' 170: 125, # '×'
171: 225, # '«' 171: 225, # '«'
172: 226, # '¬' 172: 226, # '¬'
173: 87, # '\xad' 173: 87, # '\xad'
174: 99, # '®' 174: 99, # '®'
175: 227, # '¯' 175: 227, # '¯'
176: 106, # '°' 176: 106, # '°'
177: 122, # '±' 177: 122, # '±'
178: 123, # '²' 178: 123, # '²'
179: 228, # '³' 179: 228, # '³'
180: 55, # '´' 180: 55, # '´'
181: 229, # 'µ' 181: 229, # 'µ'
182: 230, # '¶' 182: 230, # '¶'
183: 101, # '·' 183: 101, # '·'
184: 231, # '¸' 184: 231, # '¸'
185: 232, # '¹' 185: 232, # '¹'
186: 120, # '÷' 186: 120, # '÷'
187: 233, # '»' 187: 233, # '»'
188: 48, # '¼' 188: 48, # '¼'
189: 39, # '½' 189: 39, # '½'
190: 57, # '¾' 190: 57, # '¾'
191: 234, # '¿' 191: 234, # '¿'
192: 30, # 'ְ' 192: 30, # 'ְ'
193: 59, # 'ֱ' 193: 59, # 'ֱ'
194: 41, # 'ֲ' 194: 41, # 'ֲ'
195: 88, # 'ֳ' 195: 88, # 'ֳ'
196: 33, # 'ִ' 196: 33, # 'ִ'
197: 37, # 'ֵ' 197: 37, # 'ֵ'
198: 36, # 'ֶ' 198: 36, # 'ֶ'
199: 31, # 'ַ' 199: 31, # 'ַ'
200: 29, # 'ָ' 200: 29, # 'ָ'
201: 35, # 'ֹ' 201: 35, # 'ֹ'
202: 235, # None 202: 235, # None
203: 62, # 'ֻ' 203: 62, # 'ֻ'
204: 28, # 'ּ' 204: 28, # 'ּ'
205: 236, # 'ֽ' 205: 236, # 'ֽ'
206: 126, # '־' 206: 126, # '־'
207: 237, # 'ֿ' 207: 237, # 'ֿ'
208: 238, # '׀' 208: 238, # '׀'
209: 38, # 'ׁ' 209: 38, # 'ׁ'
210: 45, # 'ׂ' 210: 45, # 'ׂ'
211: 239, # '׃' 211: 239, # '׃'
212: 240, # 'װ' 212: 240, # 'װ'
213: 241, # 'ױ' 213: 241, # 'ױ'
214: 242, # 'ײ' 214: 242, # 'ײ'
215: 243, # '׳' 215: 243, # '׳'
216: 127, # '״' 216: 127, # '״'
217: 244, # None 217: 244, # None
218: 245, # None 218: 245, # None
219: 246, # None 219: 246, # None
220: 247, # None 220: 247, # None
221: 248, # None 221: 248, # None
222: 249, # None 222: 249, # None
223: 250, # None 223: 250, # None
224: 9, # 'א' 224: 9, # 'א'
225: 8, # 'ב' 225: 8, # 'ב'
226: 20, # 'ג' 226: 20, # 'ג'
227: 16, # 'ד' 227: 16, # 'ד'
228: 3, # 'ה' 228: 3, # 'ה'
229: 2, # 'ו' 229: 2, # 'ו'
230: 24, # 'ז' 230: 24, # 'ז'
231: 14, # 'ח' 231: 14, # 'ח'
232: 22, # 'ט' 232: 22, # 'ט'
233: 1, # 'י' 233: 1, # 'י'
234: 25, # 'ך' 234: 25, # 'ך'
235: 15, # 'כ' 235: 15, # 'כ'
236: 4, # 'ל' 236: 4, # 'ל'
237: 11, # 'ם' 237: 11, # 'ם'
238: 6, # 'מ' 238: 6, # 'מ'
239: 23, # 'ן' 239: 23, # 'ן'
240: 12, # 'נ' 240: 12, # 'נ'
241: 19, # 'ס' 241: 19, # 'ס'
242: 13, # 'ע' 242: 13, # 'ע'
243: 26, # 'ף' 243: 26, # 'ף'
244: 18, # 'פ' 244: 18, # 'פ'
245: 27, # 'ץ' 245: 27, # 'ץ'
246: 21, # 'צ' 246: 21, # 'צ'
247: 17, # 'ק' 247: 17, # 'ק'
248: 7, # 'ר' 248: 7, # 'ר'
249: 10, # 'ש' 249: 10, # 'ש'
250: 5, # 'ת' 250: 5, # 'ת'
251: 251, # None 251: 251, # None
252: 252, # None 252: 252, # None
253: 128, # '\u200e' 253: 128, # '\u200e'
254: 96, # '\u200f' 254: 96, # '\u200f'
255: 253, # None 255: 253, # None
} }
WINDOWS_1255_HEBREW_MODEL = SingleByteCharSetModel(charset_name='windows-1255', WINDOWS_1255_HEBREW_MODEL = SingleByteCharSetModel(
language='Hebrew', charset_name="windows-1255",
char_to_order_map=WINDOWS_1255_HEBREW_CHAR_TO_ORDER, language="Hebrew",
language_model=HEBREW_LANG_MODEL, char_to_order_map=WINDOWS_1255_HEBREW_CHAR_TO_ORDER,
typical_positive_ratio=0.984004, language_model=HEBREW_LANG_MODEL,
keep_ascii_letters=False, typical_positive_ratio=0.984004,
alphabet='אבגדהוזחטיךכלםמןנסעףפץצקרשתװױײ') keep_ascii_letters=False,
alphabet="אבגדהוזחטיךכלםמןנסעףפץצקרשתװױײ",
)
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+265 -268
View File
@@ -1,9 +1,5 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from chardet.sbcharsetprober import SingleByteCharSetModel from chardet.sbcharsetprober import SingleByteCharSetModel
# 3: Positive # 3: Positive
# 2: Likely # 2: Likely
# 1: Unlikely # 1: Unlikely
@@ -4115,269 +4111,270 @@ THAI_LANG_MODEL = {
# Character Mapping Table(s): # Character Mapping Table(s):
TIS_620_THAI_CHAR_TO_ORDER = { TIS_620_THAI_CHAR_TO_ORDER = {
0: 255, # '\x00' 0: 255, # '\x00'
1: 255, # '\x01' 1: 255, # '\x01'
2: 255, # '\x02' 2: 255, # '\x02'
3: 255, # '\x03' 3: 255, # '\x03'
4: 255, # '\x04' 4: 255, # '\x04'
5: 255, # '\x05' 5: 255, # '\x05'
6: 255, # '\x06' 6: 255, # '\x06'
7: 255, # '\x07' 7: 255, # '\x07'
8: 255, # '\x08' 8: 255, # '\x08'
9: 255, # '\t' 9: 255, # '\t'
10: 254, # '\n' 10: 254, # '\n'
11: 255, # '\x0b' 11: 255, # '\x0b'
12: 255, # '\x0c' 12: 255, # '\x0c'
13: 254, # '\r' 13: 254, # '\r'
14: 255, # '\x0e' 14: 255, # '\x0e'
15: 255, # '\x0f' 15: 255, # '\x0f'
16: 255, # '\x10' 16: 255, # '\x10'
17: 255, # '\x11' 17: 255, # '\x11'
18: 255, # '\x12' 18: 255, # '\x12'
19: 255, # '\x13' 19: 255, # '\x13'
20: 255, # '\x14' 20: 255, # '\x14'
21: 255, # '\x15' 21: 255, # '\x15'
22: 255, # '\x16' 22: 255, # '\x16'
23: 255, # '\x17' 23: 255, # '\x17'
24: 255, # '\x18' 24: 255, # '\x18'
25: 255, # '\x19' 25: 255, # '\x19'
26: 255, # '\x1a' 26: 255, # '\x1a'
27: 255, # '\x1b' 27: 255, # '\x1b'
28: 255, # '\x1c' 28: 255, # '\x1c'
29: 255, # '\x1d' 29: 255, # '\x1d'
30: 255, # '\x1e' 30: 255, # '\x1e'
31: 255, # '\x1f' 31: 255, # '\x1f'
32: 253, # ' ' 32: 253, # ' '
33: 253, # '!' 33: 253, # '!'
34: 253, # '"' 34: 253, # '"'
35: 253, # '#' 35: 253, # '#'
36: 253, # '$' 36: 253, # '$'
37: 253, # '%' 37: 253, # '%'
38: 253, # '&' 38: 253, # '&'
39: 253, # "'" 39: 253, # "'"
40: 253, # '(' 40: 253, # '('
41: 253, # ')' 41: 253, # ')'
42: 253, # '*' 42: 253, # '*'
43: 253, # '+' 43: 253, # '+'
44: 253, # ',' 44: 253, # ','
45: 253, # '-' 45: 253, # '-'
46: 253, # '.' 46: 253, # '.'
47: 253, # '/' 47: 253, # '/'
48: 252, # '0' 48: 252, # '0'
49: 252, # '1' 49: 252, # '1'
50: 252, # '2' 50: 252, # '2'
51: 252, # '3' 51: 252, # '3'
52: 252, # '4' 52: 252, # '4'
53: 252, # '5' 53: 252, # '5'
54: 252, # '6' 54: 252, # '6'
55: 252, # '7' 55: 252, # '7'
56: 252, # '8' 56: 252, # '8'
57: 252, # '9' 57: 252, # '9'
58: 253, # ':' 58: 253, # ':'
59: 253, # ';' 59: 253, # ';'
60: 253, # '<' 60: 253, # '<'
61: 253, # '=' 61: 253, # '='
62: 253, # '>' 62: 253, # '>'
63: 253, # '?' 63: 253, # '?'
64: 253, # '@' 64: 253, # '@'
65: 182, # 'A' 65: 182, # 'A'
66: 106, # 'B' 66: 106, # 'B'
67: 107, # 'C' 67: 107, # 'C'
68: 100, # 'D' 68: 100, # 'D'
69: 183, # 'E' 69: 183, # 'E'
70: 184, # 'F' 70: 184, # 'F'
71: 185, # 'G' 71: 185, # 'G'
72: 101, # 'H' 72: 101, # 'H'
73: 94, # 'I' 73: 94, # 'I'
74: 186, # 'J' 74: 186, # 'J'
75: 187, # 'K' 75: 187, # 'K'
76: 108, # 'L' 76: 108, # 'L'
77: 109, # 'M' 77: 109, # 'M'
78: 110, # 'N' 78: 110, # 'N'
79: 111, # 'O' 79: 111, # 'O'
80: 188, # 'P' 80: 188, # 'P'
81: 189, # 'Q' 81: 189, # 'Q'
82: 190, # 'R' 82: 190, # 'R'
83: 89, # 'S' 83: 89, # 'S'
84: 95, # 'T' 84: 95, # 'T'
85: 112, # 'U' 85: 112, # 'U'
86: 113, # 'V' 86: 113, # 'V'
87: 191, # 'W' 87: 191, # 'W'
88: 192, # 'X' 88: 192, # 'X'
89: 193, # 'Y' 89: 193, # 'Y'
90: 194, # 'Z' 90: 194, # 'Z'
91: 253, # '[' 91: 253, # '['
92: 253, # '\\' 92: 253, # '\\'
93: 253, # ']' 93: 253, # ']'
94: 253, # '^' 94: 253, # '^'
95: 253, # '_' 95: 253, # '_'
96: 253, # '`' 96: 253, # '`'
97: 64, # 'a' 97: 64, # 'a'
98: 72, # 'b' 98: 72, # 'b'
99: 73, # 'c' 99: 73, # 'c'
100: 114, # 'd' 100: 114, # 'd'
101: 74, # 'e' 101: 74, # 'e'
102: 115, # 'f' 102: 115, # 'f'
103: 116, # 'g' 103: 116, # 'g'
104: 102, # 'h' 104: 102, # 'h'
105: 81, # 'i' 105: 81, # 'i'
106: 201, # 'j' 106: 201, # 'j'
107: 117, # 'k' 107: 117, # 'k'
108: 90, # 'l' 108: 90, # 'l'
109: 103, # 'm' 109: 103, # 'm'
110: 78, # 'n' 110: 78, # 'n'
111: 82, # 'o' 111: 82, # 'o'
112: 96, # 'p' 112: 96, # 'p'
113: 202, # 'q' 113: 202, # 'q'
114: 91, # 'r' 114: 91, # 'r'
115: 79, # 's' 115: 79, # 's'
116: 84, # 't' 116: 84, # 't'
117: 104, # 'u' 117: 104, # 'u'
118: 105, # 'v' 118: 105, # 'v'
119: 97, # 'w' 119: 97, # 'w'
120: 98, # 'x' 120: 98, # 'x'
121: 92, # 'y' 121: 92, # 'y'
122: 203, # 'z' 122: 203, # 'z'
123: 253, # '{' 123: 253, # '{'
124: 253, # '|' 124: 253, # '|'
125: 253, # '}' 125: 253, # '}'
126: 253, # '~' 126: 253, # '~'
127: 253, # '\x7f' 127: 253, # '\x7f'
128: 209, # '\x80' 128: 209, # '\x80'
129: 210, # '\x81' 129: 210, # '\x81'
130: 211, # '\x82' 130: 211, # '\x82'
131: 212, # '\x83' 131: 212, # '\x83'
132: 213, # '\x84' 132: 213, # '\x84'
133: 88, # '\x85' 133: 88, # '\x85'
134: 214, # '\x86' 134: 214, # '\x86'
135: 215, # '\x87' 135: 215, # '\x87'
136: 216, # '\x88' 136: 216, # '\x88'
137: 217, # '\x89' 137: 217, # '\x89'
138: 218, # '\x8a' 138: 218, # '\x8a'
139: 219, # '\x8b' 139: 219, # '\x8b'
140: 220, # '\x8c' 140: 220, # '\x8c'
141: 118, # '\x8d' 141: 118, # '\x8d'
142: 221, # '\x8e' 142: 221, # '\x8e'
143: 222, # '\x8f' 143: 222, # '\x8f'
144: 223, # '\x90' 144: 223, # '\x90'
145: 224, # '\x91' 145: 224, # '\x91'
146: 99, # '\x92' 146: 99, # '\x92'
147: 85, # '\x93' 147: 85, # '\x93'
148: 83, # '\x94' 148: 83, # '\x94'
149: 225, # '\x95' 149: 225, # '\x95'
150: 226, # '\x96' 150: 226, # '\x96'
151: 227, # '\x97' 151: 227, # '\x97'
152: 228, # '\x98' 152: 228, # '\x98'
153: 229, # '\x99' 153: 229, # '\x99'
154: 230, # '\x9a' 154: 230, # '\x9a'
155: 231, # '\x9b' 155: 231, # '\x9b'
156: 232, # '\x9c' 156: 232, # '\x9c'
157: 233, # '\x9d' 157: 233, # '\x9d'
158: 234, # '\x9e' 158: 234, # '\x9e'
159: 235, # '\x9f' 159: 235, # '\x9f'
160: 236, # None 160: 236, # None
161: 5, # 'ก' 161: 5, # 'ก'
162: 30, # 'ข' 162: 30, # 'ข'
163: 237, # 'ฃ' 163: 237, # 'ฃ'
164: 24, # 'ค' 164: 24, # 'ค'
165: 238, # 'ฅ' 165: 238, # 'ฅ'
166: 75, # 'ฆ' 166: 75, # 'ฆ'
167: 8, # 'ง' 167: 8, # 'ง'
168: 26, # 'จ' 168: 26, # 'จ'
169: 52, # 'ฉ' 169: 52, # 'ฉ'
170: 34, # 'ช' 170: 34, # 'ช'
171: 51, # 'ซ' 171: 51, # 'ซ'
172: 119, # 'ฌ' 172: 119, # 'ฌ'
173: 47, # 'ญ' 173: 47, # 'ญ'
174: 58, # 'ฎ' 174: 58, # 'ฎ'
175: 57, # 'ฏ' 175: 57, # 'ฏ'
176: 49, # 'ฐ' 176: 49, # 'ฐ'
177: 53, # 'ฑ' 177: 53, # 'ฑ'
178: 55, # 'ฒ' 178: 55, # 'ฒ'
179: 43, # 'ณ' 179: 43, # 'ณ'
180: 20, # 'ด' 180: 20, # 'ด'
181: 19, # 'ต' 181: 19, # 'ต'
182: 44, # 'ถ' 182: 44, # 'ถ'
183: 14, # 'ท' 183: 14, # 'ท'
184: 48, # 'ธ' 184: 48, # 'ธ'
185: 3, # 'น' 185: 3, # 'น'
186: 17, # 'บ' 186: 17, # 'บ'
187: 25, # 'ป' 187: 25, # 'ป'
188: 39, # 'ผ' 188: 39, # 'ผ'
189: 62, # 'ฝ' 189: 62, # 'ฝ'
190: 31, # 'พ' 190: 31, # 'พ'
191: 54, # 'ฟ' 191: 54, # 'ฟ'
192: 45, # 'ภ' 192: 45, # 'ภ'
193: 9, # 'ม' 193: 9, # 'ม'
194: 16, # 'ย' 194: 16, # 'ย'
195: 2, # 'ร' 195: 2, # 'ร'
196: 61, # 'ฤ' 196: 61, # 'ฤ'
197: 15, # 'ล' 197: 15, # 'ล'
198: 239, # 'ฦ' 198: 239, # 'ฦ'
199: 12, # 'ว' 199: 12, # 'ว'
200: 42, # 'ศ' 200: 42, # 'ศ'
201: 46, # 'ษ' 201: 46, # 'ษ'
202: 18, # 'ส' 202: 18, # 'ส'
203: 21, # 'ห' 203: 21, # 'ห'
204: 76, # 'ฬ' 204: 76, # 'ฬ'
205: 4, # 'อ' 205: 4, # 'อ'
206: 66, # 'ฮ' 206: 66, # 'ฮ'
207: 63, # 'ฯ' 207: 63, # 'ฯ'
208: 22, # 'ะ' 208: 22, # 'ะ'
209: 10, # 'ั' 209: 10, # 'ั'
210: 1, # 'า' 210: 1, # 'า'
211: 36, # 'ำ' 211: 36, # 'ำ'
212: 23, # 'ิ' 212: 23, # 'ิ'
213: 13, # 'ี' 213: 13, # 'ี'
214: 40, # 'ึ' 214: 40, # 'ึ'
215: 27, # 'ื' 215: 27, # 'ื'
216: 32, # 'ุ' 216: 32, # 'ุ'
217: 35, # 'ู' 217: 35, # 'ู'
218: 86, # 'ฺ' 218: 86, # 'ฺ'
219: 240, # None 219: 240, # None
220: 241, # None 220: 241, # None
221: 242, # None 221: 242, # None
222: 243, # None 222: 243, # None
223: 244, # '฿' 223: 244, # '฿'
224: 11, # 'เ' 224: 11, # 'เ'
225: 28, # 'แ' 225: 28, # 'แ'
226: 41, # 'โ' 226: 41, # 'โ'
227: 29, # 'ใ' 227: 29, # 'ใ'
228: 33, # 'ไ' 228: 33, # 'ไ'
229: 245, # 'ๅ' 229: 245, # 'ๅ'
230: 50, # 'ๆ' 230: 50, # 'ๆ'
231: 37, # '็' 231: 37, # '็'
232: 6, # '่' 232: 6, # '่'
233: 7, # '้' 233: 7, # '้'
234: 67, # '๊' 234: 67, # '๊'
235: 77, # '๋' 235: 77, # '๋'
236: 38, # '์' 236: 38, # '์'
237: 93, # 'ํ' 237: 93, # 'ํ'
238: 246, # '๎' 238: 246, # '๎'
239: 247, # '๏' 239: 247, # '๏'
240: 68, # '' 240: 68, # ''
241: 56, # '๑' 241: 56, # '๑'
242: 59, # '๒' 242: 59, # '๒'
243: 65, # '๓' 243: 65, # '๓'
244: 69, # '๔' 244: 69, # '๔'
245: 60, # '๕' 245: 60, # '๕'
246: 70, # '๖' 246: 70, # '๖'
247: 80, # '๗' 247: 80, # '๗'
248: 71, # '๘' 248: 71, # '๘'
249: 87, # '๙' 249: 87, # '๙'
250: 248, # '๚' 250: 248, # '๚'
251: 249, # '๛' 251: 249, # '๛'
252: 250, # None 252: 250, # None
253: 251, # None 253: 251, # None
254: 252, # None 254: 252, # None
255: 253, # None 255: 253, # None
} }
TIS_620_THAI_MODEL = SingleByteCharSetModel(charset_name='TIS-620', TIS_620_THAI_MODEL = SingleByteCharSetModel(
language='Thai', charset_name="TIS-620",
char_to_order_map=TIS_620_THAI_CHAR_TO_ORDER, language="Thai",
language_model=THAI_LANG_MODEL, char_to_order_map=TIS_620_THAI_CHAR_TO_ORDER,
typical_positive_ratio=0.926386, language_model=THAI_LANG_MODEL,
keep_ascii_letters=False, typical_positive_ratio=0.926386,
alphabet='กขฃคฅฆงจฉชซฌญฎฏฐฑฒณดตถทธนบปผฝพฟภมยรฤลฦวศษสหฬอฮฯะัาำิีึืฺุู฿เแโใไๅๆ็่้๊๋์ํ๎๏๐๑๒๓๔๕๖๗๘๙๚๛') keep_ascii_letters=False,
alphabet="กขฃคฅฆงจฉชซฌญฎฏฐฑฒณดตถทธนบปผฝพฟภมยรฤลฦวศษสหฬอฮฯะัาำิีึืฺุู฿เแโใไๅๆ็่้๊๋์ํ๎๏๐๑๒๓๔๕๖๗๘๙๚๛",
)
+265 -268
View File
@@ -1,9 +1,5 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from chardet.sbcharsetprober import SingleByteCharSetModel from chardet.sbcharsetprober import SingleByteCharSetModel
# 3: Positive # 3: Positive
# 2: Likely # 2: Likely
# 1: Unlikely # 1: Unlikely
@@ -4115,269 +4111,270 @@ TURKISH_LANG_MODEL = {
# Character Mapping Table(s): # Character Mapping Table(s):
ISO_8859_9_TURKISH_CHAR_TO_ORDER = { ISO_8859_9_TURKISH_CHAR_TO_ORDER = {
0: 255, # '\x00' 0: 255, # '\x00'
1: 255, # '\x01' 1: 255, # '\x01'
2: 255, # '\x02' 2: 255, # '\x02'
3: 255, # '\x03' 3: 255, # '\x03'
4: 255, # '\x04' 4: 255, # '\x04'
5: 255, # '\x05' 5: 255, # '\x05'
6: 255, # '\x06' 6: 255, # '\x06'
7: 255, # '\x07' 7: 255, # '\x07'
8: 255, # '\x08' 8: 255, # '\x08'
9: 255, # '\t' 9: 255, # '\t'
10: 255, # '\n' 10: 255, # '\n'
11: 255, # '\x0b' 11: 255, # '\x0b'
12: 255, # '\x0c' 12: 255, # '\x0c'
13: 255, # '\r' 13: 255, # '\r'
14: 255, # '\x0e' 14: 255, # '\x0e'
15: 255, # '\x0f' 15: 255, # '\x0f'
16: 255, # '\x10' 16: 255, # '\x10'
17: 255, # '\x11' 17: 255, # '\x11'
18: 255, # '\x12' 18: 255, # '\x12'
19: 255, # '\x13' 19: 255, # '\x13'
20: 255, # '\x14' 20: 255, # '\x14'
21: 255, # '\x15' 21: 255, # '\x15'
22: 255, # '\x16' 22: 255, # '\x16'
23: 255, # '\x17' 23: 255, # '\x17'
24: 255, # '\x18' 24: 255, # '\x18'
25: 255, # '\x19' 25: 255, # '\x19'
26: 255, # '\x1a' 26: 255, # '\x1a'
27: 255, # '\x1b' 27: 255, # '\x1b'
28: 255, # '\x1c' 28: 255, # '\x1c'
29: 255, # '\x1d' 29: 255, # '\x1d'
30: 255, # '\x1e' 30: 255, # '\x1e'
31: 255, # '\x1f' 31: 255, # '\x1f'
32: 255, # ' ' 32: 255, # ' '
33: 255, # '!' 33: 255, # '!'
34: 255, # '"' 34: 255, # '"'
35: 255, # '#' 35: 255, # '#'
36: 255, # '$' 36: 255, # '$'
37: 255, # '%' 37: 255, # '%'
38: 255, # '&' 38: 255, # '&'
39: 255, # "'" 39: 255, # "'"
40: 255, # '(' 40: 255, # '('
41: 255, # ')' 41: 255, # ')'
42: 255, # '*' 42: 255, # '*'
43: 255, # '+' 43: 255, # '+'
44: 255, # ',' 44: 255, # ','
45: 255, # '-' 45: 255, # '-'
46: 255, # '.' 46: 255, # '.'
47: 255, # '/' 47: 255, # '/'
48: 255, # '0' 48: 255, # '0'
49: 255, # '1' 49: 255, # '1'
50: 255, # '2' 50: 255, # '2'
51: 255, # '3' 51: 255, # '3'
52: 255, # '4' 52: 255, # '4'
53: 255, # '5' 53: 255, # '5'
54: 255, # '6' 54: 255, # '6'
55: 255, # '7' 55: 255, # '7'
56: 255, # '8' 56: 255, # '8'
57: 255, # '9' 57: 255, # '9'
58: 255, # ':' 58: 255, # ':'
59: 255, # ';' 59: 255, # ';'
60: 255, # '<' 60: 255, # '<'
61: 255, # '=' 61: 255, # '='
62: 255, # '>' 62: 255, # '>'
63: 255, # '?' 63: 255, # '?'
64: 255, # '@' 64: 255, # '@'
65: 23, # 'A' 65: 23, # 'A'
66: 37, # 'B' 66: 37, # 'B'
67: 47, # 'C' 67: 47, # 'C'
68: 39, # 'D' 68: 39, # 'D'
69: 29, # 'E' 69: 29, # 'E'
70: 52, # 'F' 70: 52, # 'F'
71: 36, # 'G' 71: 36, # 'G'
72: 45, # 'H' 72: 45, # 'H'
73: 53, # 'I' 73: 53, # 'I'
74: 60, # 'J' 74: 60, # 'J'
75: 16, # 'K' 75: 16, # 'K'
76: 49, # 'L' 76: 49, # 'L'
77: 20, # 'M' 77: 20, # 'M'
78: 46, # 'N' 78: 46, # 'N'
79: 42, # 'O' 79: 42, # 'O'
80: 48, # 'P' 80: 48, # 'P'
81: 69, # 'Q' 81: 69, # 'Q'
82: 44, # 'R' 82: 44, # 'R'
83: 35, # 'S' 83: 35, # 'S'
84: 31, # 'T' 84: 31, # 'T'
85: 51, # 'U' 85: 51, # 'U'
86: 38, # 'V' 86: 38, # 'V'
87: 62, # 'W' 87: 62, # 'W'
88: 65, # 'X' 88: 65, # 'X'
89: 43, # 'Y' 89: 43, # 'Y'
90: 56, # 'Z' 90: 56, # 'Z'
91: 255, # '[' 91: 255, # '['
92: 255, # '\\' 92: 255, # '\\'
93: 255, # ']' 93: 255, # ']'
94: 255, # '^' 94: 255, # '^'
95: 255, # '_' 95: 255, # '_'
96: 255, # '`' 96: 255, # '`'
97: 1, # 'a' 97: 1, # 'a'
98: 21, # 'b' 98: 21, # 'b'
99: 28, # 'c' 99: 28, # 'c'
100: 12, # 'd' 100: 12, # 'd'
101: 2, # 'e' 101: 2, # 'e'
102: 18, # 'f' 102: 18, # 'f'
103: 27, # 'g' 103: 27, # 'g'
104: 25, # 'h' 104: 25, # 'h'
105: 3, # 'i' 105: 3, # 'i'
106: 24, # 'j' 106: 24, # 'j'
107: 10, # 'k' 107: 10, # 'k'
108: 5, # 'l' 108: 5, # 'l'
109: 13, # 'm' 109: 13, # 'm'
110: 4, # 'n' 110: 4, # 'n'
111: 15, # 'o' 111: 15, # 'o'
112: 26, # 'p' 112: 26, # 'p'
113: 64, # 'q' 113: 64, # 'q'
114: 7, # 'r' 114: 7, # 'r'
115: 8, # 's' 115: 8, # 's'
116: 9, # 't' 116: 9, # 't'
117: 14, # 'u' 117: 14, # 'u'
118: 32, # 'v' 118: 32, # 'v'
119: 57, # 'w' 119: 57, # 'w'
120: 58, # 'x' 120: 58, # 'x'
121: 11, # 'y' 121: 11, # 'y'
122: 22, # 'z' 122: 22, # 'z'
123: 255, # '{' 123: 255, # '{'
124: 255, # '|' 124: 255, # '|'
125: 255, # '}' 125: 255, # '}'
126: 255, # '~' 126: 255, # '~'
127: 255, # '\x7f' 127: 255, # '\x7f'
128: 180, # '\x80' 128: 180, # '\x80'
129: 179, # '\x81' 129: 179, # '\x81'
130: 178, # '\x82' 130: 178, # '\x82'
131: 177, # '\x83' 131: 177, # '\x83'
132: 176, # '\x84' 132: 176, # '\x84'
133: 175, # '\x85' 133: 175, # '\x85'
134: 174, # '\x86' 134: 174, # '\x86'
135: 173, # '\x87' 135: 173, # '\x87'
136: 172, # '\x88' 136: 172, # '\x88'
137: 171, # '\x89' 137: 171, # '\x89'
138: 170, # '\x8a' 138: 170, # '\x8a'
139: 169, # '\x8b' 139: 169, # '\x8b'
140: 168, # '\x8c' 140: 168, # '\x8c'
141: 167, # '\x8d' 141: 167, # '\x8d'
142: 166, # '\x8e' 142: 166, # '\x8e'
143: 165, # '\x8f' 143: 165, # '\x8f'
144: 164, # '\x90' 144: 164, # '\x90'
145: 163, # '\x91' 145: 163, # '\x91'
146: 162, # '\x92' 146: 162, # '\x92'
147: 161, # '\x93' 147: 161, # '\x93'
148: 160, # '\x94' 148: 160, # '\x94'
149: 159, # '\x95' 149: 159, # '\x95'
150: 101, # '\x96' 150: 101, # '\x96'
151: 158, # '\x97' 151: 158, # '\x97'
152: 157, # '\x98' 152: 157, # '\x98'
153: 156, # '\x99' 153: 156, # '\x99'
154: 155, # '\x9a' 154: 155, # '\x9a'
155: 154, # '\x9b' 155: 154, # '\x9b'
156: 153, # '\x9c' 156: 153, # '\x9c'
157: 152, # '\x9d' 157: 152, # '\x9d'
158: 151, # '\x9e' 158: 151, # '\x9e'
159: 106, # '\x9f' 159: 106, # '\x9f'
160: 150, # '\xa0' 160: 150, # '\xa0'
161: 149, # '¡' 161: 149, # '¡'
162: 148, # '¢' 162: 148, # '¢'
163: 147, # '£' 163: 147, # '£'
164: 146, # '¤' 164: 146, # '¤'
165: 145, # '¥' 165: 145, # '¥'
166: 144, # '¦' 166: 144, # '¦'
167: 100, # '§' 167: 100, # '§'
168: 143, # '¨' 168: 143, # '¨'
169: 142, # '©' 169: 142, # '©'
170: 141, # 'ª' 170: 141, # 'ª'
171: 140, # '«' 171: 140, # '«'
172: 139, # '¬' 172: 139, # '¬'
173: 138, # '\xad' 173: 138, # '\xad'
174: 137, # '®' 174: 137, # '®'
175: 136, # '¯' 175: 136, # '¯'
176: 94, # '°' 176: 94, # '°'
177: 80, # '±' 177: 80, # '±'
178: 93, # '²' 178: 93, # '²'
179: 135, # '³' 179: 135, # '³'
180: 105, # '´' 180: 105, # '´'
181: 134, # 'µ' 181: 134, # 'µ'
182: 133, # '¶' 182: 133, # '¶'
183: 63, # '·' 183: 63, # '·'
184: 132, # '¸' 184: 132, # '¸'
185: 131, # '¹' 185: 131, # '¹'
186: 130, # 'º' 186: 130, # 'º'
187: 129, # '»' 187: 129, # '»'
188: 128, # '¼' 188: 128, # '¼'
189: 127, # '½' 189: 127, # '½'
190: 126, # '¾' 190: 126, # '¾'
191: 125, # '¿' 191: 125, # '¿'
192: 124, # 'À' 192: 124, # 'À'
193: 104, # 'Á' 193: 104, # 'Á'
194: 73, # 'Â' 194: 73, # 'Â'
195: 99, # 'Ã' 195: 99, # 'Ã'
196: 79, # 'Ä' 196: 79, # 'Ä'
197: 85, # 'Å' 197: 85, # 'Å'
198: 123, # 'Æ' 198: 123, # 'Æ'
199: 54, # 'Ç' 199: 54, # 'Ç'
200: 122, # 'È' 200: 122, # 'È'
201: 98, # 'É' 201: 98, # 'É'
202: 92, # 'Ê' 202: 92, # 'Ê'
203: 121, # 'Ë' 203: 121, # 'Ë'
204: 120, # 'Ì' 204: 120, # 'Ì'
205: 91, # 'Í' 205: 91, # 'Í'
206: 103, # 'Î' 206: 103, # 'Î'
207: 119, # 'Ï' 207: 119, # 'Ï'
208: 68, # 'Ğ' 208: 68, # 'Ğ'
209: 118, # 'Ñ' 209: 118, # 'Ñ'
210: 117, # 'Ò' 210: 117, # 'Ò'
211: 97, # 'Ó' 211: 97, # 'Ó'
212: 116, # 'Ô' 212: 116, # 'Ô'
213: 115, # 'Õ' 213: 115, # 'Õ'
214: 50, # 'Ö' 214: 50, # 'Ö'
215: 90, # '×' 215: 90, # '×'
216: 114, # 'Ø' 216: 114, # 'Ø'
217: 113, # 'Ù' 217: 113, # 'Ù'
218: 112, # 'Ú' 218: 112, # 'Ú'
219: 111, # 'Û' 219: 111, # 'Û'
220: 55, # 'Ü' 220: 55, # 'Ü'
221: 41, # 'İ' 221: 41, # 'İ'
222: 40, # 'Ş' 222: 40, # 'Ş'
223: 86, # 'ß' 223: 86, # 'ß'
224: 89, # 'à' 224: 89, # 'à'
225: 70, # 'á' 225: 70, # 'á'
226: 59, # 'â' 226: 59, # 'â'
227: 78, # 'ã' 227: 78, # 'ã'
228: 71, # 'ä' 228: 71, # 'ä'
229: 82, # 'å' 229: 82, # 'å'
230: 88, # 'æ' 230: 88, # 'æ'
231: 33, # 'ç' 231: 33, # 'ç'
232: 77, # 'è' 232: 77, # 'è'
233: 66, # 'é' 233: 66, # 'é'
234: 84, # 'ê' 234: 84, # 'ê'
235: 83, # 'ë' 235: 83, # 'ë'
236: 110, # 'ì' 236: 110, # 'ì'
237: 75, # 'í' 237: 75, # 'í'
238: 61, # 'î' 238: 61, # 'î'
239: 96, # 'ï' 239: 96, # 'ï'
240: 30, # 'ğ' 240: 30, # 'ğ'
241: 67, # 'ñ' 241: 67, # 'ñ'
242: 109, # 'ò' 242: 109, # 'ò'
243: 74, # 'ó' 243: 74, # 'ó'
244: 87, # 'ô' 244: 87, # 'ô'
245: 102, # 'õ' 245: 102, # 'õ'
246: 34, # 'ö' 246: 34, # 'ö'
247: 95, # '÷' 247: 95, # '÷'
248: 81, # 'ø' 248: 81, # 'ø'
249: 108, # 'ù' 249: 108, # 'ù'
250: 76, # 'ú' 250: 76, # 'ú'
251: 72, # 'û' 251: 72, # 'û'
252: 17, # 'ü' 252: 17, # 'ü'
253: 6, # 'ı' 253: 6, # 'ı'
254: 19, # 'ş' 254: 19, # 'ş'
255: 107, # 'ÿ' 255: 107, # 'ÿ'
} }
ISO_8859_9_TURKISH_MODEL = SingleByteCharSetModel(charset_name='ISO-8859-9', ISO_8859_9_TURKISH_MODEL = SingleByteCharSetModel(
language='Turkish', charset_name="ISO-8859-9",
char_to_order_map=ISO_8859_9_TURKISH_CHAR_TO_ORDER, language="Turkish",
language_model=TURKISH_LANG_MODEL, char_to_order_map=ISO_8859_9_TURKISH_CHAR_TO_ORDER,
typical_positive_ratio=0.97029, language_model=TURKISH_LANG_MODEL,
keep_ascii_letters=True, typical_positive_ratio=0.97029,
alphabet='ABCDEFGHIJKLMNOPRSTUVYZabcdefghijklmnoprstuvyzÂÇÎÖÛÜâçîöûüĞğİıŞş') keep_ascii_letters=True,
alphabet="ABCDEFGHIJKLMNOPRSTUVYZabcdefghijklmnoprstuvyzÂÇÎÖÛÜâçîöûüĞğİıŞş",
)
+23 -21
View File
@@ -26,6 +26,8 @@
# 02110-1301 USA # 02110-1301 USA
######################### END LICENSE BLOCK ######################### ######################### END LICENSE BLOCK #########################
from typing import List, Union
from .charsetprober import CharSetProber from .charsetprober import CharSetProber
from .enums import ProbingState from .enums import ProbingState
@@ -41,6 +43,7 @@ ASV = 6 # accent small vowel
ASO = 7 # accent small other ASO = 7 # accent small other
CLASS_NUM = 8 # total classes CLASS_NUM = 8 # total classes
# fmt: off
Latin1_CharToClass = ( Latin1_CharToClass = (
OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 00 - 07 OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 00 - 07
OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 08 - 0F OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 08 - 0F
@@ -91,34 +94,34 @@ Latin1ClassModel = (
0, 3, 1, 3, 1, 1, 1, 3, # ASV 0, 3, 1, 3, 1, 1, 1, 3, # ASV
0, 3, 1, 3, 1, 1, 3, 3, # ASO 0, 3, 1, 3, 1, 1, 3, 3, # ASO
) )
# fmt: on
class Latin1Prober(CharSetProber): class Latin1Prober(CharSetProber):
def __init__(self): def __init__(self) -> None:
super(Latin1Prober, self).__init__() super().__init__()
self._last_char_class = None self._last_char_class = OTH
self._freq_counter = None self._freq_counter: List[int] = []
self.reset() self.reset()
def reset(self): def reset(self) -> None:
self._last_char_class = OTH self._last_char_class = OTH
self._freq_counter = [0] * FREQ_CAT_NUM self._freq_counter = [0] * FREQ_CAT_NUM
CharSetProber.reset(self) super().reset()
@property @property
def charset_name(self): def charset_name(self) -> str:
return "ISO-8859-1" return "ISO-8859-1"
@property @property
def language(self): def language(self) -> str:
return "" return ""
def feed(self, byte_str): def feed(self, byte_str: Union[bytes, bytearray]) -> ProbingState:
byte_str = self.filter_with_english_letters(byte_str) byte_str = self.remove_xml_tags(byte_str)
for c in byte_str: for c in byte_str:
char_class = Latin1_CharToClass[c] char_class = Latin1_CharToClass[c]
freq = Latin1ClassModel[(self._last_char_class * CLASS_NUM) freq = Latin1ClassModel[(self._last_char_class * CLASS_NUM) + char_class]
+ char_class]
if freq == 0: if freq == 0:
self._state = ProbingState.NOT_ME self._state = ProbingState.NOT_ME
break break
@@ -127,19 +130,18 @@ class Latin1Prober(CharSetProber):
return self.state return self.state
def get_confidence(self): def get_confidence(self) -> float:
if self.state == ProbingState.NOT_ME: if self.state == ProbingState.NOT_ME:
return 0.01 return 0.01
total = sum(self._freq_counter) total = sum(self._freq_counter)
if total < 0.01: confidence = (
confidence = 0.0 0.0
else: if total < 0.01
confidence = ((self._freq_counter[3] - self._freq_counter[1] * 20.0) else (self._freq_counter[3] - self._freq_counter[1] * 20.0) / total
/ total) )
if confidence < 0.0: confidence = max(confidence, 0.0)
confidence = 0.0
# lower the confidence of latin1 so that other more accurate # lower the confidence of latin1 so that other more accurate
# detector can take priority. # detector can take priority.
confidence = confidence * 0.73 confidence *= 0.73
return confidence return confidence
+162
View File
@@ -0,0 +1,162 @@
######################## BEGIN LICENSE BLOCK ########################
# This code was modified from latin1prober.py by Rob Speer <rob@lumino.so>.
# The Original Code is Mozilla Universal charset detector code.
#
# The Initial Developer of the Original Code is
# Netscape Communications Corporation.
# Portions created by the Initial Developer are Copyright (C) 2001
# the Initial Developer. All Rights Reserved.
#
# Contributor(s):
# Rob Speer - adapt to MacRoman encoding
# Mark Pilgrim - port to Python
# Shy Shalom - original C code
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
# 02110-1301 USA
######################### END LICENSE BLOCK #########################
from typing import List, Union
from .charsetprober import CharSetProber
from .enums import ProbingState
FREQ_CAT_NUM = 4
UDF = 0 # undefined
OTH = 1 # other
ASC = 2 # ascii capital letter
ASS = 3 # ascii small letter
ACV = 4 # accent capital vowel
ACO = 5 # accent capital other
ASV = 6 # accent small vowel
ASO = 7 # accent small other
ODD = 8 # character that is unlikely to appear
CLASS_NUM = 9 # total classes
# The change from Latin1 is that we explicitly look for extended characters
# that are infrequently-occurring symbols, and consider them to always be
# improbable. This should let MacRoman get out of the way of more likely
# encodings in most situations.
# fmt: off
MacRoman_CharToClass = (
OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 00 - 07
OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 08 - 0F
OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 10 - 17
OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 18 - 1F
OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 20 - 27
OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 28 - 2F
OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 30 - 37
OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 38 - 3F
OTH, ASC, ASC, ASC, ASC, ASC, ASC, ASC, # 40 - 47
ASC, ASC, ASC, ASC, ASC, ASC, ASC, ASC, # 48 - 4F
ASC, ASC, ASC, ASC, ASC, ASC, ASC, ASC, # 50 - 57
ASC, ASC, ASC, OTH, OTH, OTH, OTH, OTH, # 58 - 5F
OTH, ASS, ASS, ASS, ASS, ASS, ASS, ASS, # 60 - 67
ASS, ASS, ASS, ASS, ASS, ASS, ASS, ASS, # 68 - 6F
ASS, ASS, ASS, ASS, ASS, ASS, ASS, ASS, # 70 - 77
ASS, ASS, ASS, OTH, OTH, OTH, OTH, OTH, # 78 - 7F
ACV, ACV, ACO, ACV, ACO, ACV, ACV, ASV, # 80 - 87
ASV, ASV, ASV, ASV, ASV, ASO, ASV, ASV, # 88 - 8F
ASV, ASV, ASV, ASV, ASV, ASV, ASO, ASV, # 90 - 97
ASV, ASV, ASV, ASV, ASV, ASV, ASV, ASV, # 98 - 9F
OTH, OTH, OTH, OTH, OTH, OTH, OTH, ASO, # A0 - A7
OTH, OTH, ODD, ODD, OTH, OTH, ACV, ACV, # A8 - AF
OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # B0 - B7
OTH, OTH, OTH, OTH, OTH, OTH, ASV, ASV, # B8 - BF
OTH, OTH, ODD, OTH, ODD, OTH, OTH, OTH, # C0 - C7
OTH, OTH, OTH, ACV, ACV, ACV, ACV, ASV, # C8 - CF
OTH, OTH, OTH, OTH, OTH, OTH, OTH, ODD, # D0 - D7
ASV, ACV, ODD, OTH, OTH, OTH, OTH, OTH, # D8 - DF
OTH, OTH, OTH, OTH, OTH, ACV, ACV, ACV, # E0 - E7
ACV, ACV, ACV, ACV, ACV, ACV, ACV, ACV, # E8 - EF
ODD, ACV, ACV, ACV, ACV, ASV, ODD, ODD, # F0 - F7
ODD, ODD, ODD, ODD, ODD, ODD, ODD, ODD, # F8 - FF
)
# 0 : illegal
# 1 : very unlikely
# 2 : normal
# 3 : very likely
MacRomanClassModel = (
# UDF OTH ASC ASS ACV ACO ASV ASO ODD
0, 0, 0, 0, 0, 0, 0, 0, 0, # UDF
0, 3, 3, 3, 3, 3, 3, 3, 1, # OTH
0, 3, 3, 3, 3, 3, 3, 3, 1, # ASC
0, 3, 3, 3, 1, 1, 3, 3, 1, # ASS
0, 3, 3, 3, 1, 2, 1, 2, 1, # ACV
0, 3, 3, 3, 3, 3, 3, 3, 1, # ACO
0, 3, 1, 3, 1, 1, 1, 3, 1, # ASV
0, 3, 1, 3, 1, 1, 3, 3, 1, # ASO
0, 1, 1, 1, 1, 1, 1, 1, 1, # ODD
)
# fmt: on
class MacRomanProber(CharSetProber):
def __init__(self) -> None:
super().__init__()
self._last_char_class = OTH
self._freq_counter: List[int] = []
self.reset()
def reset(self) -> None:
self._last_char_class = OTH
self._freq_counter = [0] * FREQ_CAT_NUM
# express the prior that MacRoman is a somewhat rare encoding;
# this can be done by starting out in a slightly improbable state
# that must be overcome
self._freq_counter[2] = 10
super().reset()
@property
def charset_name(self) -> str:
return "MacRoman"
@property
def language(self) -> str:
return ""
def feed(self, byte_str: Union[bytes, bytearray]) -> ProbingState:
byte_str = self.remove_xml_tags(byte_str)
for c in byte_str:
char_class = MacRoman_CharToClass[c]
freq = MacRomanClassModel[(self._last_char_class * CLASS_NUM) + char_class]
if freq == 0:
self._state = ProbingState.NOT_ME
break
self._freq_counter[freq] += 1
self._last_char_class = char_class
return self.state
def get_confidence(self) -> float:
if self.state == ProbingState.NOT_ME:
return 0.01
total = sum(self._freq_counter)
confidence = (
0.0
if total < 0.01
else (self._freq_counter[3] - self._freq_counter[1] * 20.0) / total
)
confidence = max(confidence, 0.0)
# lower the confidence of MacRoman so that other more accurate
# detector can take priority.
confidence *= 0.73
return confidence
+33 -29
View File
@@ -27,8 +27,12 @@
# 02110-1301 USA # 02110-1301 USA
######################### END LICENSE BLOCK ######################### ######################### END LICENSE BLOCK #########################
from typing import Optional, Union
from .chardistribution import CharDistributionAnalysis
from .charsetprober import CharSetProber from .charsetprober import CharSetProber
from .enums import ProbingState, MachineState from .codingstatemachine import CodingStateMachine
from .enums import LanguageFilter, MachineState, ProbingState
class MultiByteCharSetProber(CharSetProber): class MultiByteCharSetProber(CharSetProber):
@@ -36,56 +40,56 @@ class MultiByteCharSetProber(CharSetProber):
MultiByteCharSetProber MultiByteCharSetProber
""" """
def __init__(self, lang_filter=None): def __init__(self, lang_filter: LanguageFilter = LanguageFilter.NONE) -> None:
super(MultiByteCharSetProber, self).__init__(lang_filter=lang_filter) super().__init__(lang_filter=lang_filter)
self.distribution_analyzer = None self.distribution_analyzer: Optional[CharDistributionAnalysis] = None
self.coding_sm = None self.coding_sm: Optional[CodingStateMachine] = None
self._last_char = [0, 0] self._last_char = bytearray(b"\0\0")
def reset(self): def reset(self) -> None:
super(MultiByteCharSetProber, self).reset() super().reset()
if self.coding_sm: if self.coding_sm:
self.coding_sm.reset() self.coding_sm.reset()
if self.distribution_analyzer: if self.distribution_analyzer:
self.distribution_analyzer.reset() self.distribution_analyzer.reset()
self._last_char = [0, 0] self._last_char = bytearray(b"\0\0")
@property def feed(self, byte_str: Union[bytes, bytearray]) -> ProbingState:
def charset_name(self): assert self.coding_sm is not None
raise NotImplementedError assert self.distribution_analyzer is not None
@property for i, byte in enumerate(byte_str):
def language(self): coding_state = self.coding_sm.next_state(byte)
raise NotImplementedError
def feed(self, byte_str):
for i in range(len(byte_str)):
coding_state = self.coding_sm.next_state(byte_str[i])
if coding_state == MachineState.ERROR: if coding_state == MachineState.ERROR:
self.logger.debug('%s %s prober hit error at byte %s', self.logger.debug(
self.charset_name, self.language, i) "%s %s prober hit error at byte %s",
self.charset_name,
self.language,
i,
)
self._state = ProbingState.NOT_ME self._state = ProbingState.NOT_ME
break break
elif coding_state == MachineState.ITS_ME: if coding_state == MachineState.ITS_ME:
self._state = ProbingState.FOUND_IT self._state = ProbingState.FOUND_IT
break break
elif coding_state == MachineState.START: if coding_state == MachineState.START:
char_len = self.coding_sm.get_current_charlen() char_len = self.coding_sm.get_current_charlen()
if i == 0: if i == 0:
self._last_char[1] = byte_str[0] self._last_char[1] = byte
self.distribution_analyzer.feed(self._last_char, char_len) self.distribution_analyzer.feed(self._last_char, char_len)
else: else:
self.distribution_analyzer.feed(byte_str[i - 1:i + 1], self.distribution_analyzer.feed(byte_str[i - 1 : i + 1], char_len)
char_len)
self._last_char[0] = byte_str[-1] self._last_char[0] = byte_str[-1]
if self.state == ProbingState.DETECTING: if self.state == ProbingState.DETECTING:
if (self.distribution_analyzer.got_enough_data() and if self.distribution_analyzer.got_enough_data() and (
(self.get_confidence() > self.SHORTCUT_THRESHOLD)): self.get_confidence() > self.SHORTCUT_THRESHOLD
):
self._state = ProbingState.FOUND_IT self._state = ProbingState.FOUND_IT
return self.state return self.state
def get_confidence(self): def get_confidence(self) -> float:
assert self.distribution_analyzer is not None
return self.distribution_analyzer.get_confidence() return self.distribution_analyzer.get_confidence()
+13 -10
View File
@@ -27,20 +27,22 @@
# 02110-1301 USA # 02110-1301 USA
######################### END LICENSE BLOCK ######################### ######################### END LICENSE BLOCK #########################
from .charsetgroupprober import CharSetGroupProber
from .utf8prober import UTF8Prober
from .sjisprober import SJISProber
from .eucjpprober import EUCJPProber
from .gb2312prober import GB2312Prober
from .euckrprober import EUCKRProber
from .cp949prober import CP949Prober
from .big5prober import Big5Prober from .big5prober import Big5Prober
from .charsetgroupprober import CharSetGroupProber
from .cp949prober import CP949Prober
from .enums import LanguageFilter
from .eucjpprober import EUCJPProber
from .euckrprober import EUCKRProber
from .euctwprober import EUCTWProber from .euctwprober import EUCTWProber
from .gb2312prober import GB2312Prober
from .johabprober import JOHABProber
from .sjisprober import SJISProber
from .utf8prober import UTF8Prober
class MBCSGroupProber(CharSetGroupProber): class MBCSGroupProber(CharSetGroupProber):
def __init__(self, lang_filter=None): def __init__(self, lang_filter: LanguageFilter = LanguageFilter.NONE) -> None:
super(MBCSGroupProber, self).__init__(lang_filter=lang_filter) super().__init__(lang_filter=lang_filter)
self.probers = [ self.probers = [
UTF8Prober(), UTF8Prober(),
SJISProber(), SJISProber(),
@@ -49,6 +51,7 @@ class MBCSGroupProber(CharSetGroupProber):
EUCKRProber(), EUCKRProber(),
CP949Prober(), CP949Prober(),
Big5Prober(), Big5Prober(),
EUCTWProber() EUCTWProber(),
JOHABProber(),
] ]
self.reset() self.reset()
+453 -364
View File
@@ -25,43 +25,45 @@
# 02110-1301 USA # 02110-1301 USA
######################### END LICENSE BLOCK ######################### ######################### END LICENSE BLOCK #########################
from .codingstatemachinedict import CodingStateMachineDict
from .enums import MachineState from .enums import MachineState
# BIG5 # BIG5
# fmt: off
BIG5_CLS = ( BIG5_CLS = (
1,1,1,1,1,1,1,1, # 00 - 07 #allow 0x00 as legal value 1, 1, 1, 1, 1, 1, 1, 1, # 00 - 07 #allow 0x00 as legal value
1,1,1,1,1,1,0,0, # 08 - 0f 1, 1, 1, 1, 1, 1, 0, 0, # 08 - 0f
1,1,1,1,1,1,1,1, # 10 - 17 1, 1, 1, 1, 1, 1, 1, 1, # 10 - 17
1,1,1,0,1,1,1,1, # 18 - 1f 1, 1, 1, 0, 1, 1, 1, 1, # 18 - 1f
1,1,1,1,1,1,1,1, # 20 - 27 1, 1, 1, 1, 1, 1, 1, 1, # 20 - 27
1,1,1,1,1,1,1,1, # 28 - 2f 1, 1, 1, 1, 1, 1, 1, 1, # 28 - 2f
1,1,1,1,1,1,1,1, # 30 - 37 1, 1, 1, 1, 1, 1, 1, 1, # 30 - 37
1,1,1,1,1,1,1,1, # 38 - 3f 1, 1, 1, 1, 1, 1, 1, 1, # 38 - 3f
2,2,2,2,2,2,2,2, # 40 - 47 2, 2, 2, 2, 2, 2, 2, 2, # 40 - 47
2,2,2,2,2,2,2,2, # 48 - 4f 2, 2, 2, 2, 2, 2, 2, 2, # 48 - 4f
2,2,2,2,2,2,2,2, # 50 - 57 2, 2, 2, 2, 2, 2, 2, 2, # 50 - 57
2,2,2,2,2,2,2,2, # 58 - 5f 2, 2, 2, 2, 2, 2, 2, 2, # 58 - 5f
2,2,2,2,2,2,2,2, # 60 - 67 2, 2, 2, 2, 2, 2, 2, 2, # 60 - 67
2,2,2,2,2,2,2,2, # 68 - 6f 2, 2, 2, 2, 2, 2, 2, 2, # 68 - 6f
2,2,2,2,2,2,2,2, # 70 - 77 2, 2, 2, 2, 2, 2, 2, 2, # 70 - 77
2,2,2,2,2,2,2,1, # 78 - 7f 2, 2, 2, 2, 2, 2, 2, 1, # 78 - 7f
4,4,4,4,4,4,4,4, # 80 - 87 4, 4, 4, 4, 4, 4, 4, 4, # 80 - 87
4,4,4,4,4,4,4,4, # 88 - 8f 4, 4, 4, 4, 4, 4, 4, 4, # 88 - 8f
4,4,4,4,4,4,4,4, # 90 - 97 4, 4, 4, 4, 4, 4, 4, 4, # 90 - 97
4,4,4,4,4,4,4,4, # 98 - 9f 4, 4, 4, 4, 4, 4, 4, 4, # 98 - 9f
4,3,3,3,3,3,3,3, # a0 - a7 4, 3, 3, 3, 3, 3, 3, 3, # a0 - a7
3,3,3,3,3,3,3,3, # a8 - af 3, 3, 3, 3, 3, 3, 3, 3, # a8 - af
3,3,3,3,3,3,3,3, # b0 - b7 3, 3, 3, 3, 3, 3, 3, 3, # b0 - b7
3,3,3,3,3,3,3,3, # b8 - bf 3, 3, 3, 3, 3, 3, 3, 3, # b8 - bf
3,3,3,3,3,3,3,3, # c0 - c7 3, 3, 3, 3, 3, 3, 3, 3, # c0 - c7
3,3,3,3,3,3,3,3, # c8 - cf 3, 3, 3, 3, 3, 3, 3, 3, # c8 - cf
3,3,3,3,3,3,3,3, # d0 - d7 3, 3, 3, 3, 3, 3, 3, 3, # d0 - d7
3,3,3,3,3,3,3,3, # d8 - df 3, 3, 3, 3, 3, 3, 3, 3, # d8 - df
3,3,3,3,3,3,3,3, # e0 - e7 3, 3, 3, 3, 3, 3, 3, 3, # e0 - e7
3,3,3,3,3,3,3,3, # e8 - ef 3, 3, 3, 3, 3, 3, 3, 3, # e8 - ef
3,3,3,3,3,3,3,3, # f0 - f7 3, 3, 3, 3, 3, 3, 3, 3, # f0 - f7
3,3,3,3,3,3,3,0 # f8 - ff 3, 3, 3, 3, 3, 3, 3, 0 # f8 - ff
) )
BIG5_ST = ( BIG5_ST = (
@@ -69,34 +71,37 @@ BIG5_ST = (
MachineState.ERROR,MachineState.ERROR,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ERROR,#08-0f MachineState.ERROR,MachineState.ERROR,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ERROR,#08-0f
MachineState.ERROR,MachineState.START,MachineState.START,MachineState.START,MachineState.START,MachineState.START,MachineState.START,MachineState.START#10-17 MachineState.ERROR,MachineState.START,MachineState.START,MachineState.START,MachineState.START,MachineState.START,MachineState.START,MachineState.START#10-17
) )
# fmt: on
BIG5_CHAR_LEN_TABLE = (0, 1, 1, 2, 0) BIG5_CHAR_LEN_TABLE = (0, 1, 1, 2, 0)
BIG5_SM_MODEL = {'class_table': BIG5_CLS, BIG5_SM_MODEL: CodingStateMachineDict = {
'class_factor': 5, "class_table": BIG5_CLS,
'state_table': BIG5_ST, "class_factor": 5,
'char_len_table': BIG5_CHAR_LEN_TABLE, "state_table": BIG5_ST,
'name': 'Big5'} "char_len_table": BIG5_CHAR_LEN_TABLE,
"name": "Big5",
}
# CP949 # CP949
# fmt: off
CP949_CLS = ( CP949_CLS = (
1,1,1,1,1,1,1,1, 1,1,1,1,1,1,0,0, # 00 - 0f 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, # 00 - 0f
1,1,1,1,1,1,1,1, 1,1,1,0,1,1,1,1, # 10 - 1f 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, # 10 - 1f
1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, # 20 - 2f 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, # 20 - 2f
1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, # 30 - 3f 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, # 30 - 3f
1,4,4,4,4,4,4,4, 4,4,4,4,4,4,4,4, # 40 - 4f 1, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, # 40 - 4f
4,4,5,5,5,5,5,5, 5,5,5,1,1,1,1,1, # 50 - 5f 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 1, 1, 1, 1, 1, # 50 - 5f
1,5,5,5,5,5,5,5, 5,5,5,5,5,5,5,5, # 60 - 6f 1, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, # 60 - 6f
5,5,5,5,5,5,5,5, 5,5,5,1,1,1,1,1, # 70 - 7f 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 1, 1, 1, 1, 1, # 70 - 7f
0,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6, # 80 - 8f 0, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, # 80 - 8f
6,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6, # 90 - 9f 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, # 90 - 9f
6,7,7,7,7,7,7,7, 7,7,7,7,7,8,8,8, # a0 - af 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, # a0 - af
7,7,7,7,7,7,7,7, 7,7,7,7,7,7,7,7, # b0 - bf 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, # b0 - bf
7,7,7,7,7,7,9,2, 2,3,2,2,2,2,2,2, # c0 - cf 7, 7, 7, 7, 7, 7, 9, 2, 2, 3, 2, 2, 2, 2, 2, 2, # c0 - cf
2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, # d0 - df 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, # d0 - df
2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, # e0 - ef 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, # e0 - ef
2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,0, # f0 - ff 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, # f0 - ff
) )
CP949_ST = ( CP949_ST = (
@@ -109,50 +114,53 @@ CP949_ST = (
MachineState.ERROR,MachineState.START,MachineState.START,MachineState.START,MachineState.START,MachineState.START,MachineState.START,MachineState.START,MachineState.START,MachineState.START, # 5 MachineState.ERROR,MachineState.START,MachineState.START,MachineState.START,MachineState.START,MachineState.START,MachineState.START,MachineState.START,MachineState.START,MachineState.START, # 5
MachineState.ERROR,MachineState.START,MachineState.START,MachineState.START,MachineState.START,MachineState.ERROR,MachineState.ERROR,MachineState.START,MachineState.START,MachineState.START, # 6 MachineState.ERROR,MachineState.START,MachineState.START,MachineState.START,MachineState.START,MachineState.ERROR,MachineState.ERROR,MachineState.START,MachineState.START,MachineState.START, # 6
) )
# fmt: on
CP949_CHAR_LEN_TABLE = (0, 1, 2, 0, 1, 1, 2, 2, 0, 2) CP949_CHAR_LEN_TABLE = (0, 1, 2, 0, 1, 1, 2, 2, 0, 2)
CP949_SM_MODEL = {'class_table': CP949_CLS, CP949_SM_MODEL: CodingStateMachineDict = {
'class_factor': 10, "class_table": CP949_CLS,
'state_table': CP949_ST, "class_factor": 10,
'char_len_table': CP949_CHAR_LEN_TABLE, "state_table": CP949_ST,
'name': 'CP949'} "char_len_table": CP949_CHAR_LEN_TABLE,
"name": "CP949",
}
# EUC-JP # EUC-JP
# fmt: off
EUCJP_CLS = ( EUCJP_CLS = (
4,4,4,4,4,4,4,4, # 00 - 07 4, 4, 4, 4, 4, 4, 4, 4, # 00 - 07
4,4,4,4,4,4,5,5, # 08 - 0f 4, 4, 4, 4, 4, 4, 5, 5, # 08 - 0f
4,4,4,4,4,4,4,4, # 10 - 17 4, 4, 4, 4, 4, 4, 4, 4, # 10 - 17
4,4,4,5,4,4,4,4, # 18 - 1f 4, 4, 4, 5, 4, 4, 4, 4, # 18 - 1f
4,4,4,4,4,4,4,4, # 20 - 27 4, 4, 4, 4, 4, 4, 4, 4, # 20 - 27
4,4,4,4,4,4,4,4, # 28 - 2f 4, 4, 4, 4, 4, 4, 4, 4, # 28 - 2f
4,4,4,4,4,4,4,4, # 30 - 37 4, 4, 4, 4, 4, 4, 4, 4, # 30 - 37
4,4,4,4,4,4,4,4, # 38 - 3f 4, 4, 4, 4, 4, 4, 4, 4, # 38 - 3f
4,4,4,4,4,4,4,4, # 40 - 47 4, 4, 4, 4, 4, 4, 4, 4, # 40 - 47
4,4,4,4,4,4,4,4, # 48 - 4f 4, 4, 4, 4, 4, 4, 4, 4, # 48 - 4f
4,4,4,4,4,4,4,4, # 50 - 57 4, 4, 4, 4, 4, 4, 4, 4, # 50 - 57
4,4,4,4,4,4,4,4, # 58 - 5f 4, 4, 4, 4, 4, 4, 4, 4, # 58 - 5f
4,4,4,4,4,4,4,4, # 60 - 67 4, 4, 4, 4, 4, 4, 4, 4, # 60 - 67
4,4,4,4,4,4,4,4, # 68 - 6f 4, 4, 4, 4, 4, 4, 4, 4, # 68 - 6f
4,4,4,4,4,4,4,4, # 70 - 77 4, 4, 4, 4, 4, 4, 4, 4, # 70 - 77
4,4,4,4,4,4,4,4, # 78 - 7f 4, 4, 4, 4, 4, 4, 4, 4, # 78 - 7f
5,5,5,5,5,5,5,5, # 80 - 87 5, 5, 5, 5, 5, 5, 5, 5, # 80 - 87
5,5,5,5,5,5,1,3, # 88 - 8f 5, 5, 5, 5, 5, 5, 1, 3, # 88 - 8f
5,5,5,5,5,5,5,5, # 90 - 97 5, 5, 5, 5, 5, 5, 5, 5, # 90 - 97
5,5,5,5,5,5,5,5, # 98 - 9f 5, 5, 5, 5, 5, 5, 5, 5, # 98 - 9f
5,2,2,2,2,2,2,2, # a0 - a7 5, 2, 2, 2, 2, 2, 2, 2, # a0 - a7
2,2,2,2,2,2,2,2, # a8 - af 2, 2, 2, 2, 2, 2, 2, 2, # a8 - af
2,2,2,2,2,2,2,2, # b0 - b7 2, 2, 2, 2, 2, 2, 2, 2, # b0 - b7
2,2,2,2,2,2,2,2, # b8 - bf 2, 2, 2, 2, 2, 2, 2, 2, # b8 - bf
2,2,2,2,2,2,2,2, # c0 - c7 2, 2, 2, 2, 2, 2, 2, 2, # c0 - c7
2,2,2,2,2,2,2,2, # c8 - cf 2, 2, 2, 2, 2, 2, 2, 2, # c8 - cf
2,2,2,2,2,2,2,2, # d0 - d7 2, 2, 2, 2, 2, 2, 2, 2, # d0 - d7
2,2,2,2,2,2,2,2, # d8 - df 2, 2, 2, 2, 2, 2, 2, 2, # d8 - df
0,0,0,0,0,0,0,0, # e0 - e7 0, 0, 0, 0, 0, 0, 0, 0, # e0 - e7
0,0,0,0,0,0,0,0, # e8 - ef 0, 0, 0, 0, 0, 0, 0, 0, # e8 - ef
0,0,0,0,0,0,0,0, # f0 - f7 0, 0, 0, 0, 0, 0, 0, 0, # f0 - f7
0,0,0,0,0,0,0,5 # f8 - ff 0, 0, 0, 0, 0, 0, 0, 5 # f8 - ff
) )
EUCJP_ST = ( EUCJP_ST = (
@@ -162,100 +170,163 @@ EUCJP_ST = (
MachineState.ERROR,MachineState.ERROR,MachineState.START,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR, 3,MachineState.ERROR,#18-1f MachineState.ERROR,MachineState.ERROR,MachineState.START,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR, 3,MachineState.ERROR,#18-1f
3,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.START,MachineState.START,MachineState.START,MachineState.START#20-27 3,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.START,MachineState.START,MachineState.START,MachineState.START#20-27
) )
# fmt: on
EUCJP_CHAR_LEN_TABLE = (2, 2, 2, 3, 1, 0) EUCJP_CHAR_LEN_TABLE = (2, 2, 2, 3, 1, 0)
EUCJP_SM_MODEL = {'class_table': EUCJP_CLS, EUCJP_SM_MODEL: CodingStateMachineDict = {
'class_factor': 6, "class_table": EUCJP_CLS,
'state_table': EUCJP_ST, "class_factor": 6,
'char_len_table': EUCJP_CHAR_LEN_TABLE, "state_table": EUCJP_ST,
'name': 'EUC-JP'} "char_len_table": EUCJP_CHAR_LEN_TABLE,
"name": "EUC-JP",
}
# EUC-KR # EUC-KR
# fmt: off
EUCKR_CLS = ( EUCKR_CLS = (
1,1,1,1,1,1,1,1, # 00 - 07 1, 1, 1, 1, 1, 1, 1, 1, # 00 - 07
1,1,1,1,1,1,0,0, # 08 - 0f 1, 1, 1, 1, 1, 1, 0, 0, # 08 - 0f
1,1,1,1,1,1,1,1, # 10 - 17 1, 1, 1, 1, 1, 1, 1, 1, # 10 - 17
1,1,1,0,1,1,1,1, # 18 - 1f 1, 1, 1, 0, 1, 1, 1, 1, # 18 - 1f
1,1,1,1,1,1,1,1, # 20 - 27 1, 1, 1, 1, 1, 1, 1, 1, # 20 - 27
1,1,1,1,1,1,1,1, # 28 - 2f 1, 1, 1, 1, 1, 1, 1, 1, # 28 - 2f
1,1,1,1,1,1,1,1, # 30 - 37 1, 1, 1, 1, 1, 1, 1, 1, # 30 - 37
1,1,1,1,1,1,1,1, # 38 - 3f 1, 1, 1, 1, 1, 1, 1, 1, # 38 - 3f
1,1,1,1,1,1,1,1, # 40 - 47 1, 1, 1, 1, 1, 1, 1, 1, # 40 - 47
1,1,1,1,1,1,1,1, # 48 - 4f 1, 1, 1, 1, 1, 1, 1, 1, # 48 - 4f
1,1,1,1,1,1,1,1, # 50 - 57 1, 1, 1, 1, 1, 1, 1, 1, # 50 - 57
1,1,1,1,1,1,1,1, # 58 - 5f 1, 1, 1, 1, 1, 1, 1, 1, # 58 - 5f
1,1,1,1,1,1,1,1, # 60 - 67 1, 1, 1, 1, 1, 1, 1, 1, # 60 - 67
1,1,1,1,1,1,1,1, # 68 - 6f 1, 1, 1, 1, 1, 1, 1, 1, # 68 - 6f
1,1,1,1,1,1,1,1, # 70 - 77 1, 1, 1, 1, 1, 1, 1, 1, # 70 - 77
1,1,1,1,1,1,1,1, # 78 - 7f 1, 1, 1, 1, 1, 1, 1, 1, # 78 - 7f
0,0,0,0,0,0,0,0, # 80 - 87 0, 0, 0, 0, 0, 0, 0, 0, # 80 - 87
0,0,0,0,0,0,0,0, # 88 - 8f 0, 0, 0, 0, 0, 0, 0, 0, # 88 - 8f
0,0,0,0,0,0,0,0, # 90 - 97 0, 0, 0, 0, 0, 0, 0, 0, # 90 - 97
0,0,0,0,0,0,0,0, # 98 - 9f 0, 0, 0, 0, 0, 0, 0, 0, # 98 - 9f
0,2,2,2,2,2,2,2, # a0 - a7 0, 2, 2, 2, 2, 2, 2, 2, # a0 - a7
2,2,2,2,2,3,3,3, # a8 - af 2, 2, 2, 2, 2, 3, 3, 3, # a8 - af
2,2,2,2,2,2,2,2, # b0 - b7 2, 2, 2, 2, 2, 2, 2, 2, # b0 - b7
2,2,2,2,2,2,2,2, # b8 - bf 2, 2, 2, 2, 2, 2, 2, 2, # b8 - bf
2,2,2,2,2,2,2,2, # c0 - c7 2, 2, 2, 2, 2, 2, 2, 2, # c0 - c7
2,3,2,2,2,2,2,2, # c8 - cf 2, 3, 2, 2, 2, 2, 2, 2, # c8 - cf
2,2,2,2,2,2,2,2, # d0 - d7 2, 2, 2, 2, 2, 2, 2, 2, # d0 - d7
2,2,2,2,2,2,2,2, # d8 - df 2, 2, 2, 2, 2, 2, 2, 2, # d8 - df
2,2,2,2,2,2,2,2, # e0 - e7 2, 2, 2, 2, 2, 2, 2, 2, # e0 - e7
2,2,2,2,2,2,2,2, # e8 - ef 2, 2, 2, 2, 2, 2, 2, 2, # e8 - ef
2,2,2,2,2,2,2,2, # f0 - f7 2, 2, 2, 2, 2, 2, 2, 2, # f0 - f7
2,2,2,2,2,2,2,0 # f8 - ff 2, 2, 2, 2, 2, 2, 2, 0 # f8 - ff
) )
EUCKR_ST = ( EUCKR_ST = (
MachineState.ERROR,MachineState.START, 3,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,#00-07 MachineState.ERROR,MachineState.START, 3,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,#00-07
MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ERROR,MachineState.ERROR,MachineState.START,MachineState.START #08-0f MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ERROR,MachineState.ERROR,MachineState.START,MachineState.START #08-0f
) )
# fmt: on
EUCKR_CHAR_LEN_TABLE = (0, 1, 2, 0) EUCKR_CHAR_LEN_TABLE = (0, 1, 2, 0)
EUCKR_SM_MODEL = {'class_table': EUCKR_CLS, EUCKR_SM_MODEL: CodingStateMachineDict = {
'class_factor': 4, "class_table": EUCKR_CLS,
'state_table': EUCKR_ST, "class_factor": 4,
'char_len_table': EUCKR_CHAR_LEN_TABLE, "state_table": EUCKR_ST,
'name': 'EUC-KR'} "char_len_table": EUCKR_CHAR_LEN_TABLE,
"name": "EUC-KR",
}
# JOHAB
# fmt: off
JOHAB_CLS = (
4,4,4,4,4,4,4,4, # 00 - 07
4,4,4,4,4,4,0,0, # 08 - 0f
4,4,4,4,4,4,4,4, # 10 - 17
4,4,4,0,4,4,4,4, # 18 - 1f
4,4,4,4,4,4,4,4, # 20 - 27
4,4,4,4,4,4,4,4, # 28 - 2f
4,3,3,3,3,3,3,3, # 30 - 37
3,3,3,3,3,3,3,3, # 38 - 3f
3,1,1,1,1,1,1,1, # 40 - 47
1,1,1,1,1,1,1,1, # 48 - 4f
1,1,1,1,1,1,1,1, # 50 - 57
1,1,1,1,1,1,1,1, # 58 - 5f
1,1,1,1,1,1,1,1, # 60 - 67
1,1,1,1,1,1,1,1, # 68 - 6f
1,1,1,1,1,1,1,1, # 70 - 77
1,1,1,1,1,1,1,2, # 78 - 7f
6,6,6,6,8,8,8,8, # 80 - 87
8,8,8,8,8,8,8,8, # 88 - 8f
8,7,7,7,7,7,7,7, # 90 - 97
7,7,7,7,7,7,7,7, # 98 - 9f
7,7,7,7,7,7,7,7, # a0 - a7
7,7,7,7,7,7,7,7, # a8 - af
7,7,7,7,7,7,7,7, # b0 - b7
7,7,7,7,7,7,7,7, # b8 - bf
7,7,7,7,7,7,7,7, # c0 - c7
7,7,7,7,7,7,7,7, # c8 - cf
7,7,7,7,5,5,5,5, # d0 - d7
5,9,9,9,9,9,9,5, # d8 - df
9,9,9,9,9,9,9,9, # e0 - e7
9,9,9,9,9,9,9,9, # e8 - ef
9,9,9,9,9,9,9,9, # f0 - f7
9,9,5,5,5,5,5,0 # f8 - ff
)
JOHAB_ST = (
# cls = 0 1 2 3 4 5 6 7 8 9
MachineState.ERROR ,MachineState.START ,MachineState.START ,MachineState.START ,MachineState.START ,MachineState.ERROR ,MachineState.ERROR ,3 ,3 ,4 , # MachineState.START
MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME, # MachineState.ITS_ME
MachineState.ERROR ,MachineState.ERROR ,MachineState.ERROR ,MachineState.ERROR ,MachineState.ERROR ,MachineState.ERROR ,MachineState.ERROR ,MachineState.ERROR ,MachineState.ERROR ,MachineState.ERROR , # MachineState.ERROR
MachineState.ERROR ,MachineState.START ,MachineState.START ,MachineState.ERROR ,MachineState.ERROR ,MachineState.START ,MachineState.START ,MachineState.START ,MachineState.START ,MachineState.START , # 3
MachineState.ERROR ,MachineState.START ,MachineState.ERROR ,MachineState.START ,MachineState.ERROR ,MachineState.START ,MachineState.ERROR ,MachineState.START ,MachineState.ERROR ,MachineState.START , # 4
)
# fmt: on
JOHAB_CHAR_LEN_TABLE = (0, 1, 1, 1, 1, 0, 0, 2, 2, 2)
JOHAB_SM_MODEL: CodingStateMachineDict = {
"class_table": JOHAB_CLS,
"class_factor": 10,
"state_table": JOHAB_ST,
"char_len_table": JOHAB_CHAR_LEN_TABLE,
"name": "Johab",
}
# EUC-TW # EUC-TW
# fmt: off
EUCTW_CLS = ( EUCTW_CLS = (
2,2,2,2,2,2,2,2, # 00 - 07 2, 2, 2, 2, 2, 2, 2, 2, # 00 - 07
2,2,2,2,2,2,0,0, # 08 - 0f 2, 2, 2, 2, 2, 2, 0, 0, # 08 - 0f
2,2,2,2,2,2,2,2, # 10 - 17 2, 2, 2, 2, 2, 2, 2, 2, # 10 - 17
2,2,2,0,2,2,2,2, # 18 - 1f 2, 2, 2, 0, 2, 2, 2, 2, # 18 - 1f
2,2,2,2,2,2,2,2, # 20 - 27 2, 2, 2, 2, 2, 2, 2, 2, # 20 - 27
2,2,2,2,2,2,2,2, # 28 - 2f 2, 2, 2, 2, 2, 2, 2, 2, # 28 - 2f
2,2,2,2,2,2,2,2, # 30 - 37 2, 2, 2, 2, 2, 2, 2, 2, # 30 - 37
2,2,2,2,2,2,2,2, # 38 - 3f 2, 2, 2, 2, 2, 2, 2, 2, # 38 - 3f
2,2,2,2,2,2,2,2, # 40 - 47 2, 2, 2, 2, 2, 2, 2, 2, # 40 - 47
2,2,2,2,2,2,2,2, # 48 - 4f 2, 2, 2, 2, 2, 2, 2, 2, # 48 - 4f
2,2,2,2,2,2,2,2, # 50 - 57 2, 2, 2, 2, 2, 2, 2, 2, # 50 - 57
2,2,2,2,2,2,2,2, # 58 - 5f 2, 2, 2, 2, 2, 2, 2, 2, # 58 - 5f
2,2,2,2,2,2,2,2, # 60 - 67 2, 2, 2, 2, 2, 2, 2, 2, # 60 - 67
2,2,2,2,2,2,2,2, # 68 - 6f 2, 2, 2, 2, 2, 2, 2, 2, # 68 - 6f
2,2,2,2,2,2,2,2, # 70 - 77 2, 2, 2, 2, 2, 2, 2, 2, # 70 - 77
2,2,2,2,2,2,2,2, # 78 - 7f 2, 2, 2, 2, 2, 2, 2, 2, # 78 - 7f
0,0,0,0,0,0,0,0, # 80 - 87 0, 0, 0, 0, 0, 0, 0, 0, # 80 - 87
0,0,0,0,0,0,6,0, # 88 - 8f 0, 0, 0, 0, 0, 0, 6, 0, # 88 - 8f
0,0,0,0,0,0,0,0, # 90 - 97 0, 0, 0, 0, 0, 0, 0, 0, # 90 - 97
0,0,0,0,0,0,0,0, # 98 - 9f 0, 0, 0, 0, 0, 0, 0, 0, # 98 - 9f
0,3,4,4,4,4,4,4, # a0 - a7 0, 3, 4, 4, 4, 4, 4, 4, # a0 - a7
5,5,1,1,1,1,1,1, # a8 - af 5, 5, 1, 1, 1, 1, 1, 1, # a8 - af
1,1,1,1,1,1,1,1, # b0 - b7 1, 1, 1, 1, 1, 1, 1, 1, # b0 - b7
1,1,1,1,1,1,1,1, # b8 - bf 1, 1, 1, 1, 1, 1, 1, 1, # b8 - bf
1,1,3,1,3,3,3,3, # c0 - c7 1, 1, 3, 1, 3, 3, 3, 3, # c0 - c7
3,3,3,3,3,3,3,3, # c8 - cf 3, 3, 3, 3, 3, 3, 3, 3, # c8 - cf
3,3,3,3,3,3,3,3, # d0 - d7 3, 3, 3, 3, 3, 3, 3, 3, # d0 - d7
3,3,3,3,3,3,3,3, # d8 - df 3, 3, 3, 3, 3, 3, 3, 3, # d8 - df
3,3,3,3,3,3,3,3, # e0 - e7 3, 3, 3, 3, 3, 3, 3, 3, # e0 - e7
3,3,3,3,3,3,3,3, # e8 - ef 3, 3, 3, 3, 3, 3, 3, 3, # e8 - ef
3,3,3,3,3,3,3,3, # f0 - f7 3, 3, 3, 3, 3, 3, 3, 3, # f0 - f7
3,3,3,3,3,3,3,0 # f8 - ff 3, 3, 3, 3, 3, 3, 3, 0 # f8 - ff
) )
EUCTW_ST = ( EUCTW_ST = (
@@ -266,50 +337,53 @@ EUCTW_ST = (
5,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.START,MachineState.ERROR,MachineState.START,MachineState.START,#20-27 5,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.START,MachineState.ERROR,MachineState.START,MachineState.START,#20-27
MachineState.START,MachineState.ERROR,MachineState.START,MachineState.START,MachineState.START,MachineState.START,MachineState.START,MachineState.START #28-2f MachineState.START,MachineState.ERROR,MachineState.START,MachineState.START,MachineState.START,MachineState.START,MachineState.START,MachineState.START #28-2f
) )
# fmt: on
EUCTW_CHAR_LEN_TABLE = (0, 0, 1, 2, 2, 2, 3) EUCTW_CHAR_LEN_TABLE = (0, 0, 1, 2, 2, 2, 3)
EUCTW_SM_MODEL = {'class_table': EUCTW_CLS, EUCTW_SM_MODEL: CodingStateMachineDict = {
'class_factor': 7, "class_table": EUCTW_CLS,
'state_table': EUCTW_ST, "class_factor": 7,
'char_len_table': EUCTW_CHAR_LEN_TABLE, "state_table": EUCTW_ST,
'name': 'x-euc-tw'} "char_len_table": EUCTW_CHAR_LEN_TABLE,
"name": "x-euc-tw",
}
# GB2312 # GB2312
# fmt: off
GB2312_CLS = ( GB2312_CLS = (
1,1,1,1,1,1,1,1, # 00 - 07 1, 1, 1, 1, 1, 1, 1, 1, # 00 - 07
1,1,1,1,1,1,0,0, # 08 - 0f 1, 1, 1, 1, 1, 1, 0, 0, # 08 - 0f
1,1,1,1,1,1,1,1, # 10 - 17 1, 1, 1, 1, 1, 1, 1, 1, # 10 - 17
1,1,1,0,1,1,1,1, # 18 - 1f 1, 1, 1, 0, 1, 1, 1, 1, # 18 - 1f
1,1,1,1,1,1,1,1, # 20 - 27 1, 1, 1, 1, 1, 1, 1, 1, # 20 - 27
1,1,1,1,1,1,1,1, # 28 - 2f 1, 1, 1, 1, 1, 1, 1, 1, # 28 - 2f
3,3,3,3,3,3,3,3, # 30 - 37 3, 3, 3, 3, 3, 3, 3, 3, # 30 - 37
3,3,1,1,1,1,1,1, # 38 - 3f 3, 3, 1, 1, 1, 1, 1, 1, # 38 - 3f
2,2,2,2,2,2,2,2, # 40 - 47 2, 2, 2, 2, 2, 2, 2, 2, # 40 - 47
2,2,2,2,2,2,2,2, # 48 - 4f 2, 2, 2, 2, 2, 2, 2, 2, # 48 - 4f
2,2,2,2,2,2,2,2, # 50 - 57 2, 2, 2, 2, 2, 2, 2, 2, # 50 - 57
2,2,2,2,2,2,2,2, # 58 - 5f 2, 2, 2, 2, 2, 2, 2, 2, # 58 - 5f
2,2,2,2,2,2,2,2, # 60 - 67 2, 2, 2, 2, 2, 2, 2, 2, # 60 - 67
2,2,2,2,2,2,2,2, # 68 - 6f 2, 2, 2, 2, 2, 2, 2, 2, # 68 - 6f
2,2,2,2,2,2,2,2, # 70 - 77 2, 2, 2, 2, 2, 2, 2, 2, # 70 - 77
2,2,2,2,2,2,2,4, # 78 - 7f 2, 2, 2, 2, 2, 2, 2, 4, # 78 - 7f
5,6,6,6,6,6,6,6, # 80 - 87 5, 6, 6, 6, 6, 6, 6, 6, # 80 - 87
6,6,6,6,6,6,6,6, # 88 - 8f 6, 6, 6, 6, 6, 6, 6, 6, # 88 - 8f
6,6,6,6,6,6,6,6, # 90 - 97 6, 6, 6, 6, 6, 6, 6, 6, # 90 - 97
6,6,6,6,6,6,6,6, # 98 - 9f 6, 6, 6, 6, 6, 6, 6, 6, # 98 - 9f
6,6,6,6,6,6,6,6, # a0 - a7 6, 6, 6, 6, 6, 6, 6, 6, # a0 - a7
6,6,6,6,6,6,6,6, # a8 - af 6, 6, 6, 6, 6, 6, 6, 6, # a8 - af
6,6,6,6,6,6,6,6, # b0 - b7 6, 6, 6, 6, 6, 6, 6, 6, # b0 - b7
6,6,6,6,6,6,6,6, # b8 - bf 6, 6, 6, 6, 6, 6, 6, 6, # b8 - bf
6,6,6,6,6,6,6,6, # c0 - c7 6, 6, 6, 6, 6, 6, 6, 6, # c0 - c7
6,6,6,6,6,6,6,6, # c8 - cf 6, 6, 6, 6, 6, 6, 6, 6, # c8 - cf
6,6,6,6,6,6,6,6, # d0 - d7 6, 6, 6, 6, 6, 6, 6, 6, # d0 - d7
6,6,6,6,6,6,6,6, # d8 - df 6, 6, 6, 6, 6, 6, 6, 6, # d8 - df
6,6,6,6,6,6,6,6, # e0 - e7 6, 6, 6, 6, 6, 6, 6, 6, # e0 - e7
6,6,6,6,6,6,6,6, # e8 - ef 6, 6, 6, 6, 6, 6, 6, 6, # e8 - ef
6,6,6,6,6,6,6,6, # f0 - f7 6, 6, 6, 6, 6, 6, 6, 6, # f0 - f7
6,6,6,6,6,6,6,0 # f8 - ff 6, 6, 6, 6, 6, 6, 6, 0 # f8 - ff
) )
GB2312_ST = ( GB2312_ST = (
@@ -320,6 +394,7 @@ GB2312_ST = (
MachineState.ERROR,MachineState.ERROR, 5,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ITS_ME,MachineState.ERROR,#20-27 MachineState.ERROR,MachineState.ERROR, 5,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ITS_ME,MachineState.ERROR,#20-27
MachineState.ERROR,MachineState.ERROR,MachineState.START,MachineState.START,MachineState.START,MachineState.START,MachineState.START,MachineState.START #28-2f MachineState.ERROR,MachineState.ERROR,MachineState.START,MachineState.START,MachineState.START,MachineState.START,MachineState.START,MachineState.START #28-2f
) )
# fmt: on
# To be accurate, the length of class 6 can be either 2 or 4. # To be accurate, the length of class 6 can be either 2 or 4.
# But it is not necessary to discriminate between the two since # But it is not necessary to discriminate between the two since
@@ -328,100 +403,105 @@ GB2312_ST = (
# 2 here. # 2 here.
GB2312_CHAR_LEN_TABLE = (0, 1, 1, 1, 1, 1, 2) GB2312_CHAR_LEN_TABLE = (0, 1, 1, 1, 1, 1, 2)
GB2312_SM_MODEL = {'class_table': GB2312_CLS, GB2312_SM_MODEL: CodingStateMachineDict = {
'class_factor': 7, "class_table": GB2312_CLS,
'state_table': GB2312_ST, "class_factor": 7,
'char_len_table': GB2312_CHAR_LEN_TABLE, "state_table": GB2312_ST,
'name': 'GB2312'} "char_len_table": GB2312_CHAR_LEN_TABLE,
"name": "GB2312",
}
# Shift_JIS # Shift_JIS
# fmt: off
SJIS_CLS = ( SJIS_CLS = (
1,1,1,1,1,1,1,1, # 00 - 07 1, 1, 1, 1, 1, 1, 1, 1, # 00 - 07
1,1,1,1,1,1,0,0, # 08 - 0f 1, 1, 1, 1, 1, 1, 0, 0, # 08 - 0f
1,1,1,1,1,1,1,1, # 10 - 17 1, 1, 1, 1, 1, 1, 1, 1, # 10 - 17
1,1,1,0,1,1,1,1, # 18 - 1f 1, 1, 1, 0, 1, 1, 1, 1, # 18 - 1f
1,1,1,1,1,1,1,1, # 20 - 27 1, 1, 1, 1, 1, 1, 1, 1, # 20 - 27
1,1,1,1,1,1,1,1, # 28 - 2f 1, 1, 1, 1, 1, 1, 1, 1, # 28 - 2f
1,1,1,1,1,1,1,1, # 30 - 37 1, 1, 1, 1, 1, 1, 1, 1, # 30 - 37
1,1,1,1,1,1,1,1, # 38 - 3f 1, 1, 1, 1, 1, 1, 1, 1, # 38 - 3f
2,2,2,2,2,2,2,2, # 40 - 47 2, 2, 2, 2, 2, 2, 2, 2, # 40 - 47
2,2,2,2,2,2,2,2, # 48 - 4f 2, 2, 2, 2, 2, 2, 2, 2, # 48 - 4f
2,2,2,2,2,2,2,2, # 50 - 57 2, 2, 2, 2, 2, 2, 2, 2, # 50 - 57
2,2,2,2,2,2,2,2, # 58 - 5f 2, 2, 2, 2, 2, 2, 2, 2, # 58 - 5f
2,2,2,2,2,2,2,2, # 60 - 67 2, 2, 2, 2, 2, 2, 2, 2, # 60 - 67
2,2,2,2,2,2,2,2, # 68 - 6f 2, 2, 2, 2, 2, 2, 2, 2, # 68 - 6f
2,2,2,2,2,2,2,2, # 70 - 77 2, 2, 2, 2, 2, 2, 2, 2, # 70 - 77
2,2,2,2,2,2,2,1, # 78 - 7f 2, 2, 2, 2, 2, 2, 2, 1, # 78 - 7f
3,3,3,3,3,2,2,3, # 80 - 87 3, 3, 3, 3, 3, 2, 2, 3, # 80 - 87
3,3,3,3,3,3,3,3, # 88 - 8f 3, 3, 3, 3, 3, 3, 3, 3, # 88 - 8f
3,3,3,3,3,3,3,3, # 90 - 97 3, 3, 3, 3, 3, 3, 3, 3, # 90 - 97
3,3,3,3,3,3,3,3, # 98 - 9f 3, 3, 3, 3, 3, 3, 3, 3, # 98 - 9f
#0xa0 is illegal in sjis encoding, but some pages does #0xa0 is illegal in sjis encoding, but some pages does
#contain such byte. We need to be more error forgiven. #contain such byte. We need to be more error forgiven.
2,2,2,2,2,2,2,2, # a0 - a7 2, 2, 2, 2, 2, 2, 2, 2, # a0 - a7
2,2,2,2,2,2,2,2, # a8 - af 2, 2, 2, 2, 2, 2, 2, 2, # a8 - af
2,2,2,2,2,2,2,2, # b0 - b7 2, 2, 2, 2, 2, 2, 2, 2, # b0 - b7
2,2,2,2,2,2,2,2, # b8 - bf 2, 2, 2, 2, 2, 2, 2, 2, # b8 - bf
2,2,2,2,2,2,2,2, # c0 - c7 2, 2, 2, 2, 2, 2, 2, 2, # c0 - c7
2,2,2,2,2,2,2,2, # c8 - cf 2, 2, 2, 2, 2, 2, 2, 2, # c8 - cf
2,2,2,2,2,2,2,2, # d0 - d7 2, 2, 2, 2, 2, 2, 2, 2, # d0 - d7
2,2,2,2,2,2,2,2, # d8 - df 2, 2, 2, 2, 2, 2, 2, 2, # d8 - df
3,3,3,3,3,3,3,3, # e0 - e7 3, 3, 3, 3, 3, 3, 3, 3, # e0 - e7
3,3,3,3,3,4,4,4, # e8 - ef 3, 3, 3, 3, 3, 4, 4, 4, # e8 - ef
3,3,3,3,3,3,3,3, # f0 - f7 3, 3, 3, 3, 3, 3, 3, 3, # f0 - f7
3,3,3,3,3,0,0,0) # f8 - ff 3, 3, 3, 3, 3, 0, 0, 0, # f8 - ff
)
SJIS_ST = ( SJIS_ST = (
MachineState.ERROR,MachineState.START,MachineState.START, 3,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,#00-07 MachineState.ERROR,MachineState.START,MachineState.START, 3,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,#00-07
MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,#08-0f MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,#08-0f
MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ERROR,MachineState.ERROR,MachineState.START,MachineState.START,MachineState.START,MachineState.START #10-17 MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ERROR,MachineState.ERROR,MachineState.START,MachineState.START,MachineState.START,MachineState.START #10-17
) )
# fmt: on
SJIS_CHAR_LEN_TABLE = (0, 1, 1, 2, 0, 0) SJIS_CHAR_LEN_TABLE = (0, 1, 1, 2, 0, 0)
SJIS_SM_MODEL = {'class_table': SJIS_CLS, SJIS_SM_MODEL: CodingStateMachineDict = {
'class_factor': 6, "class_table": SJIS_CLS,
'state_table': SJIS_ST, "class_factor": 6,
'char_len_table': SJIS_CHAR_LEN_TABLE, "state_table": SJIS_ST,
'name': 'Shift_JIS'} "char_len_table": SJIS_CHAR_LEN_TABLE,
"name": "Shift_JIS",
}
# UCS2-BE # UCS2-BE
# fmt: off
UCS2BE_CLS = ( UCS2BE_CLS = (
0,0,0,0,0,0,0,0, # 00 - 07 0, 0, 0, 0, 0, 0, 0, 0, # 00 - 07
0,0,1,0,0,2,0,0, # 08 - 0f 0, 0, 1, 0, 0, 2, 0, 0, # 08 - 0f
0,0,0,0,0,0,0,0, # 10 - 17 0, 0, 0, 0, 0, 0, 0, 0, # 10 - 17
0,0,0,3,0,0,0,0, # 18 - 1f 0, 0, 0, 3, 0, 0, 0, 0, # 18 - 1f
0,0,0,0,0,0,0,0, # 20 - 27 0, 0, 0, 0, 0, 0, 0, 0, # 20 - 27
0,3,3,3,3,3,0,0, # 28 - 2f 0, 3, 3, 3, 3, 3, 0, 0, # 28 - 2f
0,0,0,0,0,0,0,0, # 30 - 37 0, 0, 0, 0, 0, 0, 0, 0, # 30 - 37
0,0,0,0,0,0,0,0, # 38 - 3f 0, 0, 0, 0, 0, 0, 0, 0, # 38 - 3f
0,0,0,0,0,0,0,0, # 40 - 47 0, 0, 0, 0, 0, 0, 0, 0, # 40 - 47
0,0,0,0,0,0,0,0, # 48 - 4f 0, 0, 0, 0, 0, 0, 0, 0, # 48 - 4f
0,0,0,0,0,0,0,0, # 50 - 57 0, 0, 0, 0, 0, 0, 0, 0, # 50 - 57
0,0,0,0,0,0,0,0, # 58 - 5f 0, 0, 0, 0, 0, 0, 0, 0, # 58 - 5f
0,0,0,0,0,0,0,0, # 60 - 67 0, 0, 0, 0, 0, 0, 0, 0, # 60 - 67
0,0,0,0,0,0,0,0, # 68 - 6f 0, 0, 0, 0, 0, 0, 0, 0, # 68 - 6f
0,0,0,0,0,0,0,0, # 70 - 77 0, 0, 0, 0, 0, 0, 0, 0, # 70 - 77
0,0,0,0,0,0,0,0, # 78 - 7f 0, 0, 0, 0, 0, 0, 0, 0, # 78 - 7f
0,0,0,0,0,0,0,0, # 80 - 87 0, 0, 0, 0, 0, 0, 0, 0, # 80 - 87
0,0,0,0,0,0,0,0, # 88 - 8f 0, 0, 0, 0, 0, 0, 0, 0, # 88 - 8f
0,0,0,0,0,0,0,0, # 90 - 97 0, 0, 0, 0, 0, 0, 0, 0, # 90 - 97
0,0,0,0,0,0,0,0, # 98 - 9f 0, 0, 0, 0, 0, 0, 0, 0, # 98 - 9f
0,0,0,0,0,0,0,0, # a0 - a7 0, 0, 0, 0, 0, 0, 0, 0, # a0 - a7
0,0,0,0,0,0,0,0, # a8 - af 0, 0, 0, 0, 0, 0, 0, 0, # a8 - af
0,0,0,0,0,0,0,0, # b0 - b7 0, 0, 0, 0, 0, 0, 0, 0, # b0 - b7
0,0,0,0,0,0,0,0, # b8 - bf 0, 0, 0, 0, 0, 0, 0, 0, # b8 - bf
0,0,0,0,0,0,0,0, # c0 - c7 0, 0, 0, 0, 0, 0, 0, 0, # c0 - c7
0,0,0,0,0,0,0,0, # c8 - cf 0, 0, 0, 0, 0, 0, 0, 0, # c8 - cf
0,0,0,0,0,0,0,0, # d0 - d7 0, 0, 0, 0, 0, 0, 0, 0, # d0 - d7
0,0,0,0,0,0,0,0, # d8 - df 0, 0, 0, 0, 0, 0, 0, 0, # d8 - df
0,0,0,0,0,0,0,0, # e0 - e7 0, 0, 0, 0, 0, 0, 0, 0, # e0 - e7
0,0,0,0,0,0,0,0, # e8 - ef 0, 0, 0, 0, 0, 0, 0, 0, # e8 - ef
0,0,0,0,0,0,0,0, # f0 - f7 0, 0, 0, 0, 0, 0, 0, 0, # f0 - f7
0,0,0,0,0,0,4,5 # f8 - ff 0, 0, 0, 0, 0, 0, 4, 5 # f8 - ff
) )
UCS2BE_ST = ( UCS2BE_ST = (
@@ -433,50 +513,53 @@ UCS2BE_ST = (
5, 8, 6, 6,MachineState.ERROR, 6, 6, 6,#28-2f 5, 8, 6, 6,MachineState.ERROR, 6, 6, 6,#28-2f
6, 6, 6, 6,MachineState.ERROR,MachineState.ERROR,MachineState.START,MachineState.START #30-37 6, 6, 6, 6,MachineState.ERROR,MachineState.ERROR,MachineState.START,MachineState.START #30-37
) )
# fmt: on
UCS2BE_CHAR_LEN_TABLE = (2, 2, 2, 0, 2, 2) UCS2BE_CHAR_LEN_TABLE = (2, 2, 2, 0, 2, 2)
UCS2BE_SM_MODEL = {'class_table': UCS2BE_CLS, UCS2BE_SM_MODEL: CodingStateMachineDict = {
'class_factor': 6, "class_table": UCS2BE_CLS,
'state_table': UCS2BE_ST, "class_factor": 6,
'char_len_table': UCS2BE_CHAR_LEN_TABLE, "state_table": UCS2BE_ST,
'name': 'UTF-16BE'} "char_len_table": UCS2BE_CHAR_LEN_TABLE,
"name": "UTF-16BE",
}
# UCS2-LE # UCS2-LE
# fmt: off
UCS2LE_CLS = ( UCS2LE_CLS = (
0,0,0,0,0,0,0,0, # 00 - 07 0, 0, 0, 0, 0, 0, 0, 0, # 00 - 07
0,0,1,0,0,2,0,0, # 08 - 0f 0, 0, 1, 0, 0, 2, 0, 0, # 08 - 0f
0,0,0,0,0,0,0,0, # 10 - 17 0, 0, 0, 0, 0, 0, 0, 0, # 10 - 17
0,0,0,3,0,0,0,0, # 18 - 1f 0, 0, 0, 3, 0, 0, 0, 0, # 18 - 1f
0,0,0,0,0,0,0,0, # 20 - 27 0, 0, 0, 0, 0, 0, 0, 0, # 20 - 27
0,3,3,3,3,3,0,0, # 28 - 2f 0, 3, 3, 3, 3, 3, 0, 0, # 28 - 2f
0,0,0,0,0,0,0,0, # 30 - 37 0, 0, 0, 0, 0, 0, 0, 0, # 30 - 37
0,0,0,0,0,0,0,0, # 38 - 3f 0, 0, 0, 0, 0, 0, 0, 0, # 38 - 3f
0,0,0,0,0,0,0,0, # 40 - 47 0, 0, 0, 0, 0, 0, 0, 0, # 40 - 47
0,0,0,0,0,0,0,0, # 48 - 4f 0, 0, 0, 0, 0, 0, 0, 0, # 48 - 4f
0,0,0,0,0,0,0,0, # 50 - 57 0, 0, 0, 0, 0, 0, 0, 0, # 50 - 57
0,0,0,0,0,0,0,0, # 58 - 5f 0, 0, 0, 0, 0, 0, 0, 0, # 58 - 5f
0,0,0,0,0,0,0,0, # 60 - 67 0, 0, 0, 0, 0, 0, 0, 0, # 60 - 67
0,0,0,0,0,0,0,0, # 68 - 6f 0, 0, 0, 0, 0, 0, 0, 0, # 68 - 6f
0,0,0,0,0,0,0,0, # 70 - 77 0, 0, 0, 0, 0, 0, 0, 0, # 70 - 77
0,0,0,0,0,0,0,0, # 78 - 7f 0, 0, 0, 0, 0, 0, 0, 0, # 78 - 7f
0,0,0,0,0,0,0,0, # 80 - 87 0, 0, 0, 0, 0, 0, 0, 0, # 80 - 87
0,0,0,0,0,0,0,0, # 88 - 8f 0, 0, 0, 0, 0, 0, 0, 0, # 88 - 8f
0,0,0,0,0,0,0,0, # 90 - 97 0, 0, 0, 0, 0, 0, 0, 0, # 90 - 97
0,0,0,0,0,0,0,0, # 98 - 9f 0, 0, 0, 0, 0, 0, 0, 0, # 98 - 9f
0,0,0,0,0,0,0,0, # a0 - a7 0, 0, 0, 0, 0, 0, 0, 0, # a0 - a7
0,0,0,0,0,0,0,0, # a8 - af 0, 0, 0, 0, 0, 0, 0, 0, # a8 - af
0,0,0,0,0,0,0,0, # b0 - b7 0, 0, 0, 0, 0, 0, 0, 0, # b0 - b7
0,0,0,0,0,0,0,0, # b8 - bf 0, 0, 0, 0, 0, 0, 0, 0, # b8 - bf
0,0,0,0,0,0,0,0, # c0 - c7 0, 0, 0, 0, 0, 0, 0, 0, # c0 - c7
0,0,0,0,0,0,0,0, # c8 - cf 0, 0, 0, 0, 0, 0, 0, 0, # c8 - cf
0,0,0,0,0,0,0,0, # d0 - d7 0, 0, 0, 0, 0, 0, 0, 0, # d0 - d7
0,0,0,0,0,0,0,0, # d8 - df 0, 0, 0, 0, 0, 0, 0, 0, # d8 - df
0,0,0,0,0,0,0,0, # e0 - e7 0, 0, 0, 0, 0, 0, 0, 0, # e0 - e7
0,0,0,0,0,0,0,0, # e8 - ef 0, 0, 0, 0, 0, 0, 0, 0, # e8 - ef
0,0,0,0,0,0,0,0, # f0 - f7 0, 0, 0, 0, 0, 0, 0, 0, # f0 - f7
0,0,0,0,0,0,4,5 # f8 - ff 0, 0, 0, 0, 0, 0, 4, 5 # f8 - ff
) )
UCS2LE_ST = ( UCS2LE_ST = (
@@ -488,50 +571,53 @@ UCS2LE_ST = (
5, 5, 5,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR, 5, 5,#28-2f 5, 5, 5,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR, 5, 5,#28-2f
5, 5, 5,MachineState.ERROR, 5,MachineState.ERROR,MachineState.START,MachineState.START #30-37 5, 5, 5,MachineState.ERROR, 5,MachineState.ERROR,MachineState.START,MachineState.START #30-37
) )
# fmt: on
UCS2LE_CHAR_LEN_TABLE = (2, 2, 2, 2, 2, 2) UCS2LE_CHAR_LEN_TABLE = (2, 2, 2, 2, 2, 2)
UCS2LE_SM_MODEL = {'class_table': UCS2LE_CLS, UCS2LE_SM_MODEL: CodingStateMachineDict = {
'class_factor': 6, "class_table": UCS2LE_CLS,
'state_table': UCS2LE_ST, "class_factor": 6,
'char_len_table': UCS2LE_CHAR_LEN_TABLE, "state_table": UCS2LE_ST,
'name': 'UTF-16LE'} "char_len_table": UCS2LE_CHAR_LEN_TABLE,
"name": "UTF-16LE",
}
# UTF-8 # UTF-8
# fmt: off
UTF8_CLS = ( UTF8_CLS = (
1,1,1,1,1,1,1,1, # 00 - 07 #allow 0x00 as a legal value 1, 1, 1, 1, 1, 1, 1, 1, # 00 - 07 #allow 0x00 as a legal value
1,1,1,1,1,1,0,0, # 08 - 0f 1, 1, 1, 1, 1, 1, 0, 0, # 08 - 0f
1,1,1,1,1,1,1,1, # 10 - 17 1, 1, 1, 1, 1, 1, 1, 1, # 10 - 17
1,1,1,0,1,1,1,1, # 18 - 1f 1, 1, 1, 0, 1, 1, 1, 1, # 18 - 1f
1,1,1,1,1,1,1,1, # 20 - 27 1, 1, 1, 1, 1, 1, 1, 1, # 20 - 27
1,1,1,1,1,1,1,1, # 28 - 2f 1, 1, 1, 1, 1, 1, 1, 1, # 28 - 2f
1,1,1,1,1,1,1,1, # 30 - 37 1, 1, 1, 1, 1, 1, 1, 1, # 30 - 37
1,1,1,1,1,1,1,1, # 38 - 3f 1, 1, 1, 1, 1, 1, 1, 1, # 38 - 3f
1,1,1,1,1,1,1,1, # 40 - 47 1, 1, 1, 1, 1, 1, 1, 1, # 40 - 47
1,1,1,1,1,1,1,1, # 48 - 4f 1, 1, 1, 1, 1, 1, 1, 1, # 48 - 4f
1,1,1,1,1,1,1,1, # 50 - 57 1, 1, 1, 1, 1, 1, 1, 1, # 50 - 57
1,1,1,1,1,1,1,1, # 58 - 5f 1, 1, 1, 1, 1, 1, 1, 1, # 58 - 5f
1,1,1,1,1,1,1,1, # 60 - 67 1, 1, 1, 1, 1, 1, 1, 1, # 60 - 67
1,1,1,1,1,1,1,1, # 68 - 6f 1, 1, 1, 1, 1, 1, 1, 1, # 68 - 6f
1,1,1,1,1,1,1,1, # 70 - 77 1, 1, 1, 1, 1, 1, 1, 1, # 70 - 77
1,1,1,1,1,1,1,1, # 78 - 7f 1, 1, 1, 1, 1, 1, 1, 1, # 78 - 7f
2,2,2,2,3,3,3,3, # 80 - 87 2, 2, 2, 2, 3, 3, 3, 3, # 80 - 87
4,4,4,4,4,4,4,4, # 88 - 8f 4, 4, 4, 4, 4, 4, 4, 4, # 88 - 8f
4,4,4,4,4,4,4,4, # 90 - 97 4, 4, 4, 4, 4, 4, 4, 4, # 90 - 97
4,4,4,4,4,4,4,4, # 98 - 9f 4, 4, 4, 4, 4, 4, 4, 4, # 98 - 9f
5,5,5,5,5,5,5,5, # a0 - a7 5, 5, 5, 5, 5, 5, 5, 5, # a0 - a7
5,5,5,5,5,5,5,5, # a8 - af 5, 5, 5, 5, 5, 5, 5, 5, # a8 - af
5,5,5,5,5,5,5,5, # b0 - b7 5, 5, 5, 5, 5, 5, 5, 5, # b0 - b7
5,5,5,5,5,5,5,5, # b8 - bf 5, 5, 5, 5, 5, 5, 5, 5, # b8 - bf
0,0,6,6,6,6,6,6, # c0 - c7 0, 0, 6, 6, 6, 6, 6, 6, # c0 - c7
6,6,6,6,6,6,6,6, # c8 - cf 6, 6, 6, 6, 6, 6, 6, 6, # c8 - cf
6,6,6,6,6,6,6,6, # d0 - d7 6, 6, 6, 6, 6, 6, 6, 6, # d0 - d7
6,6,6,6,6,6,6,6, # d8 - df 6, 6, 6, 6, 6, 6, 6, 6, # d8 - df
7,8,8,8,8,8,8,8, # e0 - e7 7, 8, 8, 8, 8, 8, 8, 8, # e0 - e7
8,8,8,8,8,9,8,8, # e8 - ef 8, 8, 8, 8, 8, 9, 8, 8, # e8 - ef
10,11,11,11,11,11,11,11, # f0 - f7 10, 11, 11, 11, 11, 11, 11, 11, # f0 - f7
12,13,13,13,14,15,0,0 # f8 - ff 12, 13, 13, 13, 14, 15, 0, 0 # f8 - ff
) )
UTF8_ST = ( UTF8_ST = (
@@ -562,11 +648,14 @@ UTF8_ST = (
MachineState.ERROR,MachineState.ERROR,MachineState.START,MachineState.START,MachineState.START,MachineState.START,MachineState.ERROR,MachineState.ERROR,#c0-c7 MachineState.ERROR,MachineState.ERROR,MachineState.START,MachineState.START,MachineState.START,MachineState.START,MachineState.ERROR,MachineState.ERROR,#c0-c7
MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR #c8-cf MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR #c8-cf
) )
# fmt: on
UTF8_CHAR_LEN_TABLE = (0, 1, 0, 0, 0, 0, 2, 3, 3, 3, 4, 4, 5, 5, 6, 6) UTF8_CHAR_LEN_TABLE = (0, 1, 0, 0, 0, 0, 2, 3, 3, 3, 4, 4, 5, 5, 6, 6)
UTF8_SM_MODEL = {'class_table': UTF8_CLS, UTF8_SM_MODEL: CodingStateMachineDict = {
'class_factor': 16, "class_table": UTF8_CLS,
'state_table': UTF8_ST, "class_factor": 16,
'char_len_table': UTF8_CHAR_LEN_TABLE, "state_table": UTF8_ST,
'name': 'UTF-8'} "char_len_table": UTF8_CHAR_LEN_TABLE,
"name": "UTF-8",
}
+309 -267
View File
@@ -1,19 +1,17 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
""" """
Metadata about languages used by our model training code for our Metadata about languages used by our model training code for our
SingleByteCharSetProbers. Could be used for other things in the future. SingleByteCharSetProbers. Could be used for other things in the future.
This code is based on the language metadata from the uchardet project. This code is based on the language metadata from the uchardet project.
""" """
from __future__ import absolute_import, print_function
from string import ascii_letters from string import ascii_letters
from typing import List, Optional
# TODO: Add Ukrainian (KOI8-U)
# TODO: Add Ukranian (KOI8-U) class Language:
class Language(object):
"""Metadata about a language useful for training models """Metadata about a language useful for training models
:ivar name: The human name for the language, in English. :ivar name: The human name for the language, in English.
@@ -33,9 +31,17 @@ class Language(object):
Wikipedia for training data. Wikipedia for training data.
:type wiki_start_pages: list of str :type wiki_start_pages: list of str
""" """
def __init__(self, name=None, iso_code=None, use_ascii=True, charsets=None,
alphabet=None, wiki_start_pages=None): def __init__(
super(Language, self).__init__() self,
name: Optional[str] = None,
iso_code: Optional[str] = None,
use_ascii: bool = True,
charsets: Optional[List[str]] = None,
alphabet: Optional[str] = None,
wiki_start_pages: Optional[List[str]] = None,
) -> None:
super().__init__()
self.name = name self.name = name
self.iso_code = iso_code self.iso_code = iso_code
self.use_ascii = use_ascii self.use_ascii = use_ascii
@@ -46,265 +52,301 @@ class Language(object):
else: else:
alphabet = ascii_letters alphabet = ascii_letters
elif not alphabet: elif not alphabet:
raise ValueError('Must supply alphabet if use_ascii is False') raise ValueError("Must supply alphabet if use_ascii is False")
self.alphabet = ''.join(sorted(set(alphabet))) if alphabet else None self.alphabet = "".join(sorted(set(alphabet))) if alphabet else None
self.wiki_start_pages = wiki_start_pages self.wiki_start_pages = wiki_start_pages
def __repr__(self): def __repr__(self) -> str:
return '{}({})'.format(self.__class__.__name__, param_str = ", ".join(
', '.join('{}={!r}'.format(k, v) f"{k}={v!r}" for k, v in self.__dict__.items() if not k.startswith("_")
for k, v in self.__dict__.items() )
if not k.startswith('_'))) return f"{self.__class__.__name__}({param_str})"
LANGUAGES = {'Arabic': Language(name='Arabic', LANGUAGES = {
iso_code='ar', "Arabic": Language(
use_ascii=False, name="Arabic",
# We only support encodings that use isolated iso_code="ar",
# forms, because the current recommendation is use_ascii=False,
# that the rendering system handles presentation # We only support encodings that use isolated
# forms. This means we purposefully skip IBM864. # forms, because the current recommendation is
charsets=['ISO-8859-6', 'WINDOWS-1256', # that the rendering system handles presentation
'CP720', 'CP864'], # forms. This means we purposefully skip IBM864.
alphabet=u'ءآأؤإئابةتثجحخدذرزسشصضطظعغػؼؽؾؿـفقكلمنهوىيًٌٍَُِّ', charsets=["ISO-8859-6", "WINDOWS-1256", "CP720", "CP864"],
wiki_start_pages=[u'الصفحة_الرئيسية']), alphabet="ءآأؤإئابةتثجحخدذرزسشصضطظعغػؼؽؾؿـفقكلمنهوىيًٌٍَُِّ",
'Belarusian': Language(name='Belarusian', wiki_start_pages=["الصفحة_الرئيسية"],
iso_code='be', ),
use_ascii=False, "Belarusian": Language(
charsets=['ISO-8859-5', 'WINDOWS-1251', name="Belarusian",
'IBM866', 'MacCyrillic'], iso_code="be",
alphabet=(u'АБВГДЕЁЖЗІЙКЛМНОПРСТУЎФХЦЧШЫЬЭЮЯ' use_ascii=False,
u'абвгдеёжзійклмнопрстуўфхцчшыьэюяʼ'), charsets=["ISO-8859-5", "WINDOWS-1251", "IBM866", "MacCyrillic"],
wiki_start_pages=[u'Галоўная_старонка']), alphabet="АБВГДЕЁЖЗІЙКЛМНОПРСТУЎФХЦЧШЫЬЭЮЯабвгдеёжзійклмнопрстуўфхцчшыьэюяʼ",
'Bulgarian': Language(name='Bulgarian', wiki_start_pages=["Галоўная_старонка"],
iso_code='bg', ),
use_ascii=False, "Bulgarian": Language(
charsets=['ISO-8859-5', 'WINDOWS-1251', name="Bulgarian",
'IBM855'], iso_code="bg",
alphabet=(u'АБВГДЕЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЬЮЯ' use_ascii=False,
u'абвгдежзийклмнопрстуфхцчшщъьюя'), charsets=["ISO-8859-5", "WINDOWS-1251", "IBM855"],
wiki_start_pages=[u'Начална_страница']), alphabet="АБВГДЕЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЬЮЯабвгдежзийклмнопрстуфхцчшщъьюя",
'Czech': Language(name='Czech', wiki_start_pages=["Начална_страница"],
iso_code='cz', ),
use_ascii=True, "Czech": Language(
charsets=['ISO-8859-2', 'WINDOWS-1250'], name="Czech",
alphabet=u'áčďéěíňóřšťúůýžÁČĎÉĚÍŇÓŘŠŤÚŮÝŽ', iso_code="cz",
wiki_start_pages=[u'Hlavní_strana']), use_ascii=True,
'Danish': Language(name='Danish', charsets=["ISO-8859-2", "WINDOWS-1250"],
iso_code='da', alphabet="áčďéěíňóřšťúůýžÁČĎÉĚÍŇÓŘŠŤÚŮÝŽ",
use_ascii=True, wiki_start_pages=["Hlavní_strana"],
charsets=['ISO-8859-1', 'ISO-8859-15', ),
'WINDOWS-1252'], "Danish": Language(
alphabet=u'æøåÆØÅ', name="Danish",
wiki_start_pages=[u'Forside']), iso_code="da",
'German': Language(name='German', use_ascii=True,
iso_code='de', charsets=["ISO-8859-1", "ISO-8859-15", "WINDOWS-1252", "MacRoman"],
use_ascii=True, alphabet="æøåÆØÅ",
charsets=['ISO-8859-1', 'WINDOWS-1252'], wiki_start_pages=["Forside"],
alphabet=u'äöüßÄÖÜ', ),
wiki_start_pages=[u'Wikipedia:Hauptseite']), "German": Language(
'Greek': Language(name='Greek', name="German",
iso_code='el', iso_code="de",
use_ascii=False, use_ascii=True,
charsets=['ISO-8859-7', 'WINDOWS-1253'], charsets=["ISO-8859-1", "ISO-8859-15", "WINDOWS-1252", "MacRoman"],
alphabet=(u'αβγδεζηθικλμνξοπρσςτυφχψωάέήίόύώ' alphabet="äöüßẞÄÖÜ",
u'ΑΒΓΔΕΖΗΘΙΚΛΜΝΞΟΠΡΣΣΤΥΦΧΨΩΆΈΉΊΌΎΏ'), wiki_start_pages=["Wikipedia:Hauptseite"],
wiki_start_pages=[u'Πύλη:Κύρια']), ),
'English': Language(name='English', "Greek": Language(
iso_code='en', name="Greek",
use_ascii=True, iso_code="el",
charsets=['ISO-8859-1', 'WINDOWS-1252'], use_ascii=False,
wiki_start_pages=[u'Main_Page']), charsets=["ISO-8859-7", "WINDOWS-1253"],
'Esperanto': Language(name='Esperanto', alphabet="αβγδεζηθικλμνξοπρσςτυφχψωάέήίόύώΑΒΓΔΕΖΗΘΙΚΛΜΝΞΟΠΡΣΣΤΥΦΧΨΩΆΈΉΊΌΎΏ",
iso_code='eo', wiki_start_pages=["Πύλη:Κύρια"],
# Q, W, X, and Y not used at all ),
use_ascii=False, "English": Language(
charsets=['ISO-8859-3'], name="English",
alphabet=(u'abcĉdefgĝhĥijĵklmnoprsŝtuŭvz' iso_code="en",
u'ABCĈDEFGĜHĤIJĴKLMNOPRSŜTUŬVZ'), use_ascii=True,
wiki_start_pages=[u'Vikipedio:Ĉefpaĝo']), charsets=["ISO-8859-1", "WINDOWS-1252", "MacRoman"],
'Spanish': Language(name='Spanish', wiki_start_pages=["Main_Page"],
iso_code='es', ),
use_ascii=True, "Esperanto": Language(
charsets=['ISO-8859-1', 'ISO-8859-15', name="Esperanto",
'WINDOWS-1252'], iso_code="eo",
alphabet=u'ñáéíóúüÑÁÉÍÓÚÜ', # Q, W, X, and Y not used at all
wiki_start_pages=[u'Wikipedia:Portada']), use_ascii=False,
'Estonian': Language(name='Estonian', charsets=["ISO-8859-3"],
iso_code='et', alphabet="abcĉdefgĝhĥijĵklmnoprsŝtuŭvzABCĈDEFGĜHĤIJĴKLMNOPRSŜTUŬVZ",
use_ascii=False, wiki_start_pages=["Vikipedio:Ĉefpaĝo"],
charsets=['ISO-8859-4', 'ISO-8859-13', ),
'WINDOWS-1257'], "Spanish": Language(
# C, F, Š, Q, W, X, Y, Z, Ž are only for name="Spanish",
# loanwords iso_code="es",
alphabet=(u'ABDEGHIJKLMNOPRSTUVÕÄÖÜ' use_ascii=True,
u'abdeghijklmnoprstuvõäöü'), charsets=["ISO-8859-1", "ISO-8859-15", "WINDOWS-1252", "MacRoman"],
wiki_start_pages=[u'Esileht']), alphabet="ñáéíóúüÑÁÉÍÓÚÜ",
'Finnish': Language(name='Finnish', wiki_start_pages=["Wikipedia:Portada"],
iso_code='fi', ),
use_ascii=True, "Estonian": Language(
charsets=['ISO-8859-1', 'ISO-8859-15', name="Estonian",
'WINDOWS-1252'], iso_code="et",
alphabet=u'ÅÄÖŠŽåäöšž', use_ascii=False,
wiki_start_pages=[u'Wikipedia:Etusivu']), charsets=["ISO-8859-4", "ISO-8859-13", "WINDOWS-1257"],
'French': Language(name='French', # C, F, Š, Q, W, X, Y, Z, Ž are only for
iso_code='fr', # loanwords
use_ascii=True, alphabet="ABDEGHIJKLMNOPRSTUVÕÄÖÜabdeghijklmnoprstuvõäöü",
charsets=['ISO-8859-1', 'ISO-8859-15', wiki_start_pages=["Esileht"],
'WINDOWS-1252'], ),
alphabet=u'œàâçèéîïùûêŒÀÂÇÈÉÎÏÙÛÊ', "Finnish": Language(
wiki_start_pages=[u'Wikipédia:Accueil_principal', name="Finnish",
u'Bœuf (animal)']), iso_code="fi",
'Hebrew': Language(name='Hebrew', use_ascii=True,
iso_code='he', charsets=["ISO-8859-1", "ISO-8859-15", "WINDOWS-1252", "MacRoman"],
use_ascii=False, alphabet="ÅÄÖŠŽåäöšž",
charsets=['ISO-8859-8', 'WINDOWS-1255'], wiki_start_pages=["Wikipedia:Etusivu"],
alphabet=u'אבגדהוזחטיךכלםמןנסעףפץצקרשתװױײ', ),
wiki_start_pages=[u'עמוד_ראשי']), "French": Language(
'Croatian': Language(name='Croatian', name="French",
iso_code='hr', iso_code="fr",
# Q, W, X, Y are only used for foreign words. use_ascii=True,
use_ascii=False, charsets=["ISO-8859-1", "ISO-8859-15", "WINDOWS-1252", "MacRoman"],
charsets=['ISO-8859-2', 'WINDOWS-1250'], alphabet="œàâçèéîïùûêŒÀÂÇÈÉÎÏÙÛÊ",
alphabet=(u'abcčćdđefghijklmnoprsštuvzž' wiki_start_pages=["Wikipédia:Accueil_principal", "Bœuf (animal)"],
u'ABCČĆDĐEFGHIJKLMNOPRSŠTUVZŽ'), ),
wiki_start_pages=[u'Glavna_stranica']), "Hebrew": Language(
'Hungarian': Language(name='Hungarian', name="Hebrew",
iso_code='hu', iso_code="he",
# Q, W, X, Y are only used for foreign words. use_ascii=False,
use_ascii=False, charsets=["ISO-8859-8", "WINDOWS-1255"],
charsets=['ISO-8859-2', 'WINDOWS-1250'], alphabet="אבגדהוזחטיךכלםמןנסעףפץצקרשתװױײ",
alphabet=(u'abcdefghijklmnoprstuvzáéíóöőúüű' wiki_start_pages=["עמוד_ראשי"],
u'ABCDEFGHIJKLMNOPRSTUVZÁÉÍÓÖŐÚÜŰ'), ),
wiki_start_pages=[u'Kezdőlap']), "Croatian": Language(
'Italian': Language(name='Italian', name="Croatian",
iso_code='it', iso_code="hr",
use_ascii=True, # Q, W, X, Y are only used for foreign words.
charsets=['ISO-8859-1', 'ISO-8859-15', use_ascii=False,
'WINDOWS-1252'], charsets=["ISO-8859-2", "WINDOWS-1250"],
alphabet=u'ÀÈÉÌÒÓÙàèéìòóù', alphabet="abcčćdđefghijklmnoprsštuvzžABCČĆDĐEFGHIJKLMNOPRSŠTUVZŽ",
wiki_start_pages=[u'Pagina_principale']), wiki_start_pages=["Glavna_stranica"],
'Lithuanian': Language(name='Lithuanian', ),
iso_code='lt', "Hungarian": Language(
use_ascii=False, name="Hungarian",
charsets=['ISO-8859-13', 'WINDOWS-1257', iso_code="hu",
'ISO-8859-4'], # Q, W, X, Y are only used for foreign words.
# Q, W, and X not used at all use_ascii=False,
alphabet=(u'AĄBCČDEĘĖFGHIĮYJKLMNOPRSŠTUŲŪVZŽ' charsets=["ISO-8859-2", "WINDOWS-1250"],
u'bcčdeęėfghiįyjklmnoprsštuųūvzž'), alphabet="abcdefghijklmnoprstuvzáéíóöőúüűABCDEFGHIJKLMNOPRSTUVZÁÉÍÓÖŐÚÜŰ",
wiki_start_pages=[u'Pagrindinis_puslapis']), wiki_start_pages=["Kezdőlap"],
'Latvian': Language(name='Latvian', ),
iso_code='lv', "Italian": Language(
use_ascii=False, name="Italian",
charsets=['ISO-8859-13', 'WINDOWS-1257', iso_code="it",
'ISO-8859-4'], use_ascii=True,
# Q, W, X, Y are only for loanwords charsets=["ISO-8859-1", "ISO-8859-15", "WINDOWS-1252", "MacRoman"],
alphabet=(u'AĀBCČDEĒFGĢHIĪJKĶLĻMNŅOPRSŠTUŪVZŽ' alphabet="ÀÈÉÌÒÓÙàèéìòóù",
u'aābcčdeēfgģhiījkķlļmnņoprsštuūvzž'), wiki_start_pages=["Pagina_principale"],
wiki_start_pages=[u'Sākumlapa']), ),
'Macedonian': Language(name='Macedonian', "Lithuanian": Language(
iso_code='mk', name="Lithuanian",
use_ascii=False, iso_code="lt",
charsets=['ISO-8859-5', 'WINDOWS-1251', use_ascii=False,
'MacCyrillic', 'IBM855'], charsets=["ISO-8859-13", "WINDOWS-1257", "ISO-8859-4"],
alphabet=(u'АБВГДЃЕЖЗЅИЈКЛЉМНЊОПРСТЌУФХЦЧЏШ' # Q, W, and X not used at all
u'абвгдѓежзѕијклљмнњопрстќуфхцчџш'), alphabet="AĄBCČDEĘĖFGHIĮYJKLMNOPRSŠTUŲŪVZŽaąbcčdeęėfghiįyjklmnoprsštuųūvzž",
wiki_start_pages=[u'Главна_страница']), wiki_start_pages=["Pagrindinis_puslapis"],
'Dutch': Language(name='Dutch', ),
iso_code='nl', "Latvian": Language(
use_ascii=True, name="Latvian",
charsets=['ISO-8859-1', 'WINDOWS-1252'], iso_code="lv",
wiki_start_pages=[u'Hoofdpagina']), use_ascii=False,
'Polish': Language(name='Polish', charsets=["ISO-8859-13", "WINDOWS-1257", "ISO-8859-4"],
iso_code='pl', # Q, W, X, Y are only for loanwords
# Q and X are only used for foreign words. alphabet="AĀBCČDEĒFGĢHIĪJKĶLĻMNŅOPRSŠTUŪVZŽaābcčdeēfgģhiījkķlļmnņoprsštuūvzž",
use_ascii=False, wiki_start_pages=["Sākumlapa"],
charsets=['ISO-8859-2', 'WINDOWS-1250'], ),
alphabet=(u'AĄBCĆDEĘFGHIJKLŁMNŃOÓPRSŚTUWYZŹŻ' "Macedonian": Language(
u'aąbcćdeęfghijklłmnńoóprsśtuwyzźż'), name="Macedonian",
wiki_start_pages=[u'Wikipedia:Strona_główna']), iso_code="mk",
'Portuguese': Language(name='Portuguese', use_ascii=False,
iso_code='pt', charsets=["ISO-8859-5", "WINDOWS-1251", "MacCyrillic", "IBM855"],
use_ascii=True, alphabet="АБВГДЃЕЖЗЅИЈКЛЉМНЊОПРСТЌУФХЦЧЏШабвгдѓежзѕијклљмнњопрстќуфхцчџш",
charsets=['ISO-8859-1', 'ISO-8859-15', wiki_start_pages=["Главна_страница"],
'WINDOWS-1252'], ),
alphabet=u'ÁÂÃÀÇÉÊÍÓÔÕÚáâãàçéêíóôõú', "Dutch": Language(
wiki_start_pages=[u'Wikipédia:Página_principal']), name="Dutch",
'Romanian': Language(name='Romanian', iso_code="nl",
iso_code='ro', use_ascii=True,
use_ascii=True, charsets=["ISO-8859-1", "WINDOWS-1252", "MacRoman"],
charsets=['ISO-8859-2', 'WINDOWS-1250'], wiki_start_pages=["Hoofdpagina"],
alphabet=u'ăâîșțĂÂÎȘȚ', ),
wiki_start_pages=[u'Pagina_principală']), "Polish": Language(
'Russian': Language(name='Russian', name="Polish",
iso_code='ru', iso_code="pl",
use_ascii=False, # Q and X are only used for foreign words.
charsets=['ISO-8859-5', 'WINDOWS-1251', use_ascii=False,
'KOI8-R', 'MacCyrillic', 'IBM866', charsets=["ISO-8859-2", "WINDOWS-1250"],
'IBM855'], alphabet="AĄBCĆDEĘFGHIJKLŁMNŃOÓPRSŚTUWYZŹŻaąbcćdeęfghijklłmnńoóprsśtuwyzźż",
alphabet=(u'абвгдеёжзийклмнопрстуфхцчшщъыьэюя' wiki_start_pages=["Wikipedia:Strona_główna"],
u'АБВГДЕЁЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯ'), ),
wiki_start_pages=[u'Заглавная_страница']), "Portuguese": Language(
'Slovak': Language(name='Slovak', name="Portuguese",
iso_code='sk', iso_code="pt",
use_ascii=True, use_ascii=True,
charsets=['ISO-8859-2', 'WINDOWS-1250'], charsets=["ISO-8859-1", "ISO-8859-15", "WINDOWS-1252", "MacRoman"],
alphabet=u'áäčďéíĺľňóôŕšťúýžÁÄČĎÉÍĹĽŇÓÔŔŠŤÚÝŽ', alphabet="ÁÂÃÀÇÉÊÍÓÔÕÚáâãàçéêíóôõú",
wiki_start_pages=[u'Hlavná_stránka']), wiki_start_pages=["Wikipédia:Página_principal"],
'Slovene': Language(name='Slovene', ),
iso_code='sl', "Romanian": Language(
# Q, W, X, Y are only used for foreign words. name="Romanian",
use_ascii=False, iso_code="ro",
charsets=['ISO-8859-2', 'WINDOWS-1250'], use_ascii=True,
alphabet=(u'abcčdefghijklmnoprsštuvzž' charsets=["ISO-8859-2", "WINDOWS-1250"],
u'ABCČDEFGHIJKLMNOPRSŠTUVZŽ'), alphabet="ăâîșțĂÂÎȘȚ",
wiki_start_pages=[u'Glavna_stran']), wiki_start_pages=["Pagina_principală"],
# Serbian can be written in both Latin and Cyrillic, but there's no ),
# simple way to get the Latin alphabet pages from Wikipedia through "Russian": Language(
# the API, so for now we just support Cyrillic. name="Russian",
'Serbian': Language(name='Serbian', iso_code="ru",
iso_code='sr', use_ascii=False,
alphabet=(u'АБВГДЂЕЖЗИЈКЛЉМНЊОПРСТЋУФХЦЧЏШ' charsets=[
u'абвгдђежзијклљмнњопрстћуфхцчџш'), "ISO-8859-5",
charsets=['ISO-8859-5', 'WINDOWS-1251', "WINDOWS-1251",
'MacCyrillic', 'IBM855'], "KOI8-R",
wiki_start_pages=[u'Главна_страна']), "MacCyrillic",
'Thai': Language(name='Thai', "IBM866",
iso_code='th', "IBM855",
use_ascii=False, ],
charsets=['ISO-8859-11', 'TIS-620', 'CP874'], alphabet="абвгдеёжзийклмнопрстуфхцчшщъыьэюяАБВГДЕЁЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯ",
alphabet=u'กขฃคฅฆงจฉชซฌญฎฏฐฑฒณดตถทธนบปผฝพฟภมยรฤลฦวศษสหฬอฮฯะัาำิีึืฺุู฿เแโใไๅๆ็่้๊๋์ํ๎๏๐๑๒๓๔๕๖๗๘๙๚๛', wiki_start_pages=["Заглавная_страница"],
wiki_start_pages=[u'หน้าหลัก']), ),
'Turkish': Language(name='Turkish', "Slovak": Language(
iso_code='tr', name="Slovak",
# Q, W, and X are not used by Turkish iso_code="sk",
use_ascii=False, use_ascii=True,
charsets=['ISO-8859-3', 'ISO-8859-9', charsets=["ISO-8859-2", "WINDOWS-1250"],
'WINDOWS-1254'], alphabet="áäčďéíĺľňóôŕšťúýžÁÄČĎÉÍĹĽŇÓÔŔŠŤÚÝŽ",
alphabet=(u'abcçdefgğhıijklmnoöprsştuüvyzâîû' wiki_start_pages=["Hlavná_stránka"],
u'ABCÇDEFGĞHIİJKLMNOÖPRSŞTUÜVYZÂÎÛ'), ),
wiki_start_pages=[u'Ana_Sayfa']), "Slovene": Language(
'Vietnamese': Language(name='Vietnamese', name="Slovene",
iso_code='vi', iso_code="sl",
use_ascii=False, # Q, W, X, Y are only used for foreign words.
# Windows-1258 is the only common 8-bit use_ascii=False,
# Vietnamese encoding supported by Python. charsets=["ISO-8859-2", "WINDOWS-1250"],
# From Wikipedia: alphabet="abcčdefghijklmnoprsštuvzžABCČDEFGHIJKLMNOPRSŠTUVZŽ",
# For systems that lack support for Unicode, wiki_start_pages=["Glavna_stran"],
# dozens of 8-bit Vietnamese code pages are ),
# available.[1] The most common are VISCII # Serbian can be written in both Latin and Cyrillic, but there's no
# (TCVN 5712:1993), VPS, and Windows-1258.[3] # simple way to get the Latin alphabet pages from Wikipedia through
# Where ASCII is required, such as when # the API, so for now we just support Cyrillic.
# ensuring readability in plain text e-mail, "Serbian": Language(
# Vietnamese letters are often encoded name="Serbian",
# according to Vietnamese Quoted-Readable iso_code="sr",
# (VIQR) or VSCII Mnemonic (VSCII-MNEM),[4] alphabet="АБВГДЂЕЖЗИЈКЛЉМНЊОПРСТЋУФХЦЧЏШабвгдђежзијклљмнњопрстћуфхцчџш",
# though usage of either variable-width charsets=["ISO-8859-5", "WINDOWS-1251", "MacCyrillic", "IBM855"],
# scheme has declined dramatically following wiki_start_pages=["Главна_страна"],
# the adoption of Unicode on the World Wide ),
# Web. "Thai": Language(
charsets=['WINDOWS-1258'], name="Thai",
alphabet=(u'aăâbcdđeêghiklmnoôơpqrstuưvxy' iso_code="th",
u'AĂÂBCDĐEÊGHIKLMNOÔƠPQRSTUƯVXY'), use_ascii=False,
wiki_start_pages=[u'Chữ_Quốc_ngữ']), charsets=["ISO-8859-11", "TIS-620", "CP874"],
} alphabet="กขฃคฅฆงจฉชซฌญฎฏฐฑฒณดตถทธนบปผฝพฟภมยรฤลฦวศษสหฬอฮฯะัาำิีึืฺุู฿เแโใไๅๆ็่้๊๋์ํ๎๏๐๑๒๓๔๕๖๗๘๙๚๛",
wiki_start_pages=["หน้าหลัก"],
),
"Turkish": Language(
name="Turkish",
iso_code="tr",
# Q, W, and X are not used by Turkish
use_ascii=False,
charsets=["ISO-8859-3", "ISO-8859-9", "WINDOWS-1254"],
alphabet="abcçdefgğhıijklmnoöprsştuüvyzâîûABCÇDEFGĞHIİJKLMNOÖPRSŞTUÜVYZÂÎÛ",
wiki_start_pages=["Ana_Sayfa"],
),
"Vietnamese": Language(
name="Vietnamese",
iso_code="vi",
use_ascii=False,
# Windows-1258 is the only common 8-bit
# Vietnamese encoding supported by Python.
# From Wikipedia:
# For systems that lack support for Unicode,
# dozens of 8-bit Vietnamese code pages are
# available.[1] The most common are VISCII
# (TCVN 5712:1993), VPS, and Windows-1258.[3]
# Where ASCII is required, such as when
# ensuring readability in plain text e-mail,
# Vietnamese letters are often encoded
# according to Vietnamese Quoted-Readable
# (VIQR) or VSCII Mnemonic (VSCII-MNEM),[4]
# though usage of either variable-width
# scheme has declined dramatically following
# the adoption of Unicode on the World Wide
# Web.
charsets=["WINDOWS-1258"],
alphabet="aăâbcdđeêghiklmnoôơpqrstuưvxyAĂÂBCDĐEÊGHIKLMNOÔƠPQRSTUƯVXY",
wiki_start_pages=["Chữ_Quốc_ngữ"],
),
}
View File
+16
View File
@@ -0,0 +1,16 @@
from typing import TYPE_CHECKING, Optional
if TYPE_CHECKING:
# TypedDict was introduced in Python 3.8.
#
# TODO: Remove the else block and TYPE_CHECKING check when dropping support
# for Python 3.7.
from typing import TypedDict
class ResultDict(TypedDict):
encoding: Optional[str]
confidence: float
language: Optional[str]
else:
ResultDict = dict
+56 -39
View File
@@ -26,70 +26,77 @@
# 02110-1301 USA # 02110-1301 USA
######################### END LICENSE BLOCK ######################### ######################### END LICENSE BLOCK #########################
from collections import namedtuple from typing import Dict, List, NamedTuple, Optional, Union
from .charsetprober import CharSetProber from .charsetprober import CharSetProber
from .enums import CharacterCategory, ProbingState, SequenceLikelihood from .enums import CharacterCategory, ProbingState, SequenceLikelihood
SingleByteCharSetModel = namedtuple('SingleByteCharSetModel', class SingleByteCharSetModel(NamedTuple):
['charset_name', charset_name: str
'language', language: str
'char_to_order_map', char_to_order_map: Dict[int, int]
'language_model', language_model: Dict[int, Dict[int, int]]
'typical_positive_ratio', typical_positive_ratio: float
'keep_ascii_letters', keep_ascii_letters: bool
'alphabet']) alphabet: str
class SingleByteCharSetProber(CharSetProber): class SingleByteCharSetProber(CharSetProber):
SAMPLE_SIZE = 64 SAMPLE_SIZE = 64
SB_ENOUGH_REL_THRESHOLD = 1024 # 0.25 * SAMPLE_SIZE^2 SB_ENOUGH_REL_THRESHOLD = 1024 # 0.25 * SAMPLE_SIZE^2
POSITIVE_SHORTCUT_THRESHOLD = 0.95 POSITIVE_SHORTCUT_THRESHOLD = 0.95
NEGATIVE_SHORTCUT_THRESHOLD = 0.05 NEGATIVE_SHORTCUT_THRESHOLD = 0.05
def __init__(self, model, reversed=False, name_prober=None): def __init__(
super(SingleByteCharSetProber, self).__init__() self,
model: SingleByteCharSetModel,
is_reversed: bool = False,
name_prober: Optional[CharSetProber] = None,
) -> None:
super().__init__()
self._model = model self._model = model
# TRUE if we need to reverse every pair in the model lookup # TRUE if we need to reverse every pair in the model lookup
self._reversed = reversed self._reversed = is_reversed
# Optional auxiliary prober for name decision # Optional auxiliary prober for name decision
self._name_prober = name_prober self._name_prober = name_prober
self._last_order = None self._last_order = 255
self._seq_counters = None self._seq_counters: List[int] = []
self._total_seqs = None self._total_seqs = 0
self._total_char = None self._total_char = 0
self._freq_char = None self._control_char = 0
self._freq_char = 0
self.reset() self.reset()
def reset(self): def reset(self) -> None:
super(SingleByteCharSetProber, self).reset() super().reset()
# char order of last character # char order of last character
self._last_order = 255 self._last_order = 255
self._seq_counters = [0] * SequenceLikelihood.get_num_categories() self._seq_counters = [0] * SequenceLikelihood.get_num_categories()
self._total_seqs = 0 self._total_seqs = 0
self._total_char = 0 self._total_char = 0
self._control_char = 0
# characters that fall in our sampling range # characters that fall in our sampling range
self._freq_char = 0 self._freq_char = 0
@property @property
def charset_name(self): def charset_name(self) -> Optional[str]:
if self._name_prober: if self._name_prober:
return self._name_prober.charset_name return self._name_prober.charset_name
else: return self._model.charset_name
return self._model.charset_name
@property @property
def language(self): def language(self) -> Optional[str]:
if self._name_prober: if self._name_prober:
return self._name_prober.language return self._name_prober.language
else: return self._model.language
return self._model.language
def feed(self, byte_str): def feed(self, byte_str: Union[bytes, bytearray]) -> ProbingState:
# TODO: Make filter_international_words keep things in self.alphabet # TODO: Make filter_international_words keep things in self.alphabet
if not self._model.keep_ascii_letters: if not self._model.keep_ascii_letters:
byte_str = self.filter_international_words(byte_str) byte_str = self.filter_international_words(byte_str)
else:
byte_str = self.remove_xml_tags(byte_str)
if not byte_str: if not byte_str:
return self.state return self.state
char_to_order_map = self._model.char_to_order_map char_to_order_map = self._model.char_to_order_map
@@ -103,9 +110,6 @@ class SingleByteCharSetProber(CharSetProber):
# _total_char purposes. # _total_char purposes.
if order < CharacterCategory.CONTROL: if order < CharacterCategory.CONTROL:
self._total_char += 1 self._total_char += 1
# TODO: Follow uchardet's lead and discount confidence for frequent
# control characters.
# See https://github.com/BYVoid/uchardet/commit/55b4f23971db61
if order < self.SAMPLE_SIZE: if order < self.SAMPLE_SIZE:
self._freq_char += 1 self._freq_char += 1
if self._last_order < self.SAMPLE_SIZE: if self._last_order < self.SAMPLE_SIZE:
@@ -122,23 +126,36 @@ class SingleByteCharSetProber(CharSetProber):
if self._total_seqs > self.SB_ENOUGH_REL_THRESHOLD: if self._total_seqs > self.SB_ENOUGH_REL_THRESHOLD:
confidence = self.get_confidence() confidence = self.get_confidence()
if confidence > self.POSITIVE_SHORTCUT_THRESHOLD: if confidence > self.POSITIVE_SHORTCUT_THRESHOLD:
self.logger.debug('%s confidence = %s, we have a winner', self.logger.debug(
charset_name, confidence) "%s confidence = %s, we have a winner", charset_name, confidence
)
self._state = ProbingState.FOUND_IT self._state = ProbingState.FOUND_IT
elif confidence < self.NEGATIVE_SHORTCUT_THRESHOLD: elif confidence < self.NEGATIVE_SHORTCUT_THRESHOLD:
self.logger.debug('%s confidence = %s, below negative ' self.logger.debug(
'shortcut threshhold %s', charset_name, "%s confidence = %s, below negative shortcut threshold %s",
confidence, charset_name,
self.NEGATIVE_SHORTCUT_THRESHOLD) confidence,
self.NEGATIVE_SHORTCUT_THRESHOLD,
)
self._state = ProbingState.NOT_ME self._state = ProbingState.NOT_ME
return self.state return self.state
def get_confidence(self): def get_confidence(self) -> float:
r = 0.01 r = 0.01
if self._total_seqs > 0: if self._total_seqs > 0:
r = ((1.0 * self._seq_counters[SequenceLikelihood.POSITIVE]) / r = (
self._total_seqs / self._model.typical_positive_ratio) (
self._seq_counters[SequenceLikelihood.POSITIVE]
+ 0.25 * self._seq_counters[SequenceLikelihood.LIKELY]
)
/ self._total_seqs
/ self._model.typical_positive_ratio
)
# The more control characters (proportionnaly to the size
# of the text), the less confident we become in the current
# charset.
r = r * (self._total_char - self._control_char) / self._total_char
r = r * self._freq_char / self._total_char r = r * self._freq_char / self._total_char
if r >= 1.0: if r >= 1.0:
r = 0.99 r = 0.99
+19 -14
View File
@@ -28,33 +28,38 @@
from .charsetgroupprober import CharSetGroupProber from .charsetgroupprober import CharSetGroupProber
from .hebrewprober import HebrewProber from .hebrewprober import HebrewProber
from .langbulgarianmodel import (ISO_8859_5_BULGARIAN_MODEL, from .langbulgarianmodel import ISO_8859_5_BULGARIAN_MODEL, WINDOWS_1251_BULGARIAN_MODEL
WINDOWS_1251_BULGARIAN_MODEL)
from .langgreekmodel import ISO_8859_7_GREEK_MODEL, WINDOWS_1253_GREEK_MODEL from .langgreekmodel import ISO_8859_7_GREEK_MODEL, WINDOWS_1253_GREEK_MODEL
from .langhebrewmodel import WINDOWS_1255_HEBREW_MODEL from .langhebrewmodel import WINDOWS_1255_HEBREW_MODEL
# from .langhungarianmodel import (ISO_8859_2_HUNGARIAN_MODEL, # from .langhungarianmodel import (ISO_8859_2_HUNGARIAN_MODEL,
# WINDOWS_1250_HUNGARIAN_MODEL) # WINDOWS_1250_HUNGARIAN_MODEL)
from .langrussianmodel import (IBM855_RUSSIAN_MODEL, IBM866_RUSSIAN_MODEL, from .langrussianmodel import (
ISO_8859_5_RUSSIAN_MODEL, KOI8_R_RUSSIAN_MODEL, IBM855_RUSSIAN_MODEL,
MACCYRILLIC_RUSSIAN_MODEL, IBM866_RUSSIAN_MODEL,
WINDOWS_1251_RUSSIAN_MODEL) ISO_8859_5_RUSSIAN_MODEL,
KOI8_R_RUSSIAN_MODEL,
MACCYRILLIC_RUSSIAN_MODEL,
WINDOWS_1251_RUSSIAN_MODEL,
)
from .langthaimodel import TIS_620_THAI_MODEL from .langthaimodel import TIS_620_THAI_MODEL
from .langturkishmodel import ISO_8859_9_TURKISH_MODEL from .langturkishmodel import ISO_8859_9_TURKISH_MODEL
from .sbcharsetprober import SingleByteCharSetProber from .sbcharsetprober import SingleByteCharSetProber
class SBCSGroupProber(CharSetGroupProber): class SBCSGroupProber(CharSetGroupProber):
def __init__(self): def __init__(self) -> None:
super(SBCSGroupProber, self).__init__() super().__init__()
hebrew_prober = HebrewProber() hebrew_prober = HebrewProber()
logical_hebrew_prober = SingleByteCharSetProber(WINDOWS_1255_HEBREW_MODEL, logical_hebrew_prober = SingleByteCharSetProber(
False, hebrew_prober) WINDOWS_1255_HEBREW_MODEL, is_reversed=False, name_prober=hebrew_prober
)
# TODO: See if using ISO-8859-8 Hebrew model works better here, since # TODO: See if using ISO-8859-8 Hebrew model works better here, since
# it's actually the visual one # it's actually the visual one
visual_hebrew_prober = SingleByteCharSetProber(WINDOWS_1255_HEBREW_MODEL, visual_hebrew_prober = SingleByteCharSetProber(
True, hebrew_prober) WINDOWS_1255_HEBREW_MODEL, is_reversed=True, name_prober=hebrew_prober
hebrew_prober.set_model_probers(logical_hebrew_prober, )
visual_hebrew_prober) hebrew_prober.set_model_probers(logical_hebrew_prober, visual_hebrew_prober)
# TODO: ORDER MATTERS HERE. I changed the order vs what was in master # TODO: ORDER MATTERS HERE. I changed the order vs what was in master
# and several tests failed that did not before. Some thought # and several tests failed that did not before. Some thought
# should be put into the ordering, and we should consider making # should be put into the ordering, and we should consider making
+39 -26
View File
@@ -25,68 +25,81 @@
# 02110-1301 USA # 02110-1301 USA
######################### END LICENSE BLOCK ######################### ######################### END LICENSE BLOCK #########################
from .mbcharsetprober import MultiByteCharSetProber from typing import Union
from .codingstatemachine import CodingStateMachine
from .chardistribution import SJISDistributionAnalysis from .chardistribution import SJISDistributionAnalysis
from .codingstatemachine import CodingStateMachine
from .enums import MachineState, ProbingState
from .jpcntx import SJISContextAnalysis from .jpcntx import SJISContextAnalysis
from .mbcharsetprober import MultiByteCharSetProber
from .mbcssm import SJIS_SM_MODEL from .mbcssm import SJIS_SM_MODEL
from .enums import ProbingState, MachineState
class SJISProber(MultiByteCharSetProber): class SJISProber(MultiByteCharSetProber):
def __init__(self): def __init__(self) -> None:
super(SJISProber, self).__init__() super().__init__()
self.coding_sm = CodingStateMachine(SJIS_SM_MODEL) self.coding_sm = CodingStateMachine(SJIS_SM_MODEL)
self.distribution_analyzer = SJISDistributionAnalysis() self.distribution_analyzer = SJISDistributionAnalysis()
self.context_analyzer = SJISContextAnalysis() self.context_analyzer = SJISContextAnalysis()
self.reset() self.reset()
def reset(self): def reset(self) -> None:
super(SJISProber, self).reset() super().reset()
self.context_analyzer.reset() self.context_analyzer.reset()
@property @property
def charset_name(self): def charset_name(self) -> str:
return self.context_analyzer.charset_name return self.context_analyzer.charset_name
@property @property
def language(self): def language(self) -> str:
return "Japanese" return "Japanese"
def feed(self, byte_str): def feed(self, byte_str: Union[bytes, bytearray]) -> ProbingState:
for i in range(len(byte_str)): assert self.coding_sm is not None
coding_state = self.coding_sm.next_state(byte_str[i]) assert self.distribution_analyzer is not None
for i, byte in enumerate(byte_str):
coding_state = self.coding_sm.next_state(byte)
if coding_state == MachineState.ERROR: if coding_state == MachineState.ERROR:
self.logger.debug('%s %s prober hit error at byte %s', self.logger.debug(
self.charset_name, self.language, i) "%s %s prober hit error at byte %s",
self.charset_name,
self.language,
i,
)
self._state = ProbingState.NOT_ME self._state = ProbingState.NOT_ME
break break
elif coding_state == MachineState.ITS_ME: if coding_state == MachineState.ITS_ME:
self._state = ProbingState.FOUND_IT self._state = ProbingState.FOUND_IT
break break
elif coding_state == MachineState.START: if coding_state == MachineState.START:
char_len = self.coding_sm.get_current_charlen() char_len = self.coding_sm.get_current_charlen()
if i == 0: if i == 0:
self._last_char[1] = byte_str[0] self._last_char[1] = byte
self.context_analyzer.feed(self._last_char[2 - char_len:], self.context_analyzer.feed(
char_len) self._last_char[2 - char_len :], char_len
)
self.distribution_analyzer.feed(self._last_char, char_len) self.distribution_analyzer.feed(self._last_char, char_len)
else: else:
self.context_analyzer.feed(byte_str[i + 1 - char_len:i + 3 self.context_analyzer.feed(
- char_len], char_len) byte_str[i + 1 - char_len : i + 3 - char_len], char_len
self.distribution_analyzer.feed(byte_str[i - 1:i + 1], )
char_len) self.distribution_analyzer.feed(byte_str[i - 1 : i + 1], char_len)
self._last_char[0] = byte_str[-1] self._last_char[0] = byte_str[-1]
if self.state == ProbingState.DETECTING: if self.state == ProbingState.DETECTING:
if (self.context_analyzer.got_enough_data() and if self.context_analyzer.got_enough_data() and (
(self.get_confidence() > self.SHORTCUT_THRESHOLD)): self.get_confidence() > self.SHORTCUT_THRESHOLD
):
self._state = ProbingState.FOUND_IT self._state = ProbingState.FOUND_IT
return self.state return self.state
def get_confidence(self): def get_confidence(self) -> float:
assert self.distribution_analyzer is not None
context_conf = self.context_analyzer.get_confidence() context_conf = self.context_analyzer.get_confidence()
distrib_conf = self.distribution_analyzer.get_confidence() distrib_conf = self.distribution_analyzer.get_confidence()
return max(context_conf, distrib_conf) return max(context_conf, distrib_conf)
+155 -79
View File
@@ -39,16 +39,21 @@ class a user of ``chardet`` should use.
import codecs import codecs
import logging import logging
import re import re
from typing import List, Optional, Union
from .charsetgroupprober import CharSetGroupProber from .charsetgroupprober import CharSetGroupProber
from .charsetprober import CharSetProber
from .enums import InputState, LanguageFilter, ProbingState from .enums import InputState, LanguageFilter, ProbingState
from .escprober import EscCharSetProber from .escprober import EscCharSetProber
from .latin1prober import Latin1Prober from .latin1prober import Latin1Prober
from .macromanprober import MacRomanProber
from .mbcsgroupprober import MBCSGroupProber from .mbcsgroupprober import MBCSGroupProber
from .resultdict import ResultDict
from .sbcsgroupprober import SBCSGroupProber from .sbcsgroupprober import SBCSGroupProber
from .utf1632prober import UTF1632Prober
class UniversalDetector(object): class UniversalDetector:
""" """
The ``UniversalDetector`` class underlies the ``chardet.detect`` function The ``UniversalDetector`` class underlies the ``chardet.detect`` function
and coordinates all of the different charset probers. and coordinates all of the different charset probers.
@@ -66,49 +71,87 @@ class UniversalDetector(object):
""" """
MINIMUM_THRESHOLD = 0.20 MINIMUM_THRESHOLD = 0.20
HIGH_BYTE_DETECTOR = re.compile(b'[\x80-\xFF]') HIGH_BYTE_DETECTOR = re.compile(b"[\x80-\xFF]")
ESC_DETECTOR = re.compile(b'(\033|~{)') ESC_DETECTOR = re.compile(b"(\033|~{)")
WIN_BYTE_DETECTOR = re.compile(b'[\x80-\x9F]') WIN_BYTE_DETECTOR = re.compile(b"[\x80-\x9F]")
ISO_WIN_MAP = {'iso-8859-1': 'Windows-1252', ISO_WIN_MAP = {
'iso-8859-2': 'Windows-1250', "iso-8859-1": "Windows-1252",
'iso-8859-5': 'Windows-1251', "iso-8859-2": "Windows-1250",
'iso-8859-6': 'Windows-1256', "iso-8859-5": "Windows-1251",
'iso-8859-7': 'Windows-1253', "iso-8859-6": "Windows-1256",
'iso-8859-8': 'Windows-1255', "iso-8859-7": "Windows-1253",
'iso-8859-9': 'Windows-1254', "iso-8859-8": "Windows-1255",
'iso-8859-13': 'Windows-1257'} "iso-8859-9": "Windows-1254",
"iso-8859-13": "Windows-1257",
}
# Based on https://encoding.spec.whatwg.org/#names-and-labels
# but altered to match Python names for encodings and remove mappings
# that break tests.
LEGACY_MAP = {
"ascii": "Windows-1252",
"iso-8859-1": "Windows-1252",
"tis-620": "ISO-8859-11",
"iso-8859-9": "Windows-1254",
"gb2312": "GB18030",
"euc-kr": "CP949",
"utf-16le": "UTF-16",
}
def __init__(self, lang_filter=LanguageFilter.ALL): def __init__(
self._esc_charset_prober = None self,
self._charset_probers = [] lang_filter: LanguageFilter = LanguageFilter.ALL,
self.result = None should_rename_legacy: bool = False,
self.done = None ) -> None:
self._got_data = None self._esc_charset_prober: Optional[EscCharSetProber] = None
self._input_state = None self._utf1632_prober: Optional[UTF1632Prober] = None
self._last_char = None self._charset_probers: List[CharSetProber] = []
self.result: ResultDict = {
"encoding": None,
"confidence": 0.0,
"language": None,
}
self.done = False
self._got_data = False
self._input_state = InputState.PURE_ASCII
self._last_char = b""
self.lang_filter = lang_filter self.lang_filter = lang_filter
self.logger = logging.getLogger(__name__) self.logger = logging.getLogger(__name__)
self._has_win_bytes = None self._has_win_bytes = False
self.should_rename_legacy = should_rename_legacy
self.reset() self.reset()
def reset(self): @property
def input_state(self) -> int:
return self._input_state
@property
def has_win_bytes(self) -> bool:
return self._has_win_bytes
@property
def charset_probers(self) -> List[CharSetProber]:
return self._charset_probers
def reset(self) -> None:
""" """
Reset the UniversalDetector and all of its probers back to their Reset the UniversalDetector and all of its probers back to their
initial states. This is called by ``__init__``, so you only need to initial states. This is called by ``__init__``, so you only need to
call this directly in between analyses of different documents. call this directly in between analyses of different documents.
""" """
self.result = {'encoding': None, 'confidence': 0.0, 'language': None} self.result = {"encoding": None, "confidence": 0.0, "language": None}
self.done = False self.done = False
self._got_data = False self._got_data = False
self._has_win_bytes = False self._has_win_bytes = False
self._input_state = InputState.PURE_ASCII self._input_state = InputState.PURE_ASCII
self._last_char = b'' self._last_char = b""
if self._esc_charset_prober: if self._esc_charset_prober:
self._esc_charset_prober.reset() self._esc_charset_prober.reset()
if self._utf1632_prober:
self._utf1632_prober.reset()
for prober in self._charset_probers: for prober in self._charset_probers:
prober.reset() prober.reset()
def feed(self, byte_str): def feed(self, byte_str: Union[bytes, bytearray]) -> None:
""" """
Takes a chunk of a document and feeds it through all of the relevant Takes a chunk of a document and feeds it through all of the relevant
charset probers. charset probers.
@@ -125,7 +168,7 @@ class UniversalDetector(object):
if self.done: if self.done:
return return
if not len(byte_str): if not byte_str:
return return
if not isinstance(byte_str, bytearray): if not isinstance(byte_str, bytearray):
@@ -136,35 +179,38 @@ class UniversalDetector(object):
# If the data starts with BOM, we know it is UTF # If the data starts with BOM, we know it is UTF
if byte_str.startswith(codecs.BOM_UTF8): if byte_str.startswith(codecs.BOM_UTF8):
# EF BB BF UTF-8 with BOM # EF BB BF UTF-8 with BOM
self.result = {'encoding': "UTF-8-SIG", self.result = {
'confidence': 1.0, "encoding": "UTF-8-SIG",
'language': ''} "confidence": 1.0,
elif byte_str.startswith((codecs.BOM_UTF32_LE, "language": "",
codecs.BOM_UTF32_BE)): }
elif byte_str.startswith((codecs.BOM_UTF32_LE, codecs.BOM_UTF32_BE)):
# FF FE 00 00 UTF-32, little-endian BOM # FF FE 00 00 UTF-32, little-endian BOM
# 00 00 FE FF UTF-32, big-endian BOM # 00 00 FE FF UTF-32, big-endian BOM
self.result = {'encoding': "UTF-32", self.result = {"encoding": "UTF-32", "confidence": 1.0, "language": ""}
'confidence': 1.0, elif byte_str.startswith(b"\xFE\xFF\x00\x00"):
'language': ''}
elif byte_str.startswith(b'\xFE\xFF\x00\x00'):
# FE FF 00 00 UCS-4, unusual octet order BOM (3412) # FE FF 00 00 UCS-4, unusual octet order BOM (3412)
self.result = {'encoding': "X-ISO-10646-UCS-4-3412", self.result = {
'confidence': 1.0, # TODO: This encoding is not supported by Python. Should remove?
'language': ''} "encoding": "X-ISO-10646-UCS-4-3412",
elif byte_str.startswith(b'\x00\x00\xFF\xFE'): "confidence": 1.0,
"language": "",
}
elif byte_str.startswith(b"\x00\x00\xFF\xFE"):
# 00 00 FF FE UCS-4, unusual octet order BOM (2143) # 00 00 FF FE UCS-4, unusual octet order BOM (2143)
self.result = {'encoding': "X-ISO-10646-UCS-4-2143", self.result = {
'confidence': 1.0, # TODO: This encoding is not supported by Python. Should remove?
'language': ''} "encoding": "X-ISO-10646-UCS-4-2143",
"confidence": 1.0,
"language": "",
}
elif byte_str.startswith((codecs.BOM_LE, codecs.BOM_BE)): elif byte_str.startswith((codecs.BOM_LE, codecs.BOM_BE)):
# FF FE UTF-16, little endian BOM # FF FE UTF-16, little endian BOM
# FE FF UTF-16, big endian BOM # FE FF UTF-16, big endian BOM
self.result = {'encoding': "UTF-16", self.result = {"encoding": "UTF-16", "confidence": 1.0, "language": ""}
'confidence': 1.0,
'language': ''}
self._got_data = True self._got_data = True
if self.result['encoding'] is not None: if self.result["encoding"] is not None:
self.done = True self.done = True
return return
@@ -173,12 +219,29 @@ class UniversalDetector(object):
if self._input_state == InputState.PURE_ASCII: if self._input_state == InputState.PURE_ASCII:
if self.HIGH_BYTE_DETECTOR.search(byte_str): if self.HIGH_BYTE_DETECTOR.search(byte_str):
self._input_state = InputState.HIGH_BYTE self._input_state = InputState.HIGH_BYTE
elif self._input_state == InputState.PURE_ASCII and \ elif (
self.ESC_DETECTOR.search(self._last_char + byte_str): self._input_state == InputState.PURE_ASCII
and self.ESC_DETECTOR.search(self._last_char + byte_str)
):
self._input_state = InputState.ESC_ASCII self._input_state = InputState.ESC_ASCII
self._last_char = byte_str[-1:] self._last_char = byte_str[-1:]
# next we will look to see if it is appears to be either a UTF-16 or
# UTF-32 encoding
if not self._utf1632_prober:
self._utf1632_prober = UTF1632Prober()
if self._utf1632_prober.state == ProbingState.DETECTING:
if self._utf1632_prober.feed(byte_str) == ProbingState.FOUND_IT:
self.result = {
"encoding": self._utf1632_prober.charset_name,
"confidence": self._utf1632_prober.get_confidence(),
"language": "",
}
self.done = True
return
# If we've seen escape sequences, use the EscCharSetProber, which # If we've seen escape sequences, use the EscCharSetProber, which
# uses a simple state machine to check for known escape sequences in # uses a simple state machine to check for known escape sequences in
# HZ and ISO-2022 encodings, since those are the only encodings that # HZ and ISO-2022 encodings, since those are the only encodings that
@@ -187,12 +250,11 @@ class UniversalDetector(object):
if not self._esc_charset_prober: if not self._esc_charset_prober:
self._esc_charset_prober = EscCharSetProber(self.lang_filter) self._esc_charset_prober = EscCharSetProber(self.lang_filter)
if self._esc_charset_prober.feed(byte_str) == ProbingState.FOUND_IT: if self._esc_charset_prober.feed(byte_str) == ProbingState.FOUND_IT:
self.result = {'encoding': self.result = {
self._esc_charset_prober.charset_name, "encoding": self._esc_charset_prober.charset_name,
'confidence': "confidence": self._esc_charset_prober.get_confidence(),
self._esc_charset_prober.get_confidence(), "language": self._esc_charset_prober.language,
'language': }
self._esc_charset_prober.language}
self.done = True self.done = True
# If we've seen high bytes (i.e., those with values greater than 127), # If we've seen high bytes (i.e., those with values greater than 127),
# we need to do more complicated checks using all our multi-byte and # we need to do more complicated checks using all our multi-byte and
@@ -207,17 +269,20 @@ class UniversalDetector(object):
if self.lang_filter & LanguageFilter.NON_CJK: if self.lang_filter & LanguageFilter.NON_CJK:
self._charset_probers.append(SBCSGroupProber()) self._charset_probers.append(SBCSGroupProber())
self._charset_probers.append(Latin1Prober()) self._charset_probers.append(Latin1Prober())
self._charset_probers.append(MacRomanProber())
for prober in self._charset_probers: for prober in self._charset_probers:
if prober.feed(byte_str) == ProbingState.FOUND_IT: if prober.feed(byte_str) == ProbingState.FOUND_IT:
self.result = {'encoding': prober.charset_name, self.result = {
'confidence': prober.get_confidence(), "encoding": prober.charset_name,
'language': prober.language} "confidence": prober.get_confidence(),
"language": prober.language,
}
self.done = True self.done = True
break break
if self.WIN_BYTE_DETECTOR.search(byte_str): if self.WIN_BYTE_DETECTOR.search(byte_str):
self._has_win_bytes = True self._has_win_bytes = True
def close(self): def close(self) -> ResultDict:
""" """
Stop analyzing the current document and come up with a final Stop analyzing the current document and come up with a final
prediction. prediction.
@@ -231,13 +296,11 @@ class UniversalDetector(object):
self.done = True self.done = True
if not self._got_data: if not self._got_data:
self.logger.debug('no data received!') self.logger.debug("no data received!")
# Default to ASCII if it is all we've seen so far # Default to ASCII if it is all we've seen so far
elif self._input_state == InputState.PURE_ASCII: elif self._input_state == InputState.PURE_ASCII:
self.result = {'encoding': 'ascii', self.result = {"encoding": "ascii", "confidence": 1.0, "language": ""}
'confidence': 1.0,
'language': ''}
# If we have seen non-ASCII, return the best that met MINIMUM_THRESHOLD # If we have seen non-ASCII, return the best that met MINIMUM_THRESHOLD
elif self._input_state == InputState.HIGH_BYTE: elif self._input_state == InputState.HIGH_BYTE:
@@ -253,34 +316,47 @@ class UniversalDetector(object):
max_prober = prober max_prober = prober
if max_prober and (max_prober_confidence > self.MINIMUM_THRESHOLD): if max_prober and (max_prober_confidence > self.MINIMUM_THRESHOLD):
charset_name = max_prober.charset_name charset_name = max_prober.charset_name
lower_charset_name = max_prober.charset_name.lower() assert charset_name is not None
lower_charset_name = charset_name.lower()
confidence = max_prober.get_confidence() confidence = max_prober.get_confidence()
# Use Windows encoding name instead of ISO-8859 if we saw any # Use Windows encoding name instead of ISO-8859 if we saw any
# extra Windows-specific bytes # extra Windows-specific bytes
if lower_charset_name.startswith('iso-8859'): if lower_charset_name.startswith("iso-8859"):
if self._has_win_bytes: if self._has_win_bytes:
charset_name = self.ISO_WIN_MAP.get(lower_charset_name, charset_name = self.ISO_WIN_MAP.get(
charset_name) lower_charset_name, charset_name
self.result = {'encoding': charset_name, )
'confidence': confidence, # Rename legacy encodings with superset encodings if asked
'language': max_prober.language} if self.should_rename_legacy:
charset_name = self.LEGACY_MAP.get(
(charset_name or "").lower(), charset_name
)
self.result = {
"encoding": charset_name,
"confidence": confidence,
"language": max_prober.language,
}
# Log all prober confidences if none met MINIMUM_THRESHOLD # Log all prober confidences if none met MINIMUM_THRESHOLD
if self.logger.getEffectiveLevel() <= logging.DEBUG: if self.logger.getEffectiveLevel() <= logging.DEBUG:
if self.result['encoding'] is None: if self.result["encoding"] is None:
self.logger.debug('no probers hit minimum threshold') self.logger.debug("no probers hit minimum threshold")
for group_prober in self._charset_probers: for group_prober in self._charset_probers:
if not group_prober: if not group_prober:
continue continue
if isinstance(group_prober, CharSetGroupProber): if isinstance(group_prober, CharSetGroupProber):
for prober in group_prober.probers: for prober in group_prober.probers:
self.logger.debug('%s %s confidence = %s', self.logger.debug(
prober.charset_name, "%s %s confidence = %s",
prober.language, prober.charset_name,
prober.get_confidence()) prober.language,
prober.get_confidence(),
)
else: else:
self.logger.debug('%s %s confidence = %s', self.logger.debug(
group_prober.charset_name, "%s %s confidence = %s",
group_prober.language, group_prober.charset_name,
group_prober.get_confidence()) group_prober.language,
group_prober.get_confidence(),
)
return self.result return self.result
+225
View File
@@ -0,0 +1,225 @@
######################## BEGIN LICENSE BLOCK ########################
#
# Contributor(s):
# Jason Zavaglia
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
# 02110-1301 USA
######################### END LICENSE BLOCK #########################
from typing import List, Union
from .charsetprober import CharSetProber
from .enums import ProbingState
class UTF1632Prober(CharSetProber):
"""
This class simply looks for occurrences of zero bytes, and infers
whether the file is UTF16 or UTF32 (low-endian or big-endian)
For instance, files looking like ( \0 \0 \0 [nonzero] )+
have a good probability to be UTF32BE. Files looking like ( \0 [nonzero] )+
may be guessed to be UTF16BE, and inversely for little-endian varieties.
"""
# how many logical characters to scan before feeling confident of prediction
MIN_CHARS_FOR_DETECTION = 20
# a fixed constant ratio of expected zeros or non-zeros in modulo-position.
EXPECTED_RATIO = 0.94
def __init__(self) -> None:
super().__init__()
self.position = 0
self.zeros_at_mod = [0] * 4
self.nonzeros_at_mod = [0] * 4
self._state = ProbingState.DETECTING
self.quad = [0, 0, 0, 0]
self.invalid_utf16be = False
self.invalid_utf16le = False
self.invalid_utf32be = False
self.invalid_utf32le = False
self.first_half_surrogate_pair_detected_16be = False
self.first_half_surrogate_pair_detected_16le = False
self.reset()
def reset(self) -> None:
super().reset()
self.position = 0
self.zeros_at_mod = [0] * 4
self.nonzeros_at_mod = [0] * 4
self._state = ProbingState.DETECTING
self.invalid_utf16be = False
self.invalid_utf16le = False
self.invalid_utf32be = False
self.invalid_utf32le = False
self.first_half_surrogate_pair_detected_16be = False
self.first_half_surrogate_pair_detected_16le = False
self.quad = [0, 0, 0, 0]
@property
def charset_name(self) -> str:
if self.is_likely_utf32be():
return "utf-32be"
if self.is_likely_utf32le():
return "utf-32le"
if self.is_likely_utf16be():
return "utf-16be"
if self.is_likely_utf16le():
return "utf-16le"
# default to something valid
return "utf-16"
@property
def language(self) -> str:
return ""
def approx_32bit_chars(self) -> float:
return max(1.0, self.position / 4.0)
def approx_16bit_chars(self) -> float:
return max(1.0, self.position / 2.0)
def is_likely_utf32be(self) -> bool:
approx_chars = self.approx_32bit_chars()
return approx_chars >= self.MIN_CHARS_FOR_DETECTION and (
self.zeros_at_mod[0] / approx_chars > self.EXPECTED_RATIO
and self.zeros_at_mod[1] / approx_chars > self.EXPECTED_RATIO
and self.zeros_at_mod[2] / approx_chars > self.EXPECTED_RATIO
and self.nonzeros_at_mod[3] / approx_chars > self.EXPECTED_RATIO
and not self.invalid_utf32be
)
def is_likely_utf32le(self) -> bool:
approx_chars = self.approx_32bit_chars()
return approx_chars >= self.MIN_CHARS_FOR_DETECTION and (
self.nonzeros_at_mod[0] / approx_chars > self.EXPECTED_RATIO
and self.zeros_at_mod[1] / approx_chars > self.EXPECTED_RATIO
and self.zeros_at_mod[2] / approx_chars > self.EXPECTED_RATIO
and self.zeros_at_mod[3] / approx_chars > self.EXPECTED_RATIO
and not self.invalid_utf32le
)
def is_likely_utf16be(self) -> bool:
approx_chars = self.approx_16bit_chars()
return approx_chars >= self.MIN_CHARS_FOR_DETECTION and (
(self.nonzeros_at_mod[1] + self.nonzeros_at_mod[3]) / approx_chars
> self.EXPECTED_RATIO
and (self.zeros_at_mod[0] + self.zeros_at_mod[2]) / approx_chars
> self.EXPECTED_RATIO
and not self.invalid_utf16be
)
def is_likely_utf16le(self) -> bool:
approx_chars = self.approx_16bit_chars()
return approx_chars >= self.MIN_CHARS_FOR_DETECTION and (
(self.nonzeros_at_mod[0] + self.nonzeros_at_mod[2]) / approx_chars
> self.EXPECTED_RATIO
and (self.zeros_at_mod[1] + self.zeros_at_mod[3]) / approx_chars
> self.EXPECTED_RATIO
and not self.invalid_utf16le
)
def validate_utf32_characters(self, quad: List[int]) -> None:
"""
Validate if the quad of bytes is valid UTF-32.
UTF-32 is valid in the range 0x00000000 - 0x0010FFFF
excluding 0x0000D800 - 0x0000DFFF
https://en.wikipedia.org/wiki/UTF-32
"""
if (
quad[0] != 0
or quad[1] > 0x10
or (quad[0] == 0 and quad[1] == 0 and 0xD8 <= quad[2] <= 0xDF)
):
self.invalid_utf32be = True
if (
quad[3] != 0
or quad[2] > 0x10
or (quad[3] == 0 and quad[2] == 0 and 0xD8 <= quad[1] <= 0xDF)
):
self.invalid_utf32le = True
def validate_utf16_characters(self, pair: List[int]) -> None:
"""
Validate if the pair of bytes is valid UTF-16.
UTF-16 is valid in the range 0x0000 - 0xFFFF excluding 0xD800 - 0xFFFF
with an exception for surrogate pairs, which must be in the range
0xD800-0xDBFF followed by 0xDC00-0xDFFF
https://en.wikipedia.org/wiki/UTF-16
"""
if not self.first_half_surrogate_pair_detected_16be:
if 0xD8 <= pair[0] <= 0xDB:
self.first_half_surrogate_pair_detected_16be = True
elif 0xDC <= pair[0] <= 0xDF:
self.invalid_utf16be = True
else:
if 0xDC <= pair[0] <= 0xDF:
self.first_half_surrogate_pair_detected_16be = False
else:
self.invalid_utf16be = True
if not self.first_half_surrogate_pair_detected_16le:
if 0xD8 <= pair[1] <= 0xDB:
self.first_half_surrogate_pair_detected_16le = True
elif 0xDC <= pair[1] <= 0xDF:
self.invalid_utf16le = True
else:
if 0xDC <= pair[1] <= 0xDF:
self.first_half_surrogate_pair_detected_16le = False
else:
self.invalid_utf16le = True
def feed(self, byte_str: Union[bytes, bytearray]) -> ProbingState:
for c in byte_str:
mod4 = self.position % 4
self.quad[mod4] = c
if mod4 == 3:
self.validate_utf32_characters(self.quad)
self.validate_utf16_characters(self.quad[0:2])
self.validate_utf16_characters(self.quad[2:4])
if c == 0:
self.zeros_at_mod[mod4] += 1
else:
self.nonzeros_at_mod[mod4] += 1
self.position += 1
return self.state
@property
def state(self) -> ProbingState:
if self._state in {ProbingState.NOT_ME, ProbingState.FOUND_IT}:
# terminal, decided states
return self._state
if self.get_confidence() > 0.80:
self._state = ProbingState.FOUND_IT
elif self.position > 4 * 1024:
# if we get to 4kb into the file, and we can't conclude it's UTF,
# let's give up
self._state = ProbingState.NOT_ME
return self._state
def get_confidence(self) -> float:
return (
0.85
if (
self.is_likely_utf16le()
or self.is_likely_utf16be()
or self.is_likely_utf32le()
or self.is_likely_utf32be()
)
else 0.00
)
+18 -18
View File
@@ -25,45 +25,46 @@
# 02110-1301 USA # 02110-1301 USA
######################### END LICENSE BLOCK ######################### ######################### END LICENSE BLOCK #########################
from .charsetprober import CharSetProber from typing import Union
from .enums import ProbingState, MachineState
from .codingstatemachine import CodingStateMachine
from .mbcssm import UTF8_SM_MODEL
from .charsetprober import CharSetProber
from .codingstatemachine import CodingStateMachine
from .enums import MachineState, ProbingState
from .mbcssm import UTF8_SM_MODEL
class UTF8Prober(CharSetProber): class UTF8Prober(CharSetProber):
ONE_CHAR_PROB = 0.5 ONE_CHAR_PROB = 0.5
def __init__(self): def __init__(self) -> None:
super(UTF8Prober, self).__init__() super().__init__()
self.coding_sm = CodingStateMachine(UTF8_SM_MODEL) self.coding_sm = CodingStateMachine(UTF8_SM_MODEL)
self._num_mb_chars = None self._num_mb_chars = 0
self.reset() self.reset()
def reset(self): def reset(self) -> None:
super(UTF8Prober, self).reset() super().reset()
self.coding_sm.reset() self.coding_sm.reset()
self._num_mb_chars = 0 self._num_mb_chars = 0
@property @property
def charset_name(self): def charset_name(self) -> str:
return "utf-8" return "utf-8"
@property @property
def language(self): def language(self) -> str:
return "" return ""
def feed(self, byte_str): def feed(self, byte_str: Union[bytes, bytearray]) -> ProbingState:
for c in byte_str: for c in byte_str:
coding_state = self.coding_sm.next_state(c) coding_state = self.coding_sm.next_state(c)
if coding_state == MachineState.ERROR: if coding_state == MachineState.ERROR:
self._state = ProbingState.NOT_ME self._state = ProbingState.NOT_ME
break break
elif coding_state == MachineState.ITS_ME: if coding_state == MachineState.ITS_ME:
self._state = ProbingState.FOUND_IT self._state = ProbingState.FOUND_IT
break break
elif coding_state == MachineState.START: if coding_state == MachineState.START:
if self.coding_sm.get_current_charlen() >= 2: if self.coding_sm.get_current_charlen() >= 2:
self._num_mb_chars += 1 self._num_mb_chars += 1
@@ -73,10 +74,9 @@ class UTF8Prober(CharSetProber):
return self.state return self.state
def get_confidence(self): def get_confidence(self) -> float:
unlike = 0.99 unlike = 0.99
if self._num_mb_chars < 6: if self._num_mb_chars < 6:
unlike *= self.ONE_CHAR_PROB ** self._num_mb_chars unlike *= self.ONE_CHAR_PROB**self._num_mb_chars
return 1.0 - unlike return 1.0 - unlike
else: return unlike
return unlike
+3 -3
View File
@@ -1,9 +1,9 @@
""" """
This module exists only to simplify retrieving the version number of chardet This module exists only to simplify retrieving the version number of chardet
from within setup.py and from chardet subpackages. from within setuptools and from chardet subpackages.
:author: Dan Blanchard (dan.blanchard@gmail.com) :author: Dan Blanchard (dan.blanchard@gmail.com)
""" """
__version__ = "4.0.0" __version__ = "5.2.0"
VERSION = __version__.split('.') VERSION = __version__.split(".")
@@ -1,19 +0,0 @@
charset_normalizer/__init__.py,sha256=UzI3xC8PhmcLRMzSgPb6minTmRq0kWznnCBJ8ZCc2XI,1577
charset_normalizer/api.py,sha256=WOlWjy6wT8SeMYFpaGbXZFN1TMXa-s8vZYfkL4G29iQ,21097
charset_normalizer/cd.py,sha256=mZuiTSKq4XpweSDD2H4T4R3Axtaa-QS0tpEWdpMuAzQ,12554
charset_normalizer/constant.py,sha256=PmCeoKXqq3ZbCtCUpKHwwFBIv9DXMT_an1yd24q28mA,19101
charset_normalizer/legacy.py,sha256=T-QuVMsMeDiQEk8WSszMrzVJg_14AMeSkmHdRYhdl1k,2071
charset_normalizer/md.py,sha256=gEWM354DqBsiSoNkKzFrIW4KRFQjQLbqYnbHAdBwj74,18682
charset_normalizer/models.py,sha256=mC11wo84l00u2o03TRNX7M5ItBAbPUKKXgJSFxA35GY,11492
charset_normalizer/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
charset_normalizer/utils.py,sha256=HdwmBy9vRqcRVGHKZqYWtpiS5wA35cLjnlVQCm0Bq9s,11578
charset_normalizer/version.py,sha256=LbH8odlzMnwR4xZF9wCsnGXQA19axDO7HZ-J9hegIX0,79
charset_normalizer/assets/__init__.py,sha256=wpRfujN7GJuEE5wHHo3wEDVoJ5ovzRIxsImyimCBfGU,20069
charset_normalizer/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
charset_normalizer/cli/normalizer.py,sha256=2F-xURZJzo063Ye-2RLJ2wcmURpbKeAzKwpiws65dAs,9744
charset_normalizer-3.2.0.dist-info/LICENSE,sha256=6zGgxaT7Cbik4yBV0lweX5w1iidS_vPNcgIT0cz-4kE,1070
charset_normalizer-3.2.0.dist-info/METADATA,sha256=K2QHhX9fQ7jFxO7y4IQk7TqYZSH7iTyxgTJQxA65EH0,31284
charset_normalizer-3.2.0.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
charset_normalizer-3.2.0.dist-info/entry_points.txt,sha256=uYo8aIGLWv8YgWfSna5HnfY_En4pkF1w4bgawNAXzP0,76
charset_normalizer-3.2.0.dist-info/top_level.txt,sha256=7ASyzePr8_xuZWJsnqJjIBtyV8vhEo0wBCv1MPRRi3Q,19
charset_normalizer-3.2.0.dist-info/RECORD,,
@@ -1,2 +0,0 @@
[console_scripts]
normalizer = charset_normalizer.cli.normalizer:cli_detect
@@ -1,6 +1,6 @@
Metadata-Version: 2.1 Metadata-Version: 2.1
Name: charset-normalizer Name: charset-normalizer
Version: 3.2.0 Version: 3.3.2
Summary: The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet. Summary: The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet.
Home-page: https://github.com/Ousret/charset_normalizer Home-page: https://github.com/Ousret/charset_normalizer
Author: Ahmed TAHRI Author: Ahmed TAHRI
@@ -39,12 +39,27 @@ Provides-Extra: unicode_backport
<img src="https://img.shields.io/pypi/pyversions/charset_normalizer.svg?orange=blue" /> <img src="https://img.shields.io/pypi/pyversions/charset_normalizer.svg?orange=blue" />
</a> </a>
<a href="https://pepy.tech/project/charset-normalizer/"> <a href="https://pepy.tech/project/charset-normalizer/">
<img alt="Download Count Total" src="https://pepy.tech/badge/charset-normalizer/month" /> <img alt="Download Count Total" src="https://static.pepy.tech/badge/charset-normalizer/month" />
</a> </a>
<a href="https://bestpractices.coreinfrastructure.org/projects/7297"> <a href="https://bestpractices.coreinfrastructure.org/projects/7297">
<img src="https://bestpractices.coreinfrastructure.org/projects/7297/badge"> <img src="https://bestpractices.coreinfrastructure.org/projects/7297/badge">
</a> </a>
</p> </p>
<p align="center">
<sup><i>Featured Packages</i></sup><br>
<a href="https://github.com/jawah/niquests">
<img alt="Static Badge" src="https://img.shields.io/badge/Niquests-HTTP_1.1%2C%202%2C_and_3_Client-cyan">
</a>
<a href="https://github.com/jawah/wassima">
<img alt="Static Badge" src="https://img.shields.io/badge/Wassima-Certifi_Killer-cyan">
</a>
</p>
<p align="center">
<sup><i>In other language (unofficial port - by the community)</i></sup><br>
<a href="https://github.com/nickspring/charset-normalizer-rs">
<img alt="Static Badge" src="https://img.shields.io/badge/Rust-red">
</a>
</p>
> A library that helps you read text from an unknown charset encoding.<br /> Motivated by `chardet`, > A library that helps you read text from an unknown charset encoding.<br /> Motivated by `chardet`,
> I'm trying to resolve the issue by taking a new approach. > I'm trying to resolve the issue by taking a new approach.
@@ -56,21 +71,22 @@ Provides-Extra: unicode_backport
This project offers you an alternative to **Universal Charset Encoding Detector**, also known as **Chardet**. This project offers you an alternative to **Universal Charset Encoding Detector**, also known as **Chardet**.
| Feature | [Chardet](https://github.com/chardet/chardet) | Charset Normalizer | [cChardet](https://github.com/PyYoshi/cChardet) | | Feature | [Chardet](https://github.com/chardet/chardet) | Charset Normalizer | [cChardet](https://github.com/PyYoshi/cChardet) |
|--------------------------------------------------|:---------------------------------------------:|:------------------------------------------------------------------------------------------------------:|:-----------------------------------------------:| |--------------------------------------------------|:---------------------------------------------:|:--------------------------------------------------------------------------------------------------:|:-----------------------------------------------:|
| `Fast` | ❌<br> | ✅<br> | ✅ <br> | | `Fast` | | ✅ | |
| `Universal**` | ❌ | | ❌ | | `Universal**` | ❌ | | ❌ |
| `Reliable` **without** distinguishable standards | ❌ | | ✅ | | `Reliable` **without** distinguishable standards | ❌ | | ✅ |
| `Reliable` **with** distinguishable standards | ✅ | | ✅ | | `Reliable` **with** distinguishable standards | ✅ | | ✅ |
| `License` | LGPL-2.1<br>_restrictive_ | MIT | MPL-1.1<br>_restrictive_ | | `License` | LGPL-2.1<br>_restrictive_ | MIT | MPL-1.1<br>_restrictive_ |
| `Native Python` | ✅ | | ❌ | | `Native Python` | ✅ | | ❌ |
| `Detect spoken language` | ❌ | | N/A | | `Detect spoken language` | ❌ | | N/A |
| `UnicodeDecodeError Safety` | ❌ | | ❌ | | `UnicodeDecodeError Safety` | ❌ | | ❌ |
| `Whl Size` | 193.6 kB | 40 kB | ~200 kB | | `Whl Size (min)` | 193.6 kB | 42 kB | ~200 kB |
| `Supported Encoding` | 33 | 🎉 [90](https://charset-normalizer.readthedocs.io/en/latest/user/support.html#supported-encodings) | 40 | | `Supported Encoding` | 33 | 🎉 [99](https://charset-normalizer.readthedocs.io/en/latest/user/support.html#supported-encodings) | 40 |
<p align="center"> <p align="center">
<img src="https://i.imgflip.com/373iay.gif" alt="Reading Normalized Text" width="226"/><img src="https://media.tenor.com/images/c0180f70732a18b4965448d33adba3d0/tenor.gif" alt="Cat Reading Text" width="200"/> <img src="https://i.imgflip.com/373iay.gif" alt="Reading Normalized Text" width="226"/><img src="https://media.tenor.com/images/c0180f70732a18b4965448d33adba3d0/tenor.gif" alt="Cat Reading Text" width="200"/>
</p>
*\*\* : They are clearly using specific code for a specific encoding even if covering most of used one*<br> *\*\* : They are clearly using specific code for a specific encoding even if covering most of used one*<br>
Did you got there because of the logs? See [https://charset-normalizer.readthedocs.io/en/latest/user/miscellaneous.html](https://charset-normalizer.readthedocs.io/en/latest/user/miscellaneous.html) Did you got there because of the logs? See [https://charset-normalizer.readthedocs.io/en/latest/user/miscellaneous.html](https://charset-normalizer.readthedocs.io/en/latest/user/miscellaneous.html)
@@ -145,6 +161,12 @@ optional arguments:
normalizer ./data/sample.1.fr.srt normalizer ./data/sample.1.fr.srt
``` ```
or
```bash
python -m charset_normalizer ./data/sample.1.fr.srt
```
🎉 Since version 1.4.0 the CLI produce easily usable stdout result in JSON format. 🎉 Since version 1.4.0 the CLI produce easily usable stdout result in JSON format.
```json ```json
@@ -241,6 +263,7 @@ that intel is worth something here. So I use those records against decoded text
- Python >=2.7,<3.5: Unsupported - Python >=2.7,<3.5: Unsupported
- Python 3.5: charset-normalizer < 2.1 - Python 3.5: charset-normalizer < 2.1
- Python 3.6: charset-normalizer < 3.1 - Python 3.6: charset-normalizer < 3.1
- Python 3.7: charset-normalizer < 4.0
Upgrade your Python interpreter as soon as possible. Upgrade your Python interpreter as soon as possible.
@@ -270,6 +293,38 @@ tools.
All notable changes to charset-normalizer will be documented in this file. This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). All notable changes to charset-normalizer will be documented in this file. This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
## [3.3.2](https://github.com/Ousret/charset_normalizer/compare/3.3.1...3.3.2) (2023-10-31)
### Fixed
- Unintentional memory usage regression when using large payload that match several encoding (#376)
- Regression on some detection case showcased in the documentation (#371)
### Added
- Noise (md) probe that identify malformed arabic representation due to the presence of letters in isolated form (credit to my wife)
## [3.3.1](https://github.com/Ousret/charset_normalizer/compare/3.3.0...3.3.1) (2023-10-22)
### Changed
- Optional mypyc compilation upgraded to version 1.6.1 for Python >= 3.8
- Improved the general detection reliability based on reports from the community
## [3.3.0](https://github.com/Ousret/charset_normalizer/compare/3.2.0...3.3.0) (2023-09-30)
### Added
- Allow to execute the CLI (e.g. normalizer) through `python -m charset_normalizer.cli` or `python -m charset_normalizer`
- Support for 9 forgotten encoding that are supported by Python but unlisted in `encoding.aliases` as they have no alias (#323)
### Removed
- (internal) Redundant utils.is_ascii function and unused function is_private_use_only
- (internal) charset_normalizer.assets is moved inside charset_normalizer.constant
### Changed
- (internal) Unicode code blocks in constants are updated using the latest v15.0.0 definition to improve detection
- Optional mypyc compilation upgraded to version 1.5.1 for Python >= 3.8
### Fixed
- Unable to properly sort CharsetMatch when both chaos/noise and coherence were close due to an unreachable condition in \_\_lt\_\_ (#350)
## [3.2.0](https://github.com/Ousret/charset_normalizer/compare/3.1.0...3.2.0) (2023-06-07) ## [3.2.0](https://github.com/Ousret/charset_normalizer/compare/3.1.0...3.2.0) (2023-06-07)
### Changed ### Changed
@@ -0,0 +1,19 @@
charset_normalizer/__init__.py,sha256=UzI3xC8PhmcLRMzSgPb6minTmRq0kWznnCBJ8ZCc2XI,1577
charset_normalizer/__main__.py,sha256=JxY8bleaENOFlLRb9HfoeZCzAMnn2A1oGR5Xm2eyqg0,73
charset_normalizer/api.py,sha256=WOlWjy6wT8SeMYFpaGbXZFN1TMXa-s8vZYfkL4G29iQ,21097
charset_normalizer/cd.py,sha256=xwZliZcTQFA3jU0c00PRiu9MNxXTFxQkFLWmMW24ZzI,12560
charset_normalizer/constant.py,sha256=p0IsOVcEbPWYPOdWhnhRbjK1YVBy6fs05C5vKC-zoxU,40481
charset_normalizer/legacy.py,sha256=T-QuVMsMeDiQEk8WSszMrzVJg_14AMeSkmHdRYhdl1k,2071
charset_normalizer/md.py,sha256=NkSuVLK13_a8c7BxZ4cGIQ5vOtGIWOdh22WZEvjp-7U,19624
charset_normalizer/models.py,sha256=I5i0s4aKCCgLPY2tUY3pwkgFA-BUbbNxQ7hVkVTt62s,11624
charset_normalizer/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
charset_normalizer/utils.py,sha256=teiosMqzKjXyAHXnGdjSBOgnBZwx-SkBbCLrx0UXy8M,11894
charset_normalizer/version.py,sha256=iHKUfHD3kDRSyrh_BN2ojh43TA5-UZQjvbVIEFfpHDs,79
charset_normalizer/cli/__init__.py,sha256=D5ERp8P62llm2FuoMzydZ7d9rs8cvvLXqE-1_6oViPc,100
charset_normalizer/cli/__main__.py,sha256=2F-xURZJzo063Ye-2RLJ2wcmURpbKeAzKwpiws65dAs,9744
charset_normalizer-3.3.2.dist-info/LICENSE,sha256=6zGgxaT7Cbik4yBV0lweX5w1iidS_vPNcgIT0cz-4kE,1070
charset_normalizer-3.3.2.dist-info/METADATA,sha256=cfLhl5A6SI-F0oclm8w8ux9wshL1nipdeCdVnYb4AaA,33550
charset_normalizer-3.3.2.dist-info/WHEEL,sha256=Xo9-1PvkuimrydujYJAjF7pCkriuXBpUPEjma1nZyJ0,92
charset_normalizer-3.3.2.dist-info/entry_points.txt,sha256=ADSTKrkXZ3hhdOVFi6DcUEHQRS0xfxDIE_pEz4wLIXA,65
charset_normalizer-3.3.2.dist-info/top_level.txt,sha256=7ASyzePr8_xuZWJsnqJjIBtyV8vhEo0wBCv1MPRRi3Q,19
charset_normalizer-3.3.2.dist-info/RECORD,,
@@ -1,5 +1,5 @@
Wheel-Version: 1.0 Wheel-Version: 1.0
Generator: bdist_wheel (0.36.2) Generator: bdist_wheel (0.41.3)
Root-Is-Purelib: true Root-Is-Purelib: true
Tag: py3-none-any Tag: py3-none-any
@@ -0,0 +1,2 @@
[console_scripts]
normalizer = charset_normalizer.cli:cli_detect
@@ -0,0 +1,4 @@
from .cli import cli_detect
if __name__ == "__main__":
cli_detect()
File diff suppressed because it is too large Load Diff
@@ -4,8 +4,13 @@ from collections import Counter
from functools import lru_cache from functools import lru_cache
from typing import Counter as TypeCounter, Dict, List, Optional, Tuple from typing import Counter as TypeCounter, Dict, List, Optional, Tuple
from .assets import FREQUENCIES from .constant import (
from .constant import KO_NAMES, LANGUAGE_SUPPORTED_COUNT, TOO_SMALL_SEQUENCE, ZH_NAMES FREQUENCIES,
KO_NAMES,
LANGUAGE_SUPPORTED_COUNT,
TOO_SMALL_SEQUENCE,
ZH_NAMES,
)
from .md import is_suspiciously_successive_range from .md import is_suspiciously_successive_range
from .models import CoherenceMatches from .models import CoherenceMatches
from .utils import ( from .utils import (
@@ -0,0 +1,6 @@
from .__main__ import cli_detect, query_yes_no
__all__ = (
"cli_detect",
"query_yes_no",
)
File diff suppressed because it is too large Load Diff
@@ -9,7 +9,8 @@ from .constant import (
) )
from .utils import ( from .utils import (
is_accentuated, is_accentuated,
is_ascii, is_arabic,
is_arabic_isolated_form,
is_case_variable, is_case_variable,
is_cjk, is_cjk,
is_emoticon, is_emoticon,
@@ -128,8 +129,9 @@ class TooManyAccentuatedPlugin(MessDetectorPlugin):
@property @property
def ratio(self) -> float: def ratio(self) -> float:
if self._character_count == 0 or self._character_count < 8: if self._character_count < 8:
return 0.0 return 0.0
ratio_of_accentuation: float = self._accentuated_count / self._character_count ratio_of_accentuation: float = self._accentuated_count / self._character_count
return ratio_of_accentuation if ratio_of_accentuation >= 0.35 else 0.0 return ratio_of_accentuation if ratio_of_accentuation >= 0.35 else 0.0
@@ -234,16 +236,13 @@ class SuspiciousRange(MessDetectorPlugin):
@property @property
def ratio(self) -> float: def ratio(self) -> float:
if self._character_count == 0: if self._character_count <= 24:
return 0.0 return 0.0
ratio_of_suspicious_range_usage: float = ( ratio_of_suspicious_range_usage: float = (
self._suspicious_successive_range_count * 2 self._suspicious_successive_range_count * 2
) / self._character_count ) / self._character_count
if ratio_of_suspicious_range_usage < 0.1:
return 0.0
return ratio_of_suspicious_range_usage return ratio_of_suspicious_range_usage
@@ -296,7 +295,11 @@ class SuperWeirdWordPlugin(MessDetectorPlugin):
self._is_current_word_bad = True self._is_current_word_bad = True
# Word/Buffer ending with an upper case accentuated letter are so rare, # Word/Buffer ending with an upper case accentuated letter are so rare,
# that we will consider them all as suspicious. Same weight as foreign_long suspicious. # that we will consider them all as suspicious. Same weight as foreign_long suspicious.
if is_accentuated(self._buffer[-1]) and self._buffer[-1].isupper(): if (
is_accentuated(self._buffer[-1])
and self._buffer[-1].isupper()
and all(_.isupper() for _ in self._buffer) is False
):
self._foreign_long_count += 1 self._foreign_long_count += 1
self._is_current_word_bad = True self._is_current_word_bad = True
if buffer_length >= 24 and self._foreign_long_watch: if buffer_length >= 24 and self._foreign_long_watch:
@@ -419,7 +422,7 @@ class ArchaicUpperLowerPlugin(MessDetectorPlugin):
return return
if self._current_ascii_only is True and is_ascii(character) is False: if self._current_ascii_only is True and character.isascii() is False:
self._current_ascii_only = False self._current_ascii_only = False
if self._last_alpha_seen is not None: if self._last_alpha_seen is not None:
@@ -455,6 +458,34 @@ class ArchaicUpperLowerPlugin(MessDetectorPlugin):
return self._successive_upper_lower_count_final / self._character_count return self._successive_upper_lower_count_final / self._character_count
class ArabicIsolatedFormPlugin(MessDetectorPlugin):
def __init__(self) -> None:
self._character_count: int = 0
self._isolated_form_count: int = 0
def reset(self) -> None: # pragma: no cover
self._character_count = 0
self._isolated_form_count = 0
def eligible(self, character: str) -> bool:
return is_arabic(character)
def feed(self, character: str) -> None:
self._character_count += 1
if is_arabic_isolated_form(character):
self._isolated_form_count += 1
@property
def ratio(self) -> float:
if self._character_count < 8:
return 0.0
isolated_form_usage: float = self._isolated_form_count / self._character_count
return isolated_form_usage
@lru_cache(maxsize=1024) @lru_cache(maxsize=1024)
def is_suspiciously_successive_range( def is_suspiciously_successive_range(
unicode_range_a: Optional[str], unicode_range_b: Optional[str] unicode_range_a: Optional[str], unicode_range_b: Optional[str]
@@ -522,6 +553,8 @@ def is_suspiciously_successive_range(
return False return False
if "Forms" in unicode_range_a or "Forms" in unicode_range_b: if "Forms" in unicode_range_a or "Forms" in unicode_range_b:
return False return False
if unicode_range_a == "Basic Latin" or unicode_range_b == "Basic Latin":
return False
return True return True
@@ -54,16 +54,19 @@ class CharsetMatch:
# Below 1% difference --> Use Coherence # Below 1% difference --> Use Coherence
if chaos_difference < 0.01 and coherence_difference > 0.02: if chaos_difference < 0.01 and coherence_difference > 0.02:
# When having a tough decision, use the result that decoded as many multi-byte as possible.
if chaos_difference == 0.0 and self.coherence == other.coherence:
return self.multi_byte_usage > other.multi_byte_usage
return self.coherence > other.coherence return self.coherence > other.coherence
elif chaos_difference < 0.01 and coherence_difference <= 0.02:
# When having a difficult decision, use the result that decoded as many multi-byte as possible.
# preserve RAM usage!
if len(self._payload) >= TOO_BIG_SEQUENCE:
return self.chaos < other.chaos
return self.multi_byte_usage > other.multi_byte_usage
return self.chaos < other.chaos return self.chaos < other.chaos
@property @property
def multi_byte_usage(self) -> float: def multi_byte_usage(self) -> float:
return 1.0 - len(str(self)) / len(self.raw) return 1.0 - (len(str(self)) / len(self.raw))
def __str__(self) -> str: def __str__(self) -> str:
# Lazy Str Loading # Lazy Str Loading
@@ -32,6 +32,8 @@ def is_accentuated(character: str) -> bool:
or "WITH DIAERESIS" in description or "WITH DIAERESIS" in description
or "WITH CIRCUMFLEX" in description or "WITH CIRCUMFLEX" in description
or "WITH TILDE" in description or "WITH TILDE" in description
or "WITH MACRON" in description
or "WITH RING ABOVE" in description
) )
@@ -69,15 +71,6 @@ def is_latin(character: str) -> bool:
return "LATIN" in description return "LATIN" in description
@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
def is_ascii(character: str) -> bool:
try:
character.encode("ascii")
except UnicodeEncodeError:
return False
return True
@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION) @lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
def is_punctuation(character: str) -> bool: def is_punctuation(character: str) -> bool:
character_category: str = unicodedata.category(character) character_category: str = unicodedata.category(character)
@@ -105,7 +98,7 @@ def is_symbol(character: str) -> bool:
if character_range is None: if character_range is None:
return False return False
return "Forms" in character_range return "Forms" in character_range and character_category != "Lo"
@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION) @lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
@@ -115,7 +108,7 @@ def is_emoticon(character: str) -> bool:
if character_range is None: if character_range is None:
return False return False
return "Emoticons" in character_range return "Emoticons" in character_range or "Pictographs" in character_range
@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION) @lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
@@ -133,12 +126,6 @@ def is_case_variable(character: str) -> bool:
return character.islower() != character.isupper() return character.islower() != character.isupper()
def is_private_use_only(character: str) -> bool:
character_category: str = unicodedata.category(character)
return character_category == "Co"
@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION) @lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
def is_cjk(character: str) -> bool: def is_cjk(character: str) -> bool:
try: try:
@@ -189,6 +176,26 @@ def is_thai(character: str) -> bool:
return "THAI" in character_name return "THAI" in character_name
@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
def is_arabic(character: str) -> bool:
try:
character_name = unicodedata.name(character)
except ValueError:
return False
return "ARABIC" in character_name
@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
def is_arabic_isolated_form(character: str) -> bool:
try:
character_name = unicodedata.name(character)
except ValueError:
return False
return "ARABIC" in character_name and "ISOLATED FORM" in character_name
@lru_cache(maxsize=len(UNICODE_RANGES_COMBINED)) @lru_cache(maxsize=len(UNICODE_RANGES_COMBINED))
def is_unicode_range_secondary(range_name: str) -> bool: def is_unicode_range_secondary(range_name: str) -> bool:
return any(keyword in range_name for keyword in UNICODE_SECONDARY_RANGE_KEYWORD) return any(keyword in range_name for keyword in UNICODE_SECONDARY_RANGE_KEYWORD)
@@ -205,7 +212,7 @@ def is_unprintable(character: str) -> bool:
) )
def any_specified_encoding(sequence: bytes, search_zone: int = 4096) -> Optional[str]: def any_specified_encoding(sequence: bytes, search_zone: int = 8192) -> Optional[str]:
""" """
Extract using ASCII-only decoder any specified encoding in the first n-bytes. Extract using ASCII-only decoder any specified encoding in the first n-bytes.
""" """
@@ -2,5 +2,5 @@
Expose version Expose version
""" """
__version__ = "3.2.0" __version__ = "3.3.2"
VERSION = __version__.split(".") VERSION = __version__.split(".")

Some files were not shown because too many files have changed in this diff Show More