summaryrefslogtreecommitdiff
path: root/extra
diff options
context:
space:
mode:
authorManuel Novoa III <mjn3@codepoet.org>2002-05-06 07:37:32 +0000
committerManuel Novoa III <mjn3@codepoet.org>2002-05-06 07:37:32 +0000
commitd07fdf8b9ece2c4339b325921add50792077bf97 (patch)
treeb0886656bdd854728f2d1c05597368c4739ecc1b /extra
parent7f09a14cabbec158d683542e53f53ccfe75031fa (diff)
New locale support (in development). Supports LC_CTYPE, LC_NUMERIC,
LC_TIME, LC_MONETARY, and LC_MESSAGES for the SUSv3 items. Also, nl_langinfo() when real locale support is enabled. New implementation of ctype.h. New implementation of wctype.h. New implementation of most of the string functions (smaller). New implementation of the wcs/wmem functions. These are untested, but they're also just preprocessor-modified versions ot the corresponding str/mem functions. Tweaked qsort and new bsearch. Stuff still pending: stdlib.h and wchar.h mb<->wc functions. I actually have working versions of the stdlib ones, but the reentrant versions from wchar.h require some reworking. Basic replacement and translit support for wc->mb conversions. (groundwork laid). Simple-minded collate support such as was provided by the previous locale implementation. (mostly done -- 8-bit codesets only) Shared mmaping of the locale data and strerror message text.
Diffstat (limited to 'extra')
-rw-r--r--extra/locale/.cvsignore14
-rw-r--r--extra/locale/LOCALES288
-rw-r--r--extra/locale/README86
-rw-r--r--extra/locale/charmaps/ASCII.pairs128
-rw-r--r--extra/locale/charmaps/CP1251.pairs255
-rw-r--r--extra/locale/charmaps/ISO-8859-1.pairs256
-rw-r--r--extra/locale/charmaps/ISO-8859-10.pairs256
-rw-r--r--extra/locale/charmaps/ISO-8859-13.pairs256
-rw-r--r--extra/locale/charmaps/ISO-8859-14.pairs256
-rw-r--r--extra/locale/charmaps/ISO-8859-15.pairs256
-rw-r--r--extra/locale/charmaps/ISO-8859-16.pairs256
-rw-r--r--extra/locale/charmaps/ISO-8859-2.pairs256
-rw-r--r--extra/locale/charmaps/ISO-8859-3.pairs249
-rw-r--r--extra/locale/charmaps/ISO-8859-4.pairs256
-rw-r--r--extra/locale/charmaps/ISO-8859-5.pairs256
-rw-r--r--extra/locale/charmaps/ISO-8859-6.pairs211
-rw-r--r--extra/locale/charmaps/ISO-8859-7.pairs250
-rw-r--r--extra/locale/charmaps/ISO-8859-8.pairs220
-rw-r--r--extra/locale/charmaps/ISO-8859-9.pairs256
-rw-r--r--extra/locale/charmaps/KOI8-R.pairs256
-rw-r--r--extra/locale/charmaps/KOI8-U.pairs256
-rw-r--r--extra/locale/charmaps/TIS-620.pairs215
-rw-r--r--extra/locale/gen_collate_from_glibc.c207
-rw-r--r--extra/locale/gen_ctype_from_glibc.c277
-rw-r--r--extra/locale/gen_locale.c1254
-rw-r--r--extra/locale/gen_mmap.c237
-rw-r--r--extra/locale/gen_wc8bit.c729
-rw-r--r--extra/locale/gen_wctype.c833
-rw-r--r--extra/locale/lmmtolso.c72
-rw-r--r--extra/locale/locale_mmap.h89
30 files changed, 8180 insertions, 506 deletions
diff --git a/extra/locale/.cvsignore b/extra/locale/.cvsignore
new file mode 100644
index 000000000..e020fa9e3
--- /dev/null
+++ b/extra/locale/.cvsignore
@@ -0,0 +1,14 @@
+lmmtolso
+gen_wctype
+wctables.h
+gen_wc8bit
+c8tables.h
+gen_locale
+locale_tables.h
+locale.mmap
+gen_mmap
+lt_defines.h
+locale_data.c
+uClibc_locale_data.h
+codesets.txt
+locales.txt
diff --git a/extra/locale/LOCALES b/extra/locale/LOCALES
new file mode 100644
index 000000000..739ecb95a
--- /dev/null
+++ b/extra/locale/LOCALES
@@ -0,0 +1,288 @@
+# First some @modifier mappings. Internally, the modifier is signaled by
+# replacing '_' in the locale name with a unique identifying character.
+# For example, internally we map "ca_ES@euro" to "caeES". This allows for
+# smaller code and easier processing of locale names.
+
+@euro e
+@cyrillic c
+
+#---------------------------------------------------------------------------
+# Next, set to {y}es to enable and {n}o to disable the UTF-8 and the 8-bit
+# codeset locales. Of course, you must have built the c8tables.h and
+# the wctables.h files appropriately.
+
+
+UTF-8 yes
+8-BIT yes
+
+#---------------------------------------------------------------------------
+# Now the locales
+
+af_ZA ISO-8859-1
+ar_AE ISO-8859-6
+ar_BH ISO-8859-6
+ar_DZ ISO-8859-6
+ar_EG ISO-8859-6
+ar_IN UTF-8
+ar_IQ ISO-8859-6
+ar_JO ISO-8859-6
+ar_KW ISO-8859-6
+ar_LB ISO-8859-6
+ar_LY ISO-8859-6
+ar_MA ISO-8859-6
+ar_OM ISO-8859-6
+ar_QA ISO-8859-6
+ar_SA ISO-8859-6
+ar_SD ISO-8859-6
+ar_SY ISO-8859-6
+ar_TN ISO-8859-6
+ar_YE ISO-8859-6
+be_BY CP1251
+bg_BG CP1251
+ca_ES ISO-8859-1
+ca_ES@euro ISO-8859-15
+cs_CZ ISO-8859-2
+da_DK ISO-8859-1
+de_AT ISO-8859-1
+de_AT@euro ISO-8859-15
+de_BE ISO-8859-1
+de_BE@euro ISO-8859-15
+de_CH ISO-8859-1
+de_DE ISO-8859-1
+de_DE@euro ISO-8859-15
+de_LU ISO-8859-1
+de_LU@euro ISO-8859-15
+el_GR ISO-8859-7
+en_AU ISO-8859-1
+en_BW ISO-8859-1
+en_CA ISO-8859-1
+en_DK ISO-8859-1
+en_GB ISO-8859-1
+en_HK ISO-8859-1
+en_IE ISO-8859-1
+en_IE@euro ISO-8859-15
+en_IN UTF-8
+en_NZ ISO-8859-1
+en_PH ISO-8859-1
+en_SG ISO-8859-1
+en_US ISO-8859-1
+en_ZA ISO-8859-1
+en_ZW ISO-8859-1
+es_AR ISO-8859-1
+es_BO ISO-8859-1
+es_CL ISO-8859-1
+es_CO ISO-8859-1
+es_CR ISO-8859-1
+es_DO ISO-8859-1
+es_EC ISO-8859-1
+es_ES ISO-8859-1
+es_ES@euro ISO-8859-15
+es_GT ISO-8859-1
+es_HN ISO-8859-1
+es_MX ISO-8859-1
+es_NI ISO-8859-1
+es_PA ISO-8859-1
+es_PE ISO-8859-1
+es_PR ISO-8859-1
+es_PY ISO-8859-1
+es_SV ISO-8859-1
+es_US ISO-8859-1
+es_UY ISO-8859-1
+es_VE ISO-8859-1
+et_EE ISO-8859-1
+eu_ES ISO-8859-1
+eu_ES@euro ISO-8859-15
+fa_IR UTF-8
+fi_FI ISO-8859-1
+fi_FI@euro ISO-8859-15
+fo_FO ISO-8859-1
+fr_BE ISO-8859-1
+fr_BE@euro ISO-8859-15
+fr_CA ISO-8859-1
+fr_CH ISO-8859-1
+fr_FR ISO-8859-1
+fr_FR@euro ISO-8859-15
+fr_LU ISO-8859-1
+fr_LU@euro ISO-8859-15
+ga_IE ISO-8859-1
+ga_IE@euro ISO-8859-15
+gl_ES ISO-8859-1
+gl_ES@euro ISO-8859-15
+gv_GB ISO-8859-1
+he_IL ISO-8859-8
+hi_IN UTF-8
+hr_HR ISO-8859-2
+hu_HU ISO-8859-2
+id_ID ISO-8859-1
+is_IS ISO-8859-1
+it_CH ISO-8859-1
+it_IT ISO-8859-1
+it_IT@euro ISO-8859-15
+iw_IL ISO-8859-8
+kl_GL ISO-8859-1
+ko_KR.UTF-8 UTF-8
+kw_GB ISO-8859-1
+lt_LT ISO-8859-13
+lv_LV ISO-8859-13
+mk_MK ISO-8859-5
+mr_IN UTF-8
+ms_MY ISO-8859-1
+mt_MT ISO-8859-3
+nl_BE ISO-8859-1
+nl_BE@euro ISO-8859-15
+nl_NL ISO-8859-1
+nl_NL@euro ISO-8859-15
+nn_NO ISO-8859-1
+no_NO ISO-8859-1
+pl_PL ISO-8859-2
+pt_BR ISO-8859-1
+pt_PT ISO-8859-1
+pt_PT@euro ISO-8859-15
+ro_RO ISO-8859-2
+ru_RU ISO-8859-5
+ru_RU.KOI8-R KOI8-R
+ru_UA KOI8-U
+sk_SK ISO-8859-2
+sl_SI ISO-8859-2
+sq_AL ISO-8859-1
+sr_YU ISO-8859-2
+sr_YU@cyrillic ISO-8859-5
+sv_FI ISO-8859-1
+sv_FI@euro ISO-8859-15
+sv_SE ISO-8859-1
+ta_IN UTF-8
+te_IN UTF-8
+th_TH TIS-620
+tr_TR ISO-8859-9
+uk_UA KOI8-U
+vi_VN UTF-8
+ar_AE.UTF-8 UTF-8
+ar_BH.UTF-8 UTF-8
+ar_DZ.UTF-8 UTF-8
+ar_EG.UTF-8 UTF-8
+ar_IQ.UTF-8 UTF-8
+ar_JO.UTF-8 UTF-8
+ar_KW.UTF-8 UTF-8
+ar_LB.UTF-8 UTF-8
+ar_LY.UTF-8 UTF-8
+ar_MA.UTF-8 UTF-8
+ar_OM.UTF-8 UTF-8
+ar_QA.UTF-8 UTF-8
+ar_SA.UTF-8 UTF-8
+ar_SD.UTF-8 UTF-8
+ar_SY.UTF-8 UTF-8
+ar_TN.UTF-8 UTF-8
+ar_YE.UTF-8 UTF-8
+be_BY.UTF-8 UTF-8
+bg_BG.UTF-8 UTF-8
+ca_ES.UTF-8 UTF-8
+ca_ES.UTF-8@euro UTF-8
+cs_CZ.UTF-8 UTF-8
+da_DK.UTF-8 UTF-8
+de_AT.UTF-8 UTF-8
+de_AT.UTF-8@euro UTF-8
+de_BE.UTF-8 UTF-8
+de_BE.UTF-8@euro UTF-8
+de_CH.UTF-8 UTF-8
+de_DE.UTF-8 UTF-8
+de_DE.UTF-8@euro UTF-8
+de_LU.UTF-8 UTF-8
+de_LU.UTF-8@euro UTF-8
+el_GR.UTF-8 UTF-8
+en_AU.UTF-8 UTF-8
+en_BW.UTF-8 UTF-8
+en_CA.UTF-8 UTF-8
+en_DK.UTF-8 UTF-8
+en_GB.UTF-8 UTF-8
+en_HK.UTF-8 UTF-8
+en_IE.UTF-8 UTF-8
+en_IE.UTF-8@euro UTF-8
+en_NZ.UTF-8 UTF-8
+en_PH.UTF-8 UTF-8
+en_SG.UTF-8 UTF-8
+en_US.UTF-8 UTF-8
+en_ZA.UTF-8 UTF-8
+en_ZW.UTF-8 UTF-8
+es_AR.UTF-8 UTF-8
+es_BO.UTF-8 UTF-8
+es_CL.UTF-8 UTF-8
+es_CO.UTF-8 UTF-8
+es_CR.UTF-8 UTF-8
+es_DO.UTF-8 UTF-8
+es_EC.UTF-8 UTF-8
+es_ES.UTF-8 UTF-8
+es_ES.UTF-8@euro UTF-8
+es_GT.UTF-8 UTF-8
+es_HN.UTF-8 UTF-8
+es_MX.UTF-8 UTF-8
+es_NI.UTF-8 UTF-8
+es_PA.UTF-8 UTF-8
+es_PE.UTF-8 UTF-8
+es_PR.UTF-8 UTF-8
+es_PY.UTF-8 UTF-8
+es_SV.UTF-8 UTF-8
+es_US.UTF-8 UTF-8
+es_UY.UTF-8 UTF-8
+es_VE.UTF-8 UTF-8
+et_EE.UTF-8 UTF-8
+eu_ES.UTF-8 UTF-8
+eu_ES.UTF-8@euro UTF-8
+fi_FI.UTF-8 UTF-8
+fi_FI.UTF-8@euro UTF-8
+fo_FO.UTF-8 UTF-8
+fr_BE.UTF-8 UTF-8
+fr_BE.UTF-8@euro UTF-8
+fr_CA.UTF-8 UTF-8
+fr_CH.UTF-8 UTF-8
+fr_FR.UTF-8 UTF-8
+fr_FR.UTF-8@euro UTF-8
+fr_LU.UTF-8 UTF-8
+fr_LU.UTF-8@euro UTF-8
+ga_IE.UTF-8 UTF-8
+ga_IE.UTF-8@euro UTF-8
+gl_ES.UTF-8 UTF-8
+gl_ES.UTF-8@euro UTF-8
+gv_GB.UTF-8 UTF-8
+he_IL.UTF-8 UTF-8
+hr_HR.UTF-8 UTF-8
+hu_HU.UTF-8 UTF-8
+id_ID.UTF-8 UTF-8
+is_IS.UTF-8 UTF-8
+it_CH.UTF-8 UTF-8
+it_IT.UTF-8 UTF-8
+it_IT.UTF-8@euro UTF-8
+iw_IL.UTF-8 UTF-8
+kl_GL.UTF-8 UTF-8
+kw_GB.UTF-8 UTF-8
+lt_LT.UTF-8 UTF-8
+lv_LV.UTF-8 UTF-8
+mk_MK.UTF-8 UTF-8
+ms_MY.UTF-8 UTF-8
+mt_MT.UTF-8 UTF-8
+nl_BE.UTF-8 UTF-8
+nl_BE.UTF-8@euro UTF-8
+nl_NL.UTF-8 UTF-8
+nl_NL.UTF-8@euro UTF-8
+nn_NO.UTF-8 UTF-8
+no_NO.UTF-8 UTF-8
+pl_PL.UTF-8 UTF-8
+pt_BR.UTF-8 UTF-8
+pt_PT.UTF-8 UTF-8
+pt_PT.UTF-8@euro UTF-8
+ro_RO.UTF-8 UTF-8
+ru_RU.UTF-8 UTF-8
+ru_UA.UTF-8 UTF-8
+sk_SK.UTF-8 UTF-8
+sl_SI.UTF-8 UTF-8
+sq_AL.UTF-8 UTF-8
+sr_YU.UTF-8 UTF-8
+sr_YU.UTF-8@cyrillic UTF-8
+sv_FI.UTF-8 UTF-8
+sv_FI.UTF-8@euro UTF-8
+sv_SE.UTF-8 UTF-8
+th_TH.UTF-8 UTF-8
+tr_TR.UTF-8 UTF-8
+uk_UA.UTF-8 UTF-8
+zh_CN.UTF-8 UTF-8
+zh_HK.UTF-8 UTF-8
+zh_TW.UTF-8 UTF-8
diff --git a/extra/locale/README b/extra/locale/README
index 6ac487c9a..63723f97f 100644
--- a/extra/locale/README
+++ b/extra/locale/README
@@ -1,29 +1,71 @@
+Warning!!! Warning!!! Warning!!! Warning!!! Warning!!! Warning!!!
+Warning!!! Warning!!! Warning!!! Warning!!! Warning!!! Warning!!!
-The programs gen_ctype_from_glibc.c and gen_collate_from_glibc.c
-will generate data files which can be
-used by uClibc ctype, strcoll and setlocale functions to support locales.
-From the comments:
+The programs in this directory are strictly cut-and-paste hack jobs to
+extract the data I needed from glibc's locale database. I'm ashamed to
+even let them into the light of day, and I consider them complete garbage.
-/*
- * Generator locale ctype tables
- * You must have already setuped locale for worked libc (libc5 or glibc)
- *
- * This programm scan /usr/share/locale directories and write
- * ./LOCALE/LC_CTYPE files for system with uclibc
- *
- * Written by Vladimir Oleynik <vodz@usa.net> 2001
- */
+However, they are currently necessary to build the data needed for the
+locale support I've implemented, so I'm forced to include them here.
+NOTE: While its possible to use this stuff for native != target arch,
+you'll have to either write a converter to account for endianess and
+struct padding issues, or run the mmap file generator on your target
+arch. But all these programs will be rewritten at some point.
-Sample usage to dump all the data files in a tmp directory:
+All that being said, LC_CTYPE support has been added and supports the
+ctype.h and wctype.h function. Also, LC_TIME, LC_MONETARY, LC_NUMERIC,
+and LC_MESSAGES are supported wrt SUSv3. localeconv() works in both
+real and stub locale modes. nl_langinfo() currently only works with
+real locales enabled. That will be fixed though. wc->mb unsupported
+char replacement and basic translit support is on the way as well.
+Finally, some basic 8-bit codeset LC_COLLATE support should be in place
+in the next week or two (similar to what was in the previous locale
+implementation).
-gcc gen_ctype_from_glibc.c -o gen_ctype_from_glibc
-gcc gen_collate_from_glibc.c -o gen_ctype_from_glibc
+Also, as one can probably guess, I'm working towards having the locale
+data accessed via a shared mmap. That will allow non-mmu platforms
+to use this without the current bloat.
-mkdir tmp
-cd tmp
-../gen_ctype_from_glibc -d /usr/share/locale -c
-../gen_collate_from_glibc
+Currently, the output of size for my locale_data.o file is
+ text data bss dec hex filename
+ 59072 4 0 59076 e6c4 extra/locale/locale_data.o
+which is for the C locale (automatic of course) + all codesets in
+charmaps/ and all 268 locales in LOCALES. I estimate that the
+translit support for those 8-bit codesets will add another 7-10k.
-Then just move the directory or directories you need (not the .c files)
-to the uClibc locale file directory you set in Config.
+One difference of note is that the special case upper/lower mappings
+in the turkish locale are currently not implemented. That will be
+fixed.
+
+Manuel
+
+Warning!!! Warning!!! Warning!!! Warning!!! Warning!!! Warning!!!
+Warning!!! Warning!!! Warning!!! Warning!!! Warning!!! Warning!!!
+
+
+1) In the toplevel dir, 'make headers uClibc_config' .
+
+2) Create a codesets.txt file in this dir listing the codesets you want
+ to support. The easiest way to do this is to edit the output of
+ 'find ./charmaps -name "*.pairs" > codesets.txt'.
+ NOTE: UTF-8 support is always included if you build with wide chars enabled.
+ NOTE: The files in charmaps/ were created from glibc's charmap files
+ with the awk script at the end of this file. You can add others
+ but only single byte codesets are supported.
+
+3) Create a locales.txt file to select the locales you want to support.
+ You can copy and edit the LOCALES file for example. Other locales could
+ added provided you've included the appropriate codesets in step 2.
+ NOTE: You have to have the approprite locales available for glibc!
+
+4) Run make here.
+
+5) Continue building uClibc from the toplevel dir.
+
+
+
+
+Script used to generate the charmaps/*.pairs files:
+
+cat $1 | awk 'BEGIN { i = 0 } ; { if ($1 == "CHARMAP") i=1 ; else if ($1 == "END") i=0 ; else if (i==1) { sub("/","0",$2) ; sub("<U","0x",$1) ; sub(">","",$1) ; print "{", $2, ",", $1, "}," } }'
diff --git a/extra/locale/charmaps/ASCII.pairs b/extra/locale/charmaps/ASCII.pairs
new file mode 100644
index 000000000..038314d1f
--- /dev/null
+++ b/extra/locale/charmaps/ASCII.pairs
@@ -0,0 +1,128 @@
+{ 0x00 , 0x0000 },
+{ 0x01 , 0x0001 },
+{ 0x02 , 0x0002 },
+{ 0x03 , 0x0003 },
+{ 0x04 , 0x0004 },
+{ 0x05 , 0x0005 },
+{ 0x06 , 0x0006 },
+{ 0x07 , 0x0007 },
+{ 0x08 , 0x0008 },
+{ 0x09 , 0x0009 },
+{ 0x0a , 0x000A },
+{ 0x0b , 0x000B },
+{ 0x0c , 0x000C },
+{ 0x0d , 0x000D },
+{ 0x0e , 0x000E },
+{ 0x0f , 0x000F },
+{ 0x10 , 0x0010 },
+{ 0x11 , 0x0011 },
+{ 0x12 , 0x0012 },
+{ 0x13 , 0x0013 },
+{ 0x14 , 0x0014 },
+{ 0x15 , 0x0015 },
+{ 0x16 , 0x0016 },
+{ 0x17 , 0x0017 },
+{ 0x18 , 0x0018 },
+{ 0x19 , 0x0019 },
+{ 0x1a , 0x001A },
+{ 0x1b , 0x001B },
+{ 0x1c , 0x001C },
+{ 0x1d , 0x001D },
+{ 0x1e , 0x001E },
+{ 0x1f , 0x001F },
+{ 0x20 , 0x0020 },
+{ 0x21 , 0x0021 },
+{ 0x22 , 0x0022 },
+{ 0x23 , 0x0023 },
+{ 0x24 , 0x0024 },
+{ 0x25 , 0x0025 },
+{ 0x26 , 0x0026 },
+{ 0x27 , 0x0027 },
+{ 0x28 , 0x0028 },
+{ 0x29 , 0x0029 },
+{ 0x2a , 0x002A },
+{ 0x2b , 0x002B },
+{ 0x2c , 0x002C },
+{ 0x2d , 0x002D },
+{ 0x2e , 0x002E },
+{ 0x2f , 0x002F },
+{ 0x30 , 0x0030 },
+{ 0x31 , 0x0031 },
+{ 0x32 , 0x0032 },
+{ 0x33 , 0x0033 },
+{ 0x34 , 0x0034 },
+{ 0x35 , 0x0035 },
+{ 0x36 , 0x0036 },
+{ 0x37 , 0x0037 },
+{ 0x38 , 0x0038 },
+{ 0x39 , 0x0039 },
+{ 0x3a , 0x003A },
+{ 0x3b , 0x003B },
+{ 0x3c , 0x003C },
+{ 0x3d , 0x003D },
+{ 0x3e , 0x003E },
+{ 0x3f , 0x003F },
+{ 0x40 , 0x0040 },
+{ 0x41 , 0x0041 },
+{ 0x42 , 0x0042 },
+{ 0x43 , 0x0043 },
+{ 0x44 , 0x0044 },
+{ 0x45 , 0x0045 },
+{ 0x46 , 0x0046 },
+{ 0x47 , 0x0047 },
+{ 0x48 , 0x0048 },
+{ 0x49 , 0x0049 },
+{ 0x4a , 0x004A },
+{ 0x4b , 0x004B },
+{ 0x4c , 0x004C },
+{ 0x4d , 0x004D },
+{ 0x4e , 0x004E },
+{ 0x4f , 0x004F },
+{ 0x50 , 0x0050 },
+{ 0x51 , 0x0051 },
+{ 0x52 , 0x0052 },
+{ 0x53 , 0x0053 },
+{ 0x54 , 0x0054 },
+{ 0x55 , 0x0055 },
+{ 0x56 , 0x0056 },
+{ 0x57 , 0x0057 },
+{ 0x58 , 0x0058 },
+{ 0x59 , 0x0059 },
+{ 0x5a , 0x005A },
+{ 0x5b , 0x005B },
+{ 0x5c , 0x005C },
+{ 0x5d , 0x005D },
+{ 0x5e , 0x005E },
+{ 0x5f , 0x005F },
+{ 0x60 , 0x0060 },
+{ 0x61 , 0x0061 },
+{ 0x62 , 0x0062 },
+{ 0x63 , 0x0063 },
+{ 0x64 , 0x0064 },
+{ 0x65 , 0x0065 },
+{ 0x66 , 0x0066 },
+{ 0x67 , 0x0067 },
+{ 0x68 , 0x0068 },
+{ 0x69 , 0x0069 },
+{ 0x6a , 0x006A },
+{ 0x6b , 0x006B },
+{ 0x6c , 0x006C },
+{ 0x6d , 0x006D },
+{ 0x6e , 0x006E },
+{ 0x6f , 0x006F },
+{ 0x70 , 0x0070 },
+{ 0x71 , 0x0071 },
+{ 0x72 , 0x0072 },
+{ 0x73 , 0x0073 },
+{ 0x74 , 0x0074 },
+{ 0x75 , 0x0075 },
+{ 0x76 , 0x0076 },
+{ 0x77 , 0x0077 },
+{ 0x78 , 0x0078 },
+{ 0x79 , 0x0079 },
+{ 0x7a , 0x007A },
+{ 0x7b , 0x007B },
+{ 0x7c , 0x007C },
+{ 0x7d , 0x007D },
+{ 0x7e , 0x007E },
+{ 0x7f , 0x007F },
diff --git a/extra/locale/charmaps/CP1251.pairs b/extra/locale/charmaps/CP1251.pairs
new file mode 100644
index 000000000..a5a80d61d
--- /dev/null
+++ b/extra/locale/charmaps/CP1251.pairs
@@ -0,0 +1,255 @@
+{ 0x00 , 0x0000 },
+{ 0x01 , 0x0001 },
+{ 0x02 , 0x0002 },
+{ 0x03 , 0x0003 },
+{ 0x04 , 0x0004 },
+{ 0x05 , 0x0005 },
+{ 0x06 , 0x0006 },
+{ 0x07 , 0x0007 },
+{ 0x08 , 0x0008 },
+{ 0x09 , 0x0009 },
+{ 0x0a , 0x000A },
+{ 0x0b , 0x000B },
+{ 0x0c , 0x000C },
+{ 0x0d , 0x000D },
+{ 0x0e , 0x000E },
+{ 0x0f , 0x000F },
+{ 0x10 , 0x0010 },
+{ 0x11 , 0x0011 },
+{ 0x12 , 0x0012 },
+{ 0x13 , 0x0013 },
+{ 0x14 , 0x0014 },
+{ 0x15 , 0x0015 },
+{ 0x16 , 0x0016 },
+{ 0x17 , 0x0017 },
+{ 0x18 , 0x0018 },
+{ 0x19 , 0x0019 },
+{ 0x1a , 0x001A },
+{ 0x1b , 0x001B },
+{ 0x1c , 0x001C },
+{ 0x1d , 0x001D },
+{ 0x1e , 0x001E },
+{ 0x1f , 0x001F },
+{ 0x20 , 0x0020 },
+{ 0x21 , 0x0021 },
+{ 0x22 , 0x0022 },
+{ 0x23 , 0x0023 },
+{ 0x24 , 0x0024 },
+{ 0x25 , 0x0025 },
+{ 0x26 , 0x0026 },
+{ 0x27 , 0x0027 },
+{ 0x28 , 0x0028 },
+{ 0x29 , 0x0029 },
+{ 0x2a , 0x002A },
+{ 0x2b , 0x002B },
+{ 0x2c , 0x002C },
+{ 0x2d , 0x002D },
+{ 0x2e , 0x002E },
+{ 0x2f , 0x002F },
+{ 0x30 , 0x0030 },
+{ 0x31 , 0x0031 },
+{ 0x32 , 0x0032 },
+{ 0x33 , 0x0033 },
+{ 0x34 , 0x0034 },
+{ 0x35 , 0x0035 },
+{ 0x36 , 0x0036 },
+{ 0x37 , 0x0037 },
+{ 0x38 , 0x0038 },
+{ 0x39 , 0x0039 },
+{ 0x3a , 0x003A },
+{ 0x3b , 0x003B },
+{ 0x3c , 0x003C },
+{ 0x3d , 0x003D },
+{ 0x3e , 0x003E },
+{ 0x3f , 0x003F },
+{ 0x40 , 0x0040 },
+{ 0x41 , 0x0041 },
+{ 0x42 , 0x0042 },
+{ 0x43 , 0x0043 },
+{ 0x44 , 0x0044 },
+{ 0x45 , 0x0045 },
+{ 0x46 , 0x0046 },
+{ 0x47 , 0x0047 },
+{ 0x48 , 0x0048 },
+{ 0x49 , 0x0049 },
+{ 0x4a , 0x004A },
+{ 0x4b , 0x004B },
+{ 0x4c , 0x004C },
+{ 0x4d , 0x004D },
+{ 0x4e , 0x004E },
+{ 0x4f , 0x004F },
+{ 0x50 , 0x0050 },
+{ 0x51 , 0x0051 },
+{ 0x52 , 0x0052 },
+{ 0x53 , 0x0053 },
+{ 0x54 , 0x0054 },
+{ 0x55 , 0x0055 },
+{ 0x56 , 0x0056 },
+{ 0x57 , 0x0057 },
+{ 0x58 , 0x0058 },
+{ 0x59 , 0x0059 },
+{ 0x5a , 0x005A },
+{ 0x5b , 0x005B },
+{ 0x5c , 0x005C },
+{ 0x5d , 0x005D },
+{ 0x5e , 0x005E },
+{ 0x5f , 0x005F },
+{ 0x60 , 0x0060 },
+{ 0x61 , 0x0061 },
+{ 0x62 , 0x0062 },
+{ 0x63 , 0x0063 },
+{ 0x64 , 0x0064 },
+{ 0x65 , 0x0065 },
+{ 0x66 , 0x0066 },
+{ 0x67 , 0x0067 },
+{ 0x68 , 0x0068 },
+{ 0x69 , 0x0069 },
+{ 0x6a , 0x006A },
+{ 0x6b , 0x006B },
+{ 0x6c , 0x006C },
+{ 0x6d , 0x006D },
+{ 0x6e , 0x006E },
+{ 0x6f , 0x006F },
+{ 0x70 , 0x0070 },
+{ 0x71 , 0x0071 },
+{ 0x72 , 0x0072 },
+{ 0x73 , 0x0073 },
+{ 0x74 , 0x0074 },
+{ 0x75 , 0x0075 },
+{ 0x76 , 0x0076 },
+{ 0x77 , 0x0077 },
+{ 0x78 , 0x0078 },
+{ 0x79 , 0x0079 },
+{ 0x7a , 0x007A },
+{ 0x7b , 0x007B },
+{ 0x7c , 0x007C },
+{ 0x7d , 0x007D },
+{ 0x7e , 0x007E },
+{ 0x7f , 0x007F },
+{ 0x80 , 0x0402 },
+{ 0x81 , 0x0403 },
+{ 0x82 , 0x201A },
+{ 0x83 , 0x0453 },
+{ 0x84 , 0x201E },
+{ 0x85 , 0x2026 },
+{ 0x86 , 0x2020 },
+{ 0x87 , 0x2021 },
+{ 0x88 , 0x20AC },
+{ 0x89 , 0x2030 },
+{ 0x8a , 0x0409 },
+{ 0x8b , 0x2039 },
+{ 0x8c , 0x040A },
+{ 0x8d , 0x040C },
+{ 0x8e , 0x040B },
+{ 0x8f , 0x040F },
+{ 0x90 , 0x0452 },
+{ 0x91 , 0x2018 },
+{ 0x92 , 0x2019 },
+{ 0x93 , 0x201C },
+{ 0x94 , 0x201D },
+{ 0x95 , 0x2022 },
+{ 0x96 , 0x2013 },
+{ 0x97 , 0x2014 },
+{ 0x99 , 0x2122 },
+{ 0x9a , 0x0459 },
+{ 0x9b , 0x203A },
+{ 0x9c , 0x045A },
+{ 0x9d , 0x045C },
+{ 0x9e , 0x045B },
+{ 0x9f , 0x045F },
+{ 0xa0 , 0x00A0 },
+{ 0xa1 , 0x040E },
+{ 0xa2 , 0x045E },
+{ 0xa3 , 0x0408 },
+{ 0xa4 , 0x00A4 },
+{ 0xa5 , 0x0490 },
+{ 0xa6 , 0x00A6 },
+{ 0xa7 , 0x00A7 },
+{ 0xa8 , 0x0401 },
+{ 0xa9 , 0x00A9 },
+{ 0xaa , 0x0404 },
+{ 0xab , 0x00AB },
+{ 0xac , 0x00AC },
+{ 0xad , 0x00AD },
+{ 0xae , 0x00AE },
+{ 0xaf , 0x0407 },
+{ 0xb0 , 0x00B0 },
+{ 0xb1 , 0x00B1 },
+{ 0xb2 , 0x0406 },
+{ 0xb3 , 0x0456 },
+{ 0xb4 , 0x0491 },
+{ 0xb5 , 0x00B5 },
+{ 0xb6 , 0x00B6 },
+{ 0xb7 , 0x00B7 },
+{ 0xb8 , 0x0451 },
+{ 0xb9 , 0x2116 },
+{ 0xba , 0x0454 },
+{ 0xbb , 0x00BB },
+{ 0xbc , 0x0458 },
+{ 0xbd , 0x0405 },
+{ 0xbe , 0x0455 },
+{ 0xbf , 0x0457 },
+{ 0xc0 , 0x0410 },
+{ 0xc1 , 0x0411 },
+{ 0xc2 , 0x0412 },
+{ 0xc3 , 0x0413 },
+{ 0xc4 , 0x0414 },
+{ 0xc5 , 0x0415 },
+{ 0xc6 , 0x0416 },
+{ 0xc7 , 0x0417 },
+{ 0xc8 , 0x0418 },
+{ 0xc9 , 0x0419 },
+{ 0xca , 0x041A },
+{ 0xcb , 0x041B },
+{ 0xcc , 0x041C },
+{ 0xcd , 0x041D },
+{ 0xce , 0x041E },
+{ 0xcf , 0x041F },
+{ 0xd0 , 0x0420 },
+{ 0xd1 , 0x0421 },
+{ 0xd2 , 0x0422 },
+{ 0xd3 , 0x0423 },
+{ 0xd4 , 0x0424 },
+{ 0xd5 , 0x0425 },
+{ 0xd6 , 0x0426 },
+{ 0xd7 , 0x0427 },
+{ 0xd8 , 0x0428 },
+{ 0xd9 , 0x0429 },
+{ 0xda , 0x042A },
+{ 0xdb , 0x042B },
+{ 0xdc , 0x042C },
+{ 0xdd , 0x042D },
+{ 0xde , 0x042E },
+{ 0xdf , 0x042F },
+{ 0xe0 , 0x0430 },
+{ 0xe1 , 0x0431 },
+{ 0xe2 , 0x0432 },
+{ 0xe3 , 0x0433 },
+{ 0xe4 , 0x0434 },
+{ 0xe5 , 0x0435 },
+{ 0xe6 , 0x0436 },
+{ 0xe7 , 0x0437 },
+{ 0xe8 , 0x0438 },
+{ 0xe9 , 0x0439 },
+{ 0xea , 0x043A },
+{ 0xeb , 0x043B },
+{ 0xec , 0x043C },
+{ 0xed , 0x043D },
+{ 0xee , 0x043E },
+{ 0xef , 0x043F },
+{ 0xf0 , 0x0440 },
+{ 0xf1 , 0x0441 },
+{ 0xf2 , 0x0442 },
+{ 0xf3 , 0x0443 },
+{ 0xf4 , 0x0444 },
+{ 0xf5 , 0x0445 },
+{ 0xf6 , 0x0446 },
+{ 0xf7 , 0x0447 },
+{ 0xf8 , 0x0448 },
+{ 0xf9 , 0x0449 },
+{ 0xfa , 0x044A },
+{ 0xfb , 0x044B },
+{ 0xfc , 0x044C },
+{ 0xfd , 0x044D },
+{ 0xfe , 0x044E },
+{ 0xff , 0x044F },
diff --git a/extra/locale/charmaps/ISO-8859-1.pairs b/extra/locale/charmaps/ISO-8859-1.pairs
new file mode 100644
index 000000000..97812a0d5
--- /dev/null
+++ b/extra/locale/charmaps/ISO-8859-1.pairs
@@ -0,0 +1,256 @@
+{ 0x00 , 0x0000 },
+{ 0x01 , 0x0001 },
+{ 0x02 , 0x0002 },
+{ 0x03 , 0x0003 },
+{ 0x04 , 0x0004 },
+{ 0x05 , 0x0005 },
+{ 0x06 , 0x0006 },
+{ 0x07 , 0x0007 },
+{ 0x08 , 0x0008 },
+{ 0x09 , 0x0009 },
+{ 0x0a , 0x000A },
+{ 0x0b , 0x000B },
+{ 0x0c , 0x000C },
+{ 0x0d , 0x000D },
+{ 0x0e , 0x000E },
+{ 0x0f , 0x000F },
+{ 0x10 , 0x0010 },
+{ 0x11 , 0x0011 },
+{ 0x12 , 0x0012 },
+{ 0x13 , 0x0013 },
+{ 0x14 , 0x0014 },
+{ 0x15 , 0x0015 },
+{ 0x16 , 0x0016 },
+{ 0x17 , 0x0017 },
+{ 0x18 , 0x0018 },
+{ 0x19 , 0x0019 },
+{ 0x1a , 0x001A },
+{ 0x1b , 0x001B },
+{ 0x1c , 0x001C },
+{ 0x1d , 0x001D },
+{ 0x1e , 0x001E },
+{ 0x1f , 0x001F },
+{ 0x20 , 0x0020 },
+{ 0x21 , 0x0021 },
+{ 0x22 , 0x0022 },
+{ 0x23 , 0x0023 },
+{ 0x24 , 0x0024 },
+{ 0x25 , 0x0025 },
+{ 0x26 , 0x0026 },
+{ 0x27 , 0x0027 },
+{ 0x28 , 0x0028 },
+{ 0x29 , 0x0029 },
+{ 0x2a , 0x002A },
+{ 0x2b , 0x002B },
+{ 0x2c , 0x002C },
+{ 0x2d , 0x002D },
+{ 0x2e , 0x002E },
+{ 0x2f , 0x002F },
+{ 0x30 , 0x0030 },
+{ 0x31 , 0x0031 },
+{ 0x32 , 0x0032 },
+{ 0x33 , 0x0033 },
+{ 0x34 , 0x0034 },
+{ 0x35 , 0x0035 },
+{ 0x36 , 0x0036 },
+{ 0x37 , 0x0037 },
+{ 0x38 , 0x0038 },
+{ 0x39 , 0x0039 },
+{ 0x3a , 0x003A },
+{ 0x3b , 0x003B },
+{ 0x3c , 0x003C },
+{ 0x3d , 0x003D },
+{ 0x3e , 0x003E },
+{ 0x3f , 0x003F },
+{ 0x40 , 0x0040 },
+{ 0x41 , 0x0041 },
+{ 0x42 , 0x0042 },
+{ 0x43 , 0x0043 },
+{ 0x44 , 0x0044 },
+{ 0x45 , 0x0045 },
+{ 0x46 , 0x0046 },
+{ 0x47 , 0x0047 },
+{ 0x48 , 0x0048 },
+{ 0x49 , 0x0049 },
+{ 0x4a , 0x004A },
+{ 0x4b , 0x004B },
+{ 0x4c , 0x004C },
+{ 0x4d , 0x004D },
+{ 0x4e , 0x004E },
+{ 0x4f , 0x004F },
+{ 0x50 , 0x0050 },
+{ 0x51 , 0x0051 },
+{ 0x52 , 0x0052 },
+{ 0x53 , 0x0053 },
+{ 0x54 , 0x0054 },
+{ 0x55 , 0x0055 },
+{ 0x56 , 0x0056 },
+{ 0x57 , 0x0057 },
+{ 0x58 , 0x0058 },
+{ 0x59 , 0x0059 },
+{ 0x5a , 0x005A },
+{ 0x5b , 0x005B },
+{ 0x5c , 0x005C },
+{ 0x5d , 0x005D },
+{ 0x5e , 0x005E },
+{ 0x5f , 0x005F },
+{ 0x60 , 0x0060 },
+{ 0x61 , 0x0061 },
+{ 0x62 , 0x0062 },
+{ 0x63 , 0x0063 },
+{ 0x64 , 0x0064 },
+{ 0x65 , 0x0065 },
+{ 0x66 , 0x0066 },
+{ 0x67 , 0x0067 },
+{ 0x68 , 0x0068 },
+{ 0x69 , 0x0069 },
+{ 0x6a , 0x006A },
+{ 0x6b , 0x006B },
+{ 0x6c , 0x006C },
+{ 0x6d , 0x006D },
+{ 0x6e , 0x006E },
+{ 0x6f , 0x006F },
+{ 0x70 , 0x0070 },
+{ 0x71 , 0x0071 },
+{ 0x72 , 0x0072 },
+{ 0x73 , 0x0073 },
+{ 0x74 , 0x0074 },
+{ 0x75 , 0x0075 },
+{ 0x76 , 0x0076 },
+{ 0x77 , 0x0077 },
+{ 0x78 , 0x0078 },
+{ 0x79 , 0x0079 },
+{ 0x7a , 0x007A },
+{ 0x7b , 0x007B },
+{ 0x7c , 0x007C },
+{ 0x7d , 0x007D },
+{ 0x7e , 0x007E },
+{ 0x7f , 0x007F },
+{ 0x80 , 0x0080 },
+{ 0x81 , 0x0081 },
+{ 0x82 , 0x0082 },
+{ 0x83 , 0x0083 },
+{ 0x84 , 0x0084 },
+{ 0x85 , 0x0085 },
+{ 0x86 , 0x0086 },
+{ 0x87 , 0x0087 },
+{ 0x88 , 0x0088 },
+{ 0x89 , 0x0089 },
+{ 0x8a , 0x008A },
+{ 0x8b , 0x008B },
+{ 0x8c , 0x008C },
+{ 0x8d , 0x008D },
+{ 0x8e , 0x008E },
+{ 0x8f , 0x008F },
+{ 0x90 , 0x0090 },
+{ 0x91 , 0x0091 },
+{ 0x92 , 0x0092 },
+{ 0x93 , 0x0093 },
+{ 0x94 , 0x0094 },
+{ 0x95 , 0x0095 },
+{ 0x96 , 0x0096 },
+{ 0x97 , 0x0097 },
+{ 0x98 , 0x0098 },
+{ 0x99 , 0x0099 },
+{ 0x9a , 0x009A },
+{ 0x9b , 0x009B },
+{ 0x9c , 0x009C },
+{ 0x9d , 0x009D },
+{ 0x9e , 0x009E },
+{ 0x9f , 0x009F },
+{ 0xa0 , 0x00A0 },
+{ 0xa1 , 0x00A1 },
+{ 0xa2 , 0x00A2 },
+{ 0xa3 , 0x00A3 },
+{ 0xa4 , 0x00A4 },
+{ 0xa5 , 0x00A5 },
+{ 0xa6 , 0x00A6 },
+{ 0xa7 , 0x00A7 },
+{ 0xa8 , 0x00A8 },
+{ 0xa9 , 0x00A9 },
+{ 0xaa , 0x00AA },
+{ 0xab , 0x00AB },
+{ 0xac , 0x00AC },
+{ 0xad , 0x00AD },
+{ 0xae , 0x00AE },
+{ 0xaf , 0x00AF },
+{ 0xb0 , 0x00B0 },
+{ 0xb1 , 0x00B1 },
+{ 0xb2 , 0x00B2 },
+{ 0xb3 , 0x00B3 },
+{ 0xb4 , 0x00B4 },
+{ 0xb5 , 0x00B5 },
+{ 0xb6 , 0x00B6 },
+{ 0xb7 , 0x00B7 },
+{ 0xb8 , 0x00B8 },
+{ 0xb9 , 0x00B9 },
+{ 0xba , 0x00BA },
+{ 0xbb , 0x00BB },
+{ 0xbc , 0x00BC },
+{ 0xbd , 0x00BD },
+{ 0xbe , 0x00BE },
+{ 0xbf , 0x00BF },
+{ 0xc0 , 0x00C0 },
+{ 0xc1 , 0x00C1 },
+{ 0xc2 , 0x00C2 },
+{ 0xc3 , 0x00C3 },
+{ 0xc4 , 0x00C4 },
+{ 0xc5 , 0x00C5 },
+{ 0xc6 , 0x00C6 },
+{ 0xc7 , 0x00C7 },
+{ 0xc8 , 0x00C8 },
+{ 0xc9 , 0x00C9 },
+{ 0xca , 0x00CA },
+{ 0xcb , 0x00CB },
+{ 0xcc , 0x00CC },
+{ 0xcd , 0x00CD },
+{ 0xce , 0x00CE },
+{ 0xcf , 0x00CF },
+{ 0xd0 , 0x00D0 },
+{ 0xd1 , 0x00D1 },
+{ 0xd2 , 0x00D2 },
+{ 0xd3 , 0x00D3 },
+{ 0xd4 , 0x00D4 },
+{ 0xd5 , 0x00D5 },
+{ 0xd6 , 0x00D6 },
+{ 0xd7 , 0x00D7 },
+{ 0xd8 , 0x00D8 },
+{ 0xd9 , 0x00D9 },
+{ 0xda , 0x00DA },
+{ 0xdb , 0x00DB },
+{ 0xdc , 0x00DC },
+{ 0xdd , 0x00DD },
+{ 0xde , 0x00DE },
+{ 0xdf , 0x00DF },
+{ 0xe0 , 0x00E0 },
+{ 0xe1 , 0x00E1 },
+{ 0xe2 , 0x00E2 },
+{ 0xe3 , 0x00E3 },
+{ 0xe4 , 0x00E4 },
+{ 0xe5 , 0x00E5 },
+{ 0xe6 , 0x00E6 },
+{ 0xe7 , 0x00E7 },
+{ 0xe8 , 0x00E8 },
+{ 0xe9 , 0x00E9 },
+{ 0xea , 0x00EA },
+{ 0xeb , 0x00EB },
+{ 0xec , 0x00EC },
+{ 0xed , 0x00ED },
+{ 0xee , 0x00EE },
+{ 0xef , 0x00EF },
+{ 0xf0 , 0x00F0 },
+{ 0xf1 , 0x00F1 },
+{ 0xf2 , 0x00F2 },
+{ 0xf3 , 0x00F3 },
+{ 0xf4 , 0x00F4 },
+{ 0xf5 , 0x00F5 },
+{ 0xf6 , 0x00F6 },
+{ 0xf7 , 0x00F7 },
+{ 0xf8 , 0x00F8 },
+{ 0xf9 , 0x00F9 },
+{ 0xfa , 0x00FA },
+{ 0xfb , 0x00FB },
+{ 0xfc , 0x00FC },
+{ 0xfd , 0x00FD },
+{ 0xfe , 0x00FE },
+{ 0xff , 0x00FF },
diff --git a/extra/locale/charmaps/ISO-8859-10.pairs b/extra/locale/charmaps/ISO-8859-10.pairs
new file mode 100644
index 000000000..a39892fcc
--- /dev/null
+++ b/extra/locale/charmaps/ISO-8859-10.pairs
@@ -0,0 +1,256 @@
+{ 0x00 , 0x0000 },
+{ 0x01 , 0x0001 },
+{ 0x02 , 0x0002 },
+{ 0x03 , 0x0003 },
+{ 0x04 , 0x0004 },
+{ 0x05 , 0x0005 },
+{ 0x06 , 0x0006 },
+{ 0x07 , 0x0007 },
+{ 0x08 , 0x0008 },
+{ 0x09 , 0x0009 },
+{ 0x0a , 0x000A },
+{ 0x0b , 0x000B },
+{ 0x0c , 0x000C },
+{ 0x0d , 0x000D },
+{ 0x0e , 0x000E },
+{ 0x0f , 0x000F },
+{ 0x10 , 0x0010 },
+{ 0x11 , 0x0011 },
+{ 0x12 , 0x0012 },
+{ 0x13 , 0x0013 },
+{ 0x14 , 0x0014 },
+{ 0x15 , 0x0015 },
+{ 0x16 , 0x0016 },
+{ 0x17 , 0x0017 },
+{ 0x18 , 0x0018 },
+{ 0x19 , 0x0019 },
+{ 0x1a , 0x001A },
+{ 0x1b , 0x001B },
+{ 0x1c , 0x001C },
+{ 0x1d , 0x001D },
+{ 0x1e , 0x001E },
+{ 0x1f , 0x001F },
+{ 0x20 , 0x0020 },
+{ 0x21 , 0x0021 },
+{ 0x22 , 0x0022 },
+{ 0x23 , 0x0023 },
+{ 0x24 , 0x0024 },
+{ 0x25 , 0x0025 },
+{ 0x26 , 0x0026 },
+{ 0x27 , 0x0027 },
+{ 0x28 , 0x0028 },
+{ 0x29 , 0x0029 },
+{ 0x2a , 0x002A },
+{ 0x2b , 0x002B },
+{ 0x2c , 0x002C },
+{ 0x2d , 0x002D },
+{ 0x2e , 0x002E },
+{ 0x2f , 0x002F },
+{ 0x30 , 0x0030 },
+{ 0x31 , 0x0031 },
+{ 0x32 , 0x0032 },
+{ 0x33 , 0x0033 },
+{ 0x34 , 0x0034 },
+{ 0x35 , 0x0035 },
+{ 0x36 , 0x0036 },
+{ 0x37 , 0x0037 },
+{ 0x38 , 0x0038 },
+{ 0x39 , 0x0039 },
+{ 0x3a , 0x003A },
+{ 0x3b , 0x003B },
+{ 0x3c , 0x003C },
+{ 0x3d , 0x003D },
+{ 0x3e , 0x003E },
+{ 0x3f , 0x003F },
+{ 0x40 , 0x0040 },
+{ 0x41 , 0x0041 },
+{ 0x42 , 0x0042 },
+{ 0x43 , 0x0043 },
+{ 0x44 , 0x0044 },
+{ 0x45 , 0x0045 },
+{ 0x46 , 0x0046 },
+{ 0x47 , 0x0047 },
+{ 0x48 , 0x0048 },
+{ 0x49 , 0x0049 },
+{ 0x4a , 0x004A },
+{ 0x4b , 0x004B },
+{ 0x4c , 0x004C },
+{ 0x4d , 0x004D },
+{ 0x4e , 0x004E },
+{ 0x4f , 0x004F },
+{ 0x50 , 0x0050 },
+{ 0x51 , 0x0051 },
+{ 0x52 , 0x0052 },
+{ 0x53 , 0x0053 },
+{ 0x54 , 0x0054 },
+{ 0x55 , 0x0055 },
+{ 0x56 , 0x0056 },
+{ 0x57 , 0x0057 },
+{ 0x58 , 0x0058 },
+{ 0x59 , 0x0059 },
+{ 0x5a , 0x005A },
+{ 0x5b , 0x005B },
+{ 0x5c , 0x005C },
+{ 0x5d , 0x005D },
+{ 0x5e , 0x005E },
+{ 0x5f , 0x005F },
+{ 0x60 , 0x0060 },
+{ 0x61 , 0x0061 },
+{ 0x62 , 0x0062 },
+{ 0x63 , 0x0063 },
+{ 0x64 , 0x0064 },
+{ 0x65 , 0x0065 },
+{ 0x66 , 0x0066 },
+{ 0x67 , 0x0067 },
+{ 0x68 , 0x0068 },
+{ 0x69 , 0x0069 },
+{ 0x6a , 0x006A },
+{ 0x6b , 0x006B },
+{ 0x6c , 0x006C },
+{ 0x6d , 0x006D },
+{ 0x6e , 0x006E },
+{ 0x6f , 0x006F },
+{ 0x70 , 0x0070 },
+{ 0x71 , 0x0071 },
+{ 0x72 , 0x0072 },
+{ 0x73 , 0x0073 },
+{ 0x74 , 0x0074 },
+{ 0x75 , 0x0075 },
+{ 0x76 , 0x0076 },
+{ 0x77 , 0x0077 },
+{ 0x78 , 0x0078 },
+{ 0x79 , 0x0079 },
+{ 0x7a , 0x007A },
+{ 0x7b , 0x007B },
+{ 0x7c , 0x007C },
+{ 0x7d , 0x007D },
+{ 0x7e , 0x007E },
+{ 0x7f , 0x007F },
+{ 0x80 , 0x0080 },
+{ 0x81 , 0x0081 },
+{ 0x82 , 0x0082 },
+{ 0x83 , 0x0083 },
+{ 0x84 , 0x0084 },
+{ 0x85 , 0x0085 },
+{ 0x86 , 0x0086 },
+{ 0x87 , 0x0087 },
+{ 0x88 , 0x0088 },
+{ 0x89 , 0x0089 },
+{ 0x8a , 0x008A },
+{ 0x8b , 0x008B },
+{ 0x8c , 0x008C },
+{ 0x8d , 0x008D },
+{ 0x8e , 0x008E },
+{ 0x8f , 0x008F },
+{ 0x90 , 0x0090 },
+{ 0x91 , 0x0091 },
+{ 0x92 , 0x0092 },
+{ 0x93 , 0x0093 },
+{ 0x94 , 0x0094 },
+{ 0x95 , 0x0095 },
+{ 0x96 , 0x0096 },
+{ 0x97 , 0x0097 },
+{ 0x98 , 0x0098 },
+{ 0x99 , 0x0099 },
+{ 0x9a , 0x009A },
+{ 0x9b , 0x009B },
+{ 0x9c , 0x009C },
+{ 0x9d , 0x009D },
+{ 0x9e , 0x009E },
+{ 0x9f , 0x009F },
+{ 0xa0 , 0x00A0 },
+{ 0xa1 , 0x0104 },
+{ 0xa2 , 0x0112 },
+{ 0xa3 , 0x0122 },
+{ 0xa4 , 0x012A },
+{ 0xa5 , 0x0128 },
+{ 0xa6 , 0x0136 },
+{ 0xa7 , 0x00A7 },
+{ 0xa8 , 0x013B },
+{ 0xa9 , 0x0110 },
+{ 0xaa , 0x0160 },
+{ 0xab , 0x0166 },
+{ 0xac , 0x017D },
+{ 0xad , 0x00AD },
+{ 0xae , 0x016A },
+{ 0xaf , 0x014A },
+{ 0xb0 , 0x00B0 },
+{ 0xb1 , 0x0105 },
+{ 0xb2 , 0x0113 },
+{ 0xb3 , 0x0123 },
+{ 0xb4 , 0x012B },
+{ 0xb5 , 0x0129 },
+{ 0xb6 , 0x0137 },
+{ 0xb7 , 0x00B7 },
+{ 0xb8 , 0x013C },
+{ 0xb9 , 0x0111 },
+{ 0xba , 0x0161 },
+{ 0xbb , 0x0167 },
+{ 0xbc , 0x017E },
+{ 0xbd , 0x2015 },
+{ 0xbe , 0x016B },
+{ 0xbf , 0x014B },
+{ 0xc0 , 0x0100 },
+{ 0xc1 , 0x00C1 },
+{ 0xc2 , 0x00C2 },
+{ 0xc3 , 0x00C3 },
+{ 0xc4 , 0x00C4 },
+{ 0xc5 , 0x00C5 },
+{ 0xc6 , 0x00C6 },
+{ 0xc7 , 0x012E },
+{ 0xc8 , 0x010C },
+{ 0xc9 , 0x00C9 },
+{ 0xca , 0x0118 },
+{ 0xcb , 0x00CB },
+{ 0xcc , 0x0116 },
+{ 0xcd , 0x00CD },
+{ 0xce , 0x00CE },
+{ 0xcf , 0x00CF },
+{ 0xd0 , 0x00D0 },
+{ 0xd1 , 0x0145 },
+{ 0xd2 , 0x014C },
+{ 0xd3 , 0x00D3 },
+{ 0xd4 , 0x00D4 },
+{ 0xd5 , 0x00D5 },
+{ 0xd6 , 0x00D6 },
+{ 0xd7 , 0x0168 },
+{ 0xd8 , 0x00D8 },
+{ 0xd9 , 0x0172 },
+{ 0xda , 0x00DA },
+{ 0xdb , 0x00DB },
+{ 0xdc , 0x00DC },
+{ 0xdd , 0x00DD },
+{ 0xde , 0x00DE },
+{ 0xdf , 0x00DF },
+{ 0xe0 , 0x0101 },
+{ 0xe1 , 0x00E1 },
+{ 0xe2 , 0x00E2 },
+{ 0xe3 , 0x00E3 },
+{ 0xe4 , 0x00E4 },
+{ 0xe5 , 0x00E5 },
+{ 0xe6 , 0x00E6 },
+{ 0xe7 , 0x012F },
+{ 0xe8 , 0x010D },
+{ 0xe9 , 0x00E9 },
+{ 0xea , 0x0119 },
+{ 0xeb , 0x00EB },
+{ 0xec , 0x0117 },
+{ 0xed , 0x00ED },
+{ 0xee , 0x00EE },
+{ 0xef , 0x00EF },
+{ 0xf0 , 0x00F0 },
+{ 0xf1 , 0x0146 },
+{ 0xf2 , 0x014D },
+{ 0xf3 , 0x00F3 },
+{ 0xf4 , 0x00F4 },
+{ 0xf5 , 0x00F5 },
+{ 0xf6 , 0x00F6 },
+{ 0xf7 , 0x0169 },
+{ 0xf8 , 0x00F8 },
+{ 0xf9 , 0x0173 },
+{ 0xfa , 0x00FA },
+{ 0xfb , 0x00FB },
+{ 0xfc , 0x00FC },
+{ 0xfd , 0x00FD },
+{ 0xfe , 0x00FE },
+{ 0xff , 0x0138 },
diff --git a/extra/locale/charmaps/ISO-8859-13.pairs b/extra/locale/charmaps/ISO-8859-13.pairs
new file mode 100644
index 000000000..ff95fb449
--- /dev/null
+++ b/extra/locale/charmaps/ISO-8859-13.pairs
@@ -0,0 +1,256 @@
+{ 0x00 , 0x0000 },
+{ 0x01 , 0x0001 },
+{ 0x02 , 0x0002 },
+{ 0x03 , 0x0003 },
+{ 0x04 , 0x0004 },
+{ 0x05 , 0x0005 },
+{ 0x06 , 0x0006 },
+{ 0x07 , 0x0007 },
+{ 0x08 , 0x0008 },
+{ 0x09 , 0x0009 },
+{ 0x0a , 0x000A },
+{ 0x0b , 0x000B },
+{ 0x0c , 0x000C },
+{ 0x0d , 0x000D },
+{ 0x0e , 0x000E },
+{ 0x0f , 0x000F },
+{ 0x10 , 0x0010 },
+{ 0x11 , 0x0011 },
+{ 0x12 , 0x0012 },
+{ 0x13 , 0x0013 },
+{ 0x14 , 0x0014 },
+{ 0x15 , 0x0015 },
+{ 0x16 , 0x0016 },
+{ 0x17 , 0x0017 },
+{ 0x18 , 0x0018 },
+{ 0x19 , 0x0019 },
+{ 0x1a , 0x001A },
+{ 0x1b , 0x001B },
+{ 0x1c , 0x001C },
+{ 0x1d , 0x001D },
+{ 0x1e , 0x001E },
+{ 0x1f , 0x001F },
+{ 0x20 , 0x0020 },
+{ 0x21 , 0x0021 },
+{ 0x22 , 0x0022 },
+{ 0x23 , 0x0023 },
+{ 0x24 , 0x0024 },
+{ 0x25 , 0x0025 },
+{ 0x26 , 0x0026 },
+{ 0x27 , 0x0027 },
+{ 0x28 , 0x0028 },
+{ 0x29 , 0x0029 },
+{ 0x2a , 0x002A },
+{ 0x2b , 0x002B },
+{ 0x2c , 0x002C },
+{ 0x2d , 0x002D },
+{ 0x2e , 0x002E },
+{ 0x2f , 0x002F },
+{ 0x30 , 0x0030 },
+{ 0x31 , 0x0031 },
+{ 0x32 , 0x0032 },
+{ 0x33 , 0x0033 },
+{ 0x34 , 0x0034 },
+{ 0x35 , 0x0035 },
+{ 0x36 , 0x0036 },
+{ 0x37 , 0x0037 },
+{ 0x38 , 0x0038 },
+{ 0x39 , 0x0039 },
+{ 0x3a , 0x003A },
+{ 0x3b , 0x003B },
+{ 0x3c , 0x003C },
+{ 0x3d , 0x003D },
+{ 0x3e , 0x003E },
+{ 0x3f , 0x003F },
+{ 0x40 , 0x0040 },
+{ 0x41 , 0x0041 },
+{ 0x42 , 0x0042 },
+{ 0x43 , 0x0043 },
+{ 0x44 , 0x0044 },
+{ 0x45 , 0x0045 },
+{ 0x46 , 0x0046 },
+{ 0x47 , 0x0047 },
+{ 0x48 , 0x0048 },
+{ 0x49 , 0x0049 },
+{ 0x4a , 0x004A },
+{ 0x4b , 0x004B },
+{ 0x4c , 0x004C },
+{ 0x4d , 0x004D },
+{ 0x4e , 0x004E },
+{ 0x4f , 0x004F },
+{ 0x50 , 0x0050 },
+{ 0x51 , 0x0051 },
+{ 0x52 , 0x0052 },
+{ 0x53 , 0x0053 },
+{ 0x54 , 0x0054 },
+{ 0x55 , 0x0055 },
+{ 0x56 , 0x0056 },
+{ 0x57 , 0x0057 },
+{ 0x58 , 0x0058 },
+{ 0x59 , 0x0059 },
+{ 0x5a , 0x005A },
+{ 0x5b , 0x005B },
+{ 0x5c , 0x005C },
+{ 0x5d , 0x005D },
+{ 0x5e , 0x005E },
+{ 0x5f , 0x005F },
+{ 0x60 , 0x0060 },
+{ 0x61 , 0x0061 },
+{ 0x62 , 0x0062 },
+{ 0x63 , 0x0063 },
+{ 0x64 , 0x0064 },
+{ 0x65 , 0x0065 },
+{ 0x66 , 0x0066 },
+{ 0x67 , 0x0067 },
+{ 0x68 , 0x0068 },
+{ 0x69 , 0x0069 },
+{ 0x6a , 0x006A },
+{ 0x6b , 0x006B },
+{ 0x6c , 0x006C },
+{ 0x6d , 0x006D },
+{ 0x6e , 0x006E },
+{ 0x6f , 0x006F },
+{ 0x70 , 0x0070 },
+{ 0x71 , 0x0071 },
+{ 0x72 , 0x0072 },
+{ 0x73 , 0x0073 },
+{ 0x74 , 0x0074 },
+{ 0x75 , 0x0075 },
+{ 0x76 , 0x0076 },
+{ 0x77 , 0x0077 },
+{ 0x78 , 0x0078 },
+{ 0x79 , 0x0079 },
+{ 0x7a , 0x007A },
+{ 0x7b , 0x007B },
+{ 0x7c , 0x007C },
+{ 0x7d , 0x007D },
+{ 0x7e , 0x007E },
+{ 0x7f , 0x007F },
+{ 0x80 , 0x0080 },
+{ 0x81 , 0x0081 },
+{ 0x82 , 0x0082 },
+{ 0x83 , 0x0083 },
+{ 0x84 , 0x0084 },
+{ 0x85 , 0x0085 },
+{ 0x86 , 0x0086 },
+{ 0x87 , 0x0087 },
+{ 0x88 , 0x0088 },
+{ 0x89 , 0x0089 },
+{ 0x8a , 0x008A },
+{ 0x8b , 0x008B },
+{ 0x8c , 0x008C },
+{ 0x8d , 0x008D },
+{ 0x8e , 0x008E },
+{ 0x8f , 0x008F },
+{ 0x90 , 0x0090 },
+{ 0x91 , 0x0091 },
+{ 0x92 , 0x0092 },
+{ 0x93 , 0x0093 },
+{ 0x94 , 0x0094 },
+{ 0x95 , 0x0095 },
+{ 0x96 , 0x0096 },
+{ 0x97 , 0x0097 },
+{ 0x98 , 0x0098 },
+{ 0x99 , 0x0099 },
+{ 0x9a , 0x009A },
+{ 0x9b , 0x009B },
+{ 0x9c , 0x009C },
+{ 0x9d , 0x009D },
+{ 0x9e , 0x009E },
+{ 0x9f , 0x009F },
+{ 0xa0 , 0x00A0 },
+{ 0xa1 , 0x201D },
+{ 0xa2 , 0x00A2 },
+{ 0xa3 , 0x00A3 },
+{ 0xa4 , 0x00A4 },
+{ 0xa5 , 0x201E },
+{ 0xa6 , 0x00A6 },
+{ 0xa7 , 0x00A7 },
+{ 0xa8 , 0x00D8 },
+{ 0xa9 , 0x00A9 },
+{ 0xaa , 0x0156 },
+{ 0xab , 0x00AB },
+{ 0xac , 0x00AC },
+{ 0xad , 0x00AD },
+{ 0xae , 0x00AE },
+{ 0xaf , 0x00C6 },
+{ 0xb0 , 0x00B0 },
+{ 0xb1 , 0x00B1 },
+{ 0xb2 , 0x00B2 },
+{ 0xb3 , 0x00B3 },
+{ 0xb4 , 0x201C },
+{ 0xb5 , 0x00B5 },
+{ 0xb6 , 0x00B6 },
+{ 0xb7 , 0x00B7 },
+{ 0xb8 , 0x00F8 },
+{ 0xb9 , 0x00B9 },
+{ 0xba , 0x0157 },
+{ 0xbb , 0x00BB },
+{ 0xbc , 0x00BC },
+{ 0xbd , 0x00BD },
+{ 0xbe , 0x00BE },
+{ 0xbf , 0x00E6 },
+{ 0xc0 , 0x0104 },
+{ 0xc1 , 0x012E },
+{ 0xc2 , 0x0100 },
+{ 0xc3 , 0x0106 },
+{ 0xc4 , 0x00C4 },
+{ 0xc5 , 0x00C5 },
+{ 0xc6 , 0x0118 },
+{ 0xc7 , 0x0112 },
+{ 0xc8 , 0x010C },
+{ 0xc9 , 0x00C9 },
+{ 0xca , 0x0179 },
+{ 0xcb , 0x0116 },
+{ 0xcc , 0x0122 },
+{ 0xcd , 0x0136 },
+{ 0xce , 0x012A },
+{ 0xcf , 0x013B },
+{ 0xd0 , 0x0160 },
+{ 0xd1 , 0x0143 },
+{ 0xd2 , 0x0145 },
+{ 0xd3 , 0x00D3 },
+{ 0xd4 , 0x014C },
+{ 0xd5 , 0x00D5 },
+{ 0xd6 , 0x00D6 },
+{ 0xd7 , 0x00D7 },
+{ 0xd8 , 0x0172 },
+{ 0xd9 , 0x0141 },
+{ 0xda , 0x015A },
+{ 0xdb , 0x016A },
+{ 0xdc , 0x00DC },
+{ 0xdd , 0x017B },
+{ 0xde , 0x017D },
+{ 0xdf , 0x00DF },
+{ 0xe0 , 0x0105 },
+{ 0xe1 , 0x012F },
+{ 0xe2 , 0x0101 },
+{ 0xe3 , 0x0107 },
+{ 0xe4 , 0x00E4 },
+{ 0xe5 , 0x00E5 },
+{ 0xe6 , 0x0119 },
+{ 0xe7 , 0x0113 },
+{ 0xe8 , 0x010D },
+{ 0xe9 , 0x00E9 },
+{ 0xea , 0x017A },
+{ 0xeb , 0x0117 },
+{ 0xec , 0x0123 },
+{ 0xed , 0x0137 },
+{ 0xee , 0x012B },
+{ 0xef , 0x013C },
+{ 0xf0 , 0x0161 },
+{ 0xf1 , 0x0144 },
+{ 0xf2 , 0x0146 },
+{ 0xf3 , 0x00F3 },
+{ 0xf4 , 0x014D },
+{ 0xf5 , 0x00F5 },
+{ 0xf6 , 0x00F6 },
+{ 0xf7 , 0x00F7 },
+{ 0xf8 , 0x0173 },
+{ 0xf9 , 0x0142 },
+{ 0xfa , 0x015B },
+{ 0xfb , 0x016B },
+{ 0xfc , 0x00FC },
+{ 0xfd , 0x017C },
+{ 0xfe , 0x017E },
+{ 0xff , 0x2019 },
diff --git a/extra/locale/charmaps/ISO-8859-14.pairs b/extra/locale/charmaps/ISO-8859-14.pairs
new file mode 100644
index 000000000..8a8f06ec0
--- /dev/null
+++ b/extra/locale/charmaps/ISO-8859-14.pairs
@@ -0,0 +1,256 @@
+{ 0x00 , 0x0000 },
+{ 0x01 , 0x0001 },
+{ 0x02 , 0x0002 },
+{ 0x03 , 0x0003 },
+{ 0x04 , 0x0004 },
+{ 0x05 , 0x0005 },
+{ 0x06 , 0x0006 },
+{ 0x07 , 0x0007 },
+{ 0x08 , 0x0008 },
+{ 0x09 , 0x0009 },
+{ 0x0a , 0x000A },
+{ 0x0b , 0x000B },
+{ 0x0c , 0x000C },
+{ 0x0d , 0x000D },
+{ 0x0e , 0x000E },
+{ 0x0f , 0x000F },
+{ 0x10 , 0x0010 },
+{ 0x11 , 0x0011 },
+{ 0x12 , 0x0012 },
+{ 0x13 , 0x0013 },
+{ 0x14 , 0x0014 },
+{ 0x15 , 0x0015 },
+{ 0x16 , 0x0016 },
+{ 0x17 , 0x0017 },
+{ 0x18 , 0x0018 },
+{ 0x19 , 0x0019 },
+{ 0x1a , 0x001A },
+{ 0x1b , 0x001B },
+{ 0x1c , 0x001C },
+{ 0x1d , 0x001D },
+{ 0x1e , 0x001E },
+{ 0x1f , 0x001F },
+{ 0x20 , 0x0020 },
+{ 0x21 , 0x0021 },
+{ 0x22 , 0x0022 },
+{ 0x23 , 0x0023 },
+{ 0x24 , 0x0024 },
+{ 0x25 , 0x0025 },
+{ 0x26 , 0x0026 },
+{ 0x27 , 0x0027 },
+{ 0x28 , 0x0028 },
+{ 0x29 , 0x0029 },
+{ 0x2a , 0x002A },
+{ 0x2b , 0x002B },
+{ 0x2c , 0x002C },
+{ 0x2d , 0x002D },
+{ 0x2e , 0x002E },
+{ 0x2f , 0x002F },
+{ 0x30 , 0x0030 },
+{ 0x31 , 0x0031 },
+{ 0x32 , 0x0032 },
+{ 0x33 , 0x0033 },
+{ 0x34 , 0x0034 },
+{ 0x35 , 0x0035 },
+{ 0x36 , 0x0036 },
+{ 0x37 , 0x0037 },
+{ 0x38 , 0x0038 },
+{ 0x39 , 0x0039 },
+{ 0x3a , 0x003A },
+{ 0x3b , 0x003B },
+{ 0x3c , 0x003C },
+{ 0x3d , 0x003D },
+{ 0x3e , 0x003E },
+{ 0x3f , 0x003F },
+{ 0x40 , 0x0040 },
+{ 0x41 , 0x0041 },
+{ 0x42 , 0x0042 },
+{ 0x43 , 0x0043 },
+{ 0x44 , 0x0044 },
+{ 0x45 , 0x0045 },
+{ 0x46 , 0x0046 },
+{ 0x47 , 0x0047 },
+{ 0x48 , 0x0048 },
+{ 0x49 , 0x0049 },
+{ 0x4a , 0x004A },
+{ 0x4b , 0x004B },
+{ 0x4c , 0x004C },
+{ 0x4d , 0x004D },
+{ 0x4e , 0x004E },
+{ 0x4f , 0x004F },
+{ 0x50 , 0x0050 },
+{ 0x51 , 0x0051 },
+{ 0x52 , 0x0052 },
+{ 0x53 , 0x0053 },
+{ 0x54 , 0x0054 },
+{ 0x55 , 0x0055 },
+{ 0x56 , 0x0056 },
+{ 0x57 , 0x0057 },
+{ 0x58 , 0x0058 },
+{ 0x59 , 0x0059 },
+{ 0x5a , 0x005A },
+{ 0x5b , 0x005B },
+{ 0x5c , 0x005C },
+{ 0x5d , 0x005D },
+{ 0x5e , 0x005E },
+{ 0x5f , 0x005F },
+{ 0x60 , 0x0060 },
+{ 0x61 , 0x0061 },
+{ 0x62 , 0x0062 },
+{ 0x63 , 0x0063 },
+{ 0x64 , 0x0064 },
+{ 0x65 , 0x0065 },
+{ 0x66 , 0x0066 },
+{ 0x67 , 0x0067 },
+{ 0x68 , 0x0068 },
+{ 0x69 , 0x0069 },
+{ 0x6a , 0x006A },
+{ 0x6b , 0x006B },
+{ 0x6c , 0x006C },
+{ 0x6d , 0x006D },
+{ 0x6e , 0x006E },
+{ 0x6f , 0x006F },
+{ 0x70 , 0x0070 },
+{ 0x71 , 0x0071 },
+{ 0x72 , 0x0072 },
+{ 0x73 , 0x0073 },
+{ 0x74 , 0x0074 },
+{ 0x75 , 0x0075 },
+{ 0x76 , 0x0076 },
+{ 0x77 , 0x0077 },
+{ 0x78 , 0x0078 },
+{ 0x79 , 0x0079 },
+{ 0x7a , 0x007A },
+{ 0x7b , 0x007B },
+{ 0x7c , 0x007C },
+{ 0x7d , 0x007D },
+{ 0x7e , 0x007E },
+{ 0x7f , 0x007F },
+{ 0x80 , 0x0080 },
+{ 0x81 , 0x0081 },
+{ 0x82 , 0x0082 },
+{ 0x83 , 0x0083 },
+{ 0x84 , 0x0084 },
+{ 0x85 , 0x0085 },
+{ 0x86 , 0x0086 },
+{ 0x87 , 0x0087 },
+{ 0x88 , 0x0088 },
+{ 0x89 , 0x0089 },
+{ 0x8a , 0x008A },
+{ 0x8b , 0x008B },
+{ 0x8c , 0x008C },
+{ 0x8d , 0x008D },
+{ 0x8e , 0x008E },
+{ 0x8f , 0x008F },
+{ 0x90 , 0x0090 },
+{ 0x91 , 0x0091 },
+{ 0x92 , 0x0092 },
+{ 0x93 , 0x0093 },
+{ 0x94 , 0x0094 },
+{ 0x95 , 0x0095 },
+{ 0x96 , 0x0096 },
+{ 0x97 , 0x0097 },
+{ 0x98 , 0x0098 },
+{ 0x99 , 0x0099 },
+{ 0x9a , 0x009A },
+{ 0x9b , 0x009B },
+{ 0x9c , 0x009C },
+{ 0x9d , 0x009D },
+{ 0x9e , 0x009E },
+{ 0x9f , 0x009F },
+{ 0xa0 , 0x00A0 },
+{ 0xa1 , 0x1E02 },
+{ 0xa2 , 0x1E03 },
+{ 0xa3 , 0x00A3 },
+{ 0xa4 , 0x010A },
+{ 0xa5 , 0x010B },
+{ 0xa6 , 0x1E0A },
+{ 0xa7 , 0x00A7 },
+{ 0xa8 , 0x1E80 },
+{ 0xa9 , 0x00A9 },
+{ 0xaa , 0x1E82 },
+{ 0xab , 0x1E0B },
+{ 0xac , 0x1EF2 },
+{ 0xad , 0x00AD },
+{ 0xae , 0x00AE },
+{ 0xaf , 0x0178 },
+{ 0xb0 , 0x1E1E },
+{ 0xb1 , 0x1E1F },
+{ 0xb2 , 0x0120 },
+{ 0xb3 , 0x0121 },
+{ 0xb4 , 0x1E40 },
+{ 0xb5 , 0x1E41 },
+{ 0xb6 , 0x00B6 },
+{ 0xb7 , 0x1E56 },
+{ 0xb8 , 0x1E81 },
+{ 0xb9 , 0x1E57 },
+{ 0xba , 0x1E83 },
+{ 0xbb , 0x1E60 },
+{ 0xbc , 0x1EF3 },
+{ 0xbd , 0x1E84 },
+{ 0xbe , 0x1E85 },
+{ 0xbf , 0x1E61 },
+{ 0xc0 , 0x00C0 },
+{ 0xc1 , 0x00C1 },
+{ 0xc2 , 0x00C2 },
+{ 0xc3 , 0x00C3 },
+{ 0xc4 , 0x00C4 },
+{ 0xc5 , 0x00C5 },
+{ 0xc6 , 0x00C6 },
+{ 0xc7 , 0x00C7 },
+{ 0xc8 , 0x00C8 },
+{ 0xc9 , 0x00C9 },
+{ 0xca , 0x00CA },
+{ 0xcb , 0x00CB },
+{ 0xcc , 0x00CC },
+{ 0xcd , 0x00CD },
+{ 0xce , 0x00CE },
+{ 0xcf , 0x00CF },
+{ 0xd0 , 0x0174 },
+{ 0xd1 , 0x00D1 },
+{ 0xd2 , 0x00D2 },
+{ 0xd3 , 0x00D3 },
+{ 0xd4 , 0x00D4 },
+{ 0xd5 , 0x00D5 },
+{ 0xd6 , 0x00D6 },
+{ 0xd7 , 0x1E6A },
+{ 0xd8 , 0x00D8 },
+{ 0xd9 , 0x00D9 },
+{ 0xda , 0x00DA },
+{ 0xdb , 0x00DB },
+{ 0xdc , 0x00DC },
+{ 0xdd , 0x00DD },
+{ 0xde , 0x0176 },
+{ 0xdf , 0x00DF },
+{ 0xe0 , 0x00E0 },
+{ 0xe1 , 0x00E1 },
+{ 0xe2 , 0x00E2 },
+{ 0xe3 , 0x00E3 },
+{ 0xe4 , 0x00E4 },
+{ 0xe5 , 0x00E5 },
+{ 0xe6 , 0x00E6 },
+{ 0xe7 , 0x00E7 },
+{ 0xe8 , 0x00E8 },
+{ 0xe9 , 0x00E9 },
+{ 0xea , 0x00EA },
+{ 0xeb , 0x00EB },
+{ 0xec , 0x00EC },
+{ 0xed , 0x00ED },
+{ 0xee , 0x00EE },
+{ 0xef , 0x00EF },
+{ 0xf0 , 0x0175 },
+{ 0xf1 , 0x00F1 },
+{ 0xf2 , 0x00F2 },
+{ 0xf3 , 0x00F3 },
+{ 0xf4 , 0x00F4 },
+{ 0xf5 , 0x00F5 },
+{ 0xf6 , 0x00F6 },
+{ 0xf7 , 0x1E6B },
+{ 0xf8 , 0x00F8 },
+{ 0xf9 , 0x00F9 },
+{ 0xfa , 0x00FA },
+{ 0xfb , 0x00FB },
+{ 0xfc , 0x00FC },
+{ 0xfd , 0x00FD },
+{ 0xfe , 0x0177 },
+{ 0xff , 0x00FF },
diff --git a/extra/locale/charmaps/ISO-8859-15.pairs b/extra/locale/charmaps/ISO-8859-15.pairs
new file mode 100644
index 000000000..ca50e22bd
--- /dev/null
+++ b/extra/locale/charmaps/ISO-8859-15.pairs
@@ -0,0 +1,256 @@
+{ 0x00 , 0x0000 },
+{ 0x01 , 0x0001 },
+{ 0x02 , 0x0002 },
+{ 0x03 , 0x0003 },
+{ 0x04 , 0x0004 },
+{ 0x05 , 0x0005 },
+{ 0x06 , 0x0006 },
+{ 0x07 , 0x0007 },
+{ 0x08 , 0x0008 },
+{ 0x09 , 0x0009 },
+{ 0x0a , 0x000A },
+{ 0x0b , 0x000B },
+{ 0x0c , 0x000C },
+{ 0x0d , 0x000D },
+{ 0x0e , 0x000E },
+{ 0x0f , 0x000F },
+{ 0x10 , 0x0010 },
+{ 0x11 , 0x0011 },
+{ 0x12 , 0x0012 },
+{ 0x13 , 0x0013 },
+{ 0x14 , 0x0014 },
+{ 0x15 , 0x0015 },
+{ 0x16 , 0x0016 },
+{ 0x17 , 0x0017 },
+{ 0x18 , 0x0018 },
+{ 0x19 , 0x0019 },
+{ 0x1a , 0x001A },
+{ 0x1b , 0x001B },
+{ 0x1c , 0x001C },
+{ 0x1d , 0x001D },
+{ 0x1e , 0x001E },
+{ 0x1f , 0x001F },
+{ 0x20 , 0x0020 },
+{ 0x21 , 0x0021 },
+{ 0x22 , 0x0022 },
+{ 0x23 , 0x0023 },
+{ 0x24 , 0x0024 },
+{ 0x25 , 0x0025 },
+{ 0x26 , 0x0026 },
+{ 0x27 , 0x0027 },
+{ 0x28 , 0x0028 },
+{ 0x29 , 0x0029 },
+{ 0x2a , 0x002A },
+{ 0x2b , 0x002B },
+{ 0x2c , 0x002C },
+{ 0x2d , 0x002D },
+{ 0x2e , 0x002E },
+{ 0x2f , 0x002F },
+{ 0x30 , 0x0030 },
+{ 0x31 , 0x0031 },
+{ 0x32 , 0x0032 },
+{ 0x33 , 0x0033 },
+{ 0x34 , 0x0034 },
+{ 0x35 , 0x0035 },
+{ 0x36 , 0x0036 },
+{ 0x37 , 0x0037 },
+{ 0x38 , 0x0038 },
+{ 0x39 , 0x0039 },
+{ 0x3a , 0x003A },
+{ 0x3b , 0x003B },
+{ 0x3c , 0x003C },
+{ 0x3d , 0x003D },
+{ 0x3e , 0x003E },
+{ 0x3f , 0x003F },
+{ 0x40 , 0x0040 },
+{ 0x41 , 0x0041 },
+{ 0x42 , 0x0042 },
+{ 0x43 , 0x0043 },
+{ 0x44 , 0x0044 },
+{ 0x45 , 0x0045 },
+{ 0x46 , 0x0046 },
+{ 0x47 , 0x0047 },
+{ 0x48 , 0x0048 },
+{ 0x49 , 0x0049 },
+{ 0x4a , 0x004A },
+{ 0x4b , 0x004B },
+{ 0x4c , 0x004C },
+{ 0x4d , 0x004D },
+{ 0x4e , 0x004E },
+{ 0x4f , 0x004F },
+{ 0x50 , 0x0050 },
+{ 0x51 , 0x0051 },
+{ 0x52 , 0x0052 },
+{ 0x53 , 0x0053 },
+{ 0x54 , 0x0054 },
+{ 0x55 , 0x0055 },
+{ 0x56 , 0x0056 },
+{ 0x57 , 0x0057 },
+{ 0x58 , 0x0058 },
+{ 0x59 , 0x0059 },
+{ 0x5a , 0x005A },
+{ 0x5b , 0x005B },
+{ 0x5c , 0x005C },
+{ 0x5d , 0x005D },
+{ 0x5e , 0x005E },
+{ 0x5f , 0x005F },
+{ 0x60 , 0x0060 },
+{ 0x61 , 0x0061 },
+{ 0x62 , 0x0062 },
+{ 0x63 , 0x0063 },
+{ 0x64 , 0x0064 },
+{ 0x65 , 0x0065 },
+{ 0x66 , 0x0066 },
+{ 0x67 , 0x0067 },
+{ 0x68 , 0x0068 },
+{ 0x69 , 0x0069 },
+{ 0x6a , 0x006A },
+{ 0x6b , 0x006B },
+{ 0x6c , 0x006C },
+{ 0x6d , 0x006D },
+{ 0x6e , 0x006E },
+{ 0x6f , 0x006F },
+{ 0x70 , 0x0070 },
+{ 0x71 , 0x0071 },
+{ 0x72 , 0x0072 },
+{ 0x73 , 0x0073 },
+{ 0x74 , 0x0074 },
+{ 0x75 , 0x0075 },
+{ 0x76 , 0x0076 },
+{ 0x77 , 0x0077 },
+{ 0x78 , 0x0078 },
+{ 0x79 , 0x0079 },
+{ 0x7a , 0x007A },
+{ 0x7b , 0x007B },
+{ 0x7c , 0x007C },
+{ 0x7d , 0x007D },
+{ 0x7e , 0x007E },
+{ 0x7f , 0x007F },
+{ 0x80 , 0x0080 },
+{ 0x81 , 0x0081 },
+{ 0x82 , 0x0082 },
+{ 0x83 , 0x0083 },
+{ 0x84 , 0x0084 },
+{ 0x85 , 0x0085 },
+{ 0x86 , 0x0086 },
+{ 0x87 , 0x0087 },
+{ 0x88 , 0x0088 },
+{ 0x89 , 0x0089 },
+{ 0x8a , 0x008A },
+{ 0x8b , 0x008B },
+{ 0x8c , 0x008C },
+{ 0x8d , 0x008D },
+{ 0x8e , 0x008E },
+{ 0x8f , 0x008F },
+{ 0x90 , 0x0090 },
+{ 0x91 , 0x0091 },
+{ 0x92 , 0x0092 },
+{ 0x93 , 0x0093 },
+{ 0x94 , 0x0094 },
+{ 0x95 , 0x0095 },
+{ 0x96 , 0x0096 },
+{ 0x97 , 0x0097 },
+{ 0x98 , 0x0098 },
+{ 0x99 , 0x0099 },
+{ 0x9a , 0x009A },
+{ 0x9b , 0x009B },
+{ 0x9c , 0x009C },
+{ 0x9d , 0x009D },
+{ 0x9e , 0x009E },
+{ 0x9f , 0x009F },
+{ 0xa0 , 0x00A0 },
+{ 0xa1 , 0x00A1 },
+{ 0xa2 , 0x00A2 },
+{ 0xa3 , 0x00A3 },
+{ 0xa4 , 0x20AC },
+{ 0xa5 , 0x00A5 },
+{ 0xa6 , 0x0160 },
+{ 0xa7 , 0x00A7 },
+{ 0xa8 , 0x0161 },
+{ 0xa9 , 0x00A9 },
+{ 0xaa , 0x00AA },
+{ 0xab , 0x00AB },
+{ 0xac , 0x00AC },
+{ 0xad , 0x00AD },
+{ 0xae , 0x00AE },
+{ 0xaf , 0x00AF },
+{ 0xb0 , 0x00B0 },
+{ 0xb1 , 0x00B1 },
+{ 0xb2 , 0x00B2 },
+{ 0xb3 , 0x00B3 },
+{ 0xb4 , 0x017D },
+{ 0xb5 , 0x00B5 },
+{ 0xb6 , 0x00B6 },
+{ 0xb7 , 0x00B7 },
+{ 0xb8 , 0x017E },
+{ 0xb9 , 0x00B9 },
+{ 0xba , 0x00BA },
+{ 0xbb , 0x00BB },
+{ 0xbc , 0x0152 },
+{ 0xbd , 0x0153 },
+{ 0xbe , 0x0178 },
+{ 0xbf , 0x00BF },
+{ 0xc0 , 0x00C0 },
+{ 0xc1 , 0x00C1 },
+{ 0xc2 , 0x00C2 },
+{ 0xc3 , 0x00C3 },
+{ 0xc4 , 0x00C4 },
+{ 0xc5 , 0x00C5 },
+{ 0xc6 , 0x00C6 },
+{ 0xc7 , 0x00C7 },
+{ 0xc8 , 0x00C8 },
+{ 0xc9 , 0x00C9 },
+{ 0xca , 0x00CA },
+{ 0xcb , 0x00CB },
+{ 0xcc , 0x00CC },
+{ 0xcd , 0x00CD },
+{ 0xce , 0x00CE },
+{ 0xcf , 0x00CF },
+{ 0xd0 , 0x00D0 },
+{ 0xd1 , 0x00D1 },
+{ 0xd2 , 0x00D2 },
+{ 0xd3 , 0x00D3 },
+{ 0xd4 , 0x00D4 },
+{ 0xd5 , 0x00D5 },
+{ 0xd6 , 0x00D6 },
+{ 0xd7 , 0x00D7 },
+{ 0xd8 , 0x00D8 },
+{ 0xd9 , 0x00D9 },
+{ 0xda , 0x00DA },
+{ 0xdb , 0x00DB },
+{ 0xdc , 0x00DC },
+{ 0xdd , 0x00DD },
+{ 0xde , 0x00DE },
+{ 0xdf , 0x00DF },
+{ 0xe0 , 0x00E0 },
+{ 0xe1 , 0x00E1 },
+{ 0xe2 , 0x00E2 },
+{ 0xe3 , 0x00E3 },
+{ 0xe4 , 0x00E4 },
+{ 0xe5 , 0x00E5 },
+{ 0xe6 , 0x00E6 },
+{ 0xe7 , 0x00E7 },
+{ 0xe8 , 0x00E8 },
+{ 0xe9 , 0x00E9 },
+{ 0xea , 0x00EA },
+{ 0xeb , 0x00EB },
+{ 0xec , 0x00EC },
+{ 0xed , 0x00ED },
+{ 0xee , 0x00EE },
+{ 0xef , 0x00EF },
+{ 0xf0 , 0x00F0 },
+{ 0xf1 , 0x00F1 },
+{ 0xf2 , 0x00F2 },
+{ 0xf3 , 0x00F3 },
+{ 0xf4 , 0x00F4 },
+{ 0xf5 , 0x00F5 },
+{ 0xf6 , 0x00F6 },
+{ 0xf7 , 0x00F7 },
+{ 0xf8 , 0x00F8 },
+{ 0xf9 , 0x00F9 },
+{ 0xfa , 0x00FA },
+{ 0xfb , 0x00FB },
+{ 0xfc , 0x00FC },
+{ 0xfd , 0x00FD },
+{ 0xfe , 0x00FE },
+{ 0xff , 0x00FF },
diff --git a/extra/locale/charmaps/ISO-8859-16.pairs b/extra/locale/charmaps/ISO-8859-16.pairs
new file mode 100644
index 000000000..6b749639e
--- /dev/null
+++ b/extra/locale/charmaps/ISO-8859-16.pairs
@@ -0,0 +1,256 @@
+{ 0x00 , 0x0000 },
+{ 0x01 , 0x0001 },
+{ 0x02 , 0x0002 },
+{ 0x03 , 0x0003 },
+{ 0x04 , 0x0004 },
+{ 0x05 , 0x0005 },
+{ 0x06 , 0x0006 },
+{ 0x07 , 0x0007 },
+{ 0x08 , 0x0008 },
+{ 0x09 , 0x0009 },
+{ 0x0a , 0x000A },
+{ 0x0b , 0x000B },
+{ 0x0c , 0x000C },
+{ 0x0d , 0x000D },
+{ 0x0e , 0x000E },
+{ 0x0f , 0x000F },
+{ 0x10 , 0x0010 },
+{ 0x11 , 0x0011 },
+{ 0x12 , 0x0012 },
+{ 0x13 , 0x0013 },
+{ 0x14 , 0x0014 },
+{ 0x15 , 0x0015 },
+{ 0x16 , 0x0016 },
+{ 0x17 , 0x0017 },
+{ 0x18 , 0x0018 },
+{ 0x19 , 0x0019 },
+{ 0x1a , 0x001A },
+{ 0x1b , 0x001B },
+{ 0x1c , 0x001C },
+{ 0x1d , 0x001D },
+{ 0x1e , 0x001E },
+{ 0x1f , 0x001F },
+{ 0x20 , 0x0020 },
+{ 0x21 , 0x0021 },
+{ 0x22 , 0x0022 },
+{ 0x23 , 0x0023 },
+{ 0x24 , 0x0024 },
+{ 0x25 , 0x0025 },
+{ 0x26 , 0x0026 },
+{ 0x27 , 0x0027 },
+{ 0x28 , 0x0028 },
+{ 0x29 , 0x0029 },
+{ 0x2a , 0x002A },
+{ 0x2b , 0x002B },
+{ 0x2c , 0x002C },
+{ 0x2d , 0x002D },
+{ 0x2e , 0x002E },
+{ 0x2f , 0x002F },
+{ 0x30 , 0x0030 },
+{ 0x31 , 0x0031 },
+{ 0x32 , 0x0032 },
+{ 0x33 , 0x0033 },
+{ 0x34 , 0x0034 },
+{ 0x35 , 0x0035 },
+{ 0x36 , 0x0036 },
+{ 0x37 , 0x0037 },
+{ 0x38 , 0x0038 },
+{ 0x39 , 0x0039 },
+{ 0x3a , 0x003A },
+{ 0x3b , 0x003B },
+{ 0x3c , 0x003C },
+{ 0x3d , 0x003D },
+{ 0x3e , 0x003E },
+{ 0x3f , 0x003F },
+{ 0x40 , 0x0040 },
+{ 0x41 , 0x0041 },
+{ 0x42 , 0x0042 },
+{ 0x43 , 0x0043 },
+{ 0x44 , 0x0044 },
+{ 0x45 , 0x0045 },
+{ 0x46 , 0x0046 },
+{ 0x47 , 0x0047 },
+{ 0x48 , 0x0048 },
+{ 0x49 , 0x0049 },
+{ 0x4a , 0x004A },
+{ 0x4b , 0x004B },
+{ 0x4c , 0x004C },
+{ 0x4d , 0x004D },
+{ 0x4e , 0x004E },
+{ 0x4f , 0x004F },
+{ 0x50 , 0x0050 },
+{ 0x51 , 0x0051 },
+{ 0x52 , 0x0052 },
+{ 0x53 , 0x0053 },
+{ 0x54 , 0x0054 },
+{ 0x55 , 0x0055 },
+{ 0x56 , 0x0056 },
+{ 0x57 , 0x0057 },
+{ 0x58 , 0x0058 },
+{ 0x59 , 0x0059 },
+{ 0x5a , 0x005A },
+{ 0x5b , 0x005B },
+{ 0x5c , 0x005C },
+{ 0x5d , 0x005D },
+{ 0x5e , 0x005E },
+{ 0x5f , 0x005F },
+{ 0x60 , 0x0060 },
+{ 0x61 , 0x0061 },
+{ 0x62 , 0x0062 },
+{ 0x63 , 0x0063 },
+{ 0x64 , 0x0064 },
+{ 0x65 , 0x0065 },
+{ 0x66 , 0x0066 },
+{ 0x67 , 0x0067 },
+{ 0x68 , 0x0068 },
+{ 0x69 , 0x0069 },
+{ 0x6a , 0x006A },
+{ 0x6b , 0x006B },
+{ 0x6c , 0x006C },
+{ 0x6d , 0x006D },
+{ 0x6e , 0x006E },
+{ 0x6f , 0x006F },
+{ 0x70 , 0x0070 },
+{ 0x71 , 0x0071 },
+{ 0x72 , 0x0072 },
+{ 0x73 , 0x0073 },
+{ 0x74 , 0x0074 },
+{ 0x75 , 0x0075 },
+{ 0x76 , 0x0076 },
+{ 0x77 , 0x0077 },
+{ 0x78 , 0x0078 },
+{ 0x79 , 0x0079 },
+{ 0x7a , 0x007A },
+{ 0x7b , 0x007B },
+{ 0x7c , 0x007C },
+{ 0x7d , 0x007D },
+{ 0x7e , 0x007E },
+{ 0x7f , 0x007F },
+{ 0x80 , 0x0080 },
+{ 0x81 , 0x0081 },
+{ 0x82 , 0x0082 },
+{ 0x83 , 0x0083 },
+{ 0x84 , 0x0084 },
+{ 0x85 , 0x0085 },
+{ 0x86 , 0x0086 },
+{ 0x87 , 0x0087 },
+{ 0x88 , 0x0088 },
+{ 0x89 , 0x0089 },
+{ 0x8a , 0x008A },
+{ 0x8b , 0x008B },
+{ 0x8c , 0x008C },
+{ 0x8d , 0x008D },
+{ 0x8e , 0x008E },
+{ 0x8f , 0x008F },
+{ 0x90 , 0x0090 },
+{ 0x91 , 0x0091 },
+{ 0x92 , 0x0092 },
+{ 0x93 , 0x0093 },
+{ 0x94 , 0x0094 },
+{ 0x95 , 0x0095 },
+{ 0x96 , 0x0096 },
+{ 0x97 , 0x0097 },
+{ 0x98 , 0x0098 },
+{ 0x99 , 0x0099 },
+{ 0x9a , 0x009A },
+{ 0x9b , 0x009B },
+{ 0x9c , 0x009C },
+{ 0x9d , 0x009D },
+{ 0x9e , 0x009E },
+{ 0x9f , 0x009F },
+{ 0xa0 , 0x00A0 },
+{ 0xa1 , 0x0104 },
+{ 0xa2 , 0x0105 },
+{ 0xa3 , 0x0141 },
+{ 0xa4 , 0x20AC },
+{ 0xa5 , 0x00AB },
+{ 0xa6 , 0x0160 },
+{ 0xa7 , 0x00A7 },
+{ 0xa8 , 0x0161 },
+{ 0xa9 , 0x00A9 },
+{ 0xaa , 0x0218 },
+{ 0xab , 0x201E },
+{ 0xac , 0x0179 },
+{ 0xad , 0x00AD },
+{ 0xae , 0x017A },
+{ 0xaf , 0x017B },
+{ 0xb0 , 0x00B0 },
+{ 0xb1 , 0x00B1 },
+{ 0xb2 , 0x010C },
+{ 0xb3 , 0x0142 },
+{ 0xb4 , 0x017D },
+{ 0xb5 , 0x201D },
+{ 0xb6 , 0x00B6 },
+{ 0xb7 , 0x00B7 },
+{ 0xb8 , 0x017E },
+{ 0xb9 , 0x010D },
+{ 0xba , 0x0219 },
+{ 0xbb , 0x00BB },
+{ 0xbc , 0x0152 },
+{ 0xbd , 0x0153 },
+{ 0xbe , 0x0178 },
+{ 0xbf , 0x017C },
+{ 0xc0 , 0x00C0 },
+{ 0xc1 , 0x00C1 },
+{ 0xc2 , 0x00C2 },
+{ 0xc3 , 0x0102 },
+{ 0xc4 , 0x00C4 },
+{ 0xc5 , 0x0106 },
+{ 0xc6 , 0x00C6 },
+{ 0xc7 , 0x00C7 },
+{ 0xc8 , 0x00C8 },
+{ 0xc9 , 0x00C9 },
+{ 0xca , 0x00CA },
+{ 0xcb , 0x00CB },
+{ 0xcc , 0x00CC },
+{ 0xcd , 0x00CD },
+{ 0xce , 0x00CE },
+{ 0xcf , 0x00CF },
+{ 0xd0 , 0x0110 },
+{ 0xd1 , 0x0143 },
+{ 0xd2 , 0x00D2 },
+{ 0xd3 , 0x00D3 },
+{ 0xd4 , 0x00D4 },
+{ 0xd5 , 0x0150 },
+{ 0xd6 , 0x00D6 },
+{ 0xd7 , 0x015A },
+{ 0xd8 , 0x0170 },
+{ 0xd9 , 0x00D9 },
+{ 0xda , 0x00DA },
+{ 0xdb , 0x00DB },
+{ 0xdc , 0x00DC },
+{ 0xdd , 0x0118 },
+{ 0xde , 0x021A },
+{ 0xdf , 0x00DF },
+{ 0xe0 , 0x00E0 },
+{ 0xe1 , 0x00E1 },
+{ 0xe2 , 0x00E2 },
+{ 0xe3 , 0x0103 },
+{ 0xe4 , 0x00E4 },
+{ 0xe5 , 0x0107 },
+{ 0xe6 , 0x00E6 },
+{ 0xe7 , 0x00E7 },
+{ 0xe8 , 0x00E8 },
+{ 0xe9 , 0x00E9 },
+{ 0xea , 0x00EA },
+{ 0xeb , 0x00EB },
+{ 0xec , 0x00EC },
+{ 0xed , 0x00ED },
+{ 0xee , 0x00EE },
+{ 0xef , 0x00EF },
+{ 0xf0 , 0x0111 },
+{ 0xf1 , 0x0144 },
+{ 0xf2 , 0x00F2 },
+{ 0xf3 , 0x00F3 },
+{ 0xf4 , 0x00F4 },
+{ 0xf5 , 0x0151 },
+{ 0xf6 , 0x00F6 },
+{ 0xf7 , 0x015B },
+{ 0xf8 , 0x0171 },
+{ 0xf9 , 0x00F9 },
+{ 0xfa , 0x00FA },
+{ 0xfb , 0x00FB },
+{ 0xfc , 0x00FC },
+{ 0xfd , 0x0119 },
+{ 0xfe , 0x021B },
+{ 0xff , 0x00FF },
diff --git a/extra/locale/charmaps/ISO-8859-2.pairs b/extra/locale/charmaps/ISO-8859-2.pairs
new file mode 100644
index 000000000..4e92f9432
--- /dev/null
+++ b/extra/locale/charmaps/ISO-8859-2.pairs
@@ -0,0 +1,256 @@
+{ 0x00 , 0x0000 },
+{ 0x01 , 0x0001 },
+{ 0x02 , 0x0002 },
+{ 0x03 , 0x0003 },
+{ 0x04 , 0x0004 },
+{ 0x05 , 0x0005 },
+{ 0x06 , 0x0006 },
+{ 0x07 , 0x0007 },
+{ 0x08 , 0x0008 },
+{ 0x09 , 0x0009 },
+{ 0x0a , 0x000A },
+{ 0x0b , 0x000B },
+{ 0x0c , 0x000C },
+{ 0x0d , 0x000D },
+{ 0x0e , 0x000E },
+{ 0x0f , 0x000F },
+{ 0x10 , 0x0010 },
+{ 0x11 , 0x0011 },
+{ 0x12 , 0x0012 },
+{ 0x13 , 0x0013 },
+{ 0x14 , 0x0014 },
+{ 0x15 , 0x0015 },
+{ 0x16 , 0x0016 },
+{ 0x17 , 0x0017 },
+{ 0x18 , 0x0018 },
+{ 0x19 , 0x0019 },
+{ 0x1a , 0x001A },
+{ 0x1b , 0x001B },
+{ 0x1c , 0x001C },
+{ 0x1d , 0x001D },
+{ 0x1e , 0x001E },
+{ 0x1f , 0x001F },
+{ 0x20 , 0x0020 },
+{ 0x21 , 0x0021 },
+{ 0x22 , 0x0022 },
+{ 0x23 , 0x0023 },
+{ 0x24 , 0x0024 },
+{ 0x25 , 0x0025 },
+{ 0x26 , 0x0026 },
+{ 0x27 , 0x0027 },
+{ 0x28 , 0x0028 },
+{ 0x29 , 0x0029 },
+{ 0x2a , 0x002A },
+{ 0x2b , 0x002B },
+{ 0x2c , 0x002C },
+{ 0x2d , 0x002D },
+{ 0x2e , 0x002E },
+{ 0x2f , 0x002F },
+{ 0x30 , 0x0030 },
+{ 0x31 , 0x0031 },
+{ 0x32 , 0x0032 },
+{ 0x33 , 0x0033 },
+{ 0x34 , 0x0034 },
+{ 0x35 , 0x0035 },
+{ 0x36 , 0x0036 },
+{ 0x37 , 0x0037 },
+{ 0x38 , 0x0038 },
+{ 0x39 , 0x0039 },
+{ 0x3a , 0x003A },
+{ 0x3b , 0x003B },
+{ 0x3c , 0x003C },
+{ 0x3d , 0x003D },
+{ 0x3e , 0x003E },
+{ 0x3f , 0x003F },
+{ 0x40 , 0x0040 },
+{ 0x41 , 0x0041 },
+{ 0x42 , 0x0042 },
+{ 0x43 , 0x0043 },
+{ 0x44 , 0x0044 },
+{ 0x45 , 0x0045 },
+{ 0x46 , 0x0046 },
+{ 0x47 , 0x0047 },
+{ 0x48 , 0x0048 },
+{ 0x49 , 0x0049 },
+{ 0x4a , 0x004A },
+{ 0x4b , 0x004B },
+{ 0x4c , 0x004C },
+{ 0x4d , 0x004D },
+{ 0x4e , 0x004E },
+{ 0x4f , 0x004F },
+{ 0x50 , 0x0050 },
+{ 0x51 , 0x0051 },
+{ 0x52 , 0x0052 },
+{ 0x53 , 0x0053 },
+{ 0x54 , 0x0054 },
+{ 0x55 , 0x0055 },
+{ 0x56 , 0x0056 },
+{ 0x57 , 0x0057 },
+{ 0x58 , 0x0058 },
+{ 0x59 , 0x0059 },
+{ 0x5a , 0x005A },
+{ 0x5b , 0x005B },
+{ 0x5c , 0x005C },
+{ 0x5d , 0x005D },
+{ 0x5e , 0x005E },
+{ 0x5f , 0x005F },
+{ 0x60 , 0x0060 },
+{ 0x61 , 0x0061 },
+{ 0x62 , 0x0062 },
+{ 0x63 , 0x0063 },
+{ 0x64 , 0x0064 },
+{ 0x65 , 0x0065 },
+{ 0x66 , 0x0066 },
+{ 0x67 , 0x0067 },
+{ 0x68 , 0x0068 },
+{ 0x69 , 0x0069 },
+{ 0x6a , 0x006A },
+{ 0x6b , 0x006B },
+{ 0x6c , 0x006C },
+{ 0x6d , 0x006D },
+{ 0x6e , 0x006E },
+{ 0x6f , 0x006F },
+{ 0x70 , 0x0070 },
+{ 0x71 , 0x0071 },
+{ 0x72 , 0x0072 },
+{ 0x73 , 0x0073 },
+{ 0x74 , 0x0074 },
+{ 0x75 , 0x0075 },
+{ 0x76 , 0x0076 },
+{ 0x77 , 0x0077 },
+{ 0x78 , 0x0078 },
+{ 0x79 , 0x0079 },
+{ 0x7a , 0x007A },
+{ 0x7b , 0x007B },
+{ 0x7c , 0x007C },
+{ 0x7d , 0x007D },
+{ 0x7e , 0x007E },
+{ 0x7f , 0x007F },
+{ 0x80 , 0x0080 },
+{ 0x81 , 0x0081 },
+{ 0x82 , 0x0082 },
+{ 0x83 , 0x0083 },
+{ 0x84 , 0x0084 },
+{ 0x85 , 0x0085 },
+{ 0x86 , 0x0086 },
+{ 0x87 , 0x0087 },
+{ 0x88 , 0x0088 },
+{ 0x89 , 0x0089 },
+{ 0x8a , 0x008A },
+{ 0x8b , 0x008B },
+{ 0x8c , 0x008C },
+{ 0x8d , 0x008D },
+{ 0x8e , 0x008E },
+{ 0x8f , 0x008F },
+{ 0x90 , 0x0090 },
+{ 0x91 , 0x0091 },
+{ 0x92 , 0x0092 },
+{ 0x93 , 0x0093 },
+{ 0x94 , 0x0094 },
+{ 0x95 , 0x0095 },
+{ 0x96 , 0x0096 },
+{ 0x97 , 0x0097 },
+{ 0x98 , 0x0098 },
+{ 0x99 , 0x0099 },
+{ 0x9a , 0x009A },
+{ 0x9b , 0x009B },
+{ 0x9c , 0x009C },
+{ 0x9d , 0x009D },
+{ 0x9e , 0x009E },
+{ 0x9f , 0x009F },
+{ 0xa0 , 0x00A0 },
+{ 0xa1 , 0x0104 },
+{ 0xa2 , 0x02D8 },
+{ 0xa3 , 0x0141 },
+{ 0xa4 , 0x00A4 },
+{ 0xa5 , 0x013D },
+{ 0xa6 , 0x015A },
+{ 0xa7 , 0x00A7 },
+{ 0xa8 , 0x00A8 },
+{ 0xa9 , 0x0160 },
+{ 0xaa , 0x015E },
+{ 0xab , 0x0164 },
+{ 0xac , 0x0179 },
+{ 0xad , 0x00AD },
+{ 0xae , 0x017D },
+{ 0xaf , 0x017B },
+{ 0xb0 , 0x00B0 },
+{ 0xb1 , 0x0105 },
+{ 0xb2 , 0x02DB },
+{ 0xb3 , 0x0142 },
+{ 0xb4 , 0x00B4 },
+{ 0xb5 , 0x013E },
+{ 0xb6 , 0x015B },
+{ 0xb7 , 0x02C7 },
+{ 0xb8 , 0x00B8 },
+{ 0xb9 , 0x0161 },
+{ 0xba , 0x015F },
+{ 0xbb , 0x0165 },
+{ 0xbc , 0x017A },
+{ 0xbd , 0x02DD },
+{ 0xbe , 0x017E },
+{ 0xbf , 0x017C },
+{ 0xc0 , 0x0154 },
+{ 0xc1 , 0x00C1 },
+{ 0xc2 , 0x00C2 },
+{ 0xc3 , 0x0102 },
+{ 0xc4 , 0x00C4 },
+{ 0xc5 , 0x0139 },
+{ 0xc6 , 0x0106 },
+{ 0xc7 , 0x00C7 },
+{ 0xc8 , 0x010C },
+{ 0xc9 , 0x00C9 },
+{ 0xca , 0x0118 },
+{ 0xcb , 0x00CB },
+{ 0xcc , 0x011A },
+{ 0xcd , 0x00CD },
+{ 0xce , 0x00CE },
+{ 0xcf , 0x010E },
+{ 0xd0 , 0x0110 },
+{ 0xd1 , 0x0143 },
+{ 0xd2 , 0x0147 },
+{ 0xd3 , 0x00D3 },
+{ 0xd4 , 0x00D4 },
+{ 0xd5 , 0x0150 },
+{ 0xd6 , 0x00D6 },
+{ 0xd7 , 0x00D7 },
+{ 0xd8 , 0x0158 },
+{ 0xd9 , 0x016E },
+{ 0xda , 0x00DA },
+{ 0xdb , 0x0170 },
+{ 0xdc , 0x00DC },
+{ 0xdd , 0x00DD },
+{ 0xde , 0x0162 },
+{ 0xdf , 0x00DF },
+{ 0xe0 , 0x0155 },
+{ 0xe1 , 0x00E1 },
+{ 0xe2 , 0x00E2 },
+{ 0xe3 , 0x0103 },
+{ 0xe4 , 0x00E4 },
+{ 0xe5 , 0x013A },
+{ 0xe6 , 0x0107 },
+{ 0xe7 , 0x00E7 },
+{ 0xe8 , 0x010D },
+{ 0xe9 , 0x00E9 },
+{ 0xea , 0x0119 },
+{ 0xeb , 0x00EB },
+{ 0xec , 0x011B },
+{ 0xed , 0x00ED },
+{ 0xee , 0x00EE },
+{ 0xef , 0x010F },
+{ 0xf0 , 0x0111 },
+{ 0xf1 , 0x0144 },
+{ 0xf2 , 0x0148 },
+{ 0xf3 , 0x00F3 },
+{ 0xf4 , 0x00F4 },
+{ 0xf5 , 0x0151 },
+{ 0xf6 , 0x00F6 },
+{ 0xf7 , 0x00F7 },
+{ 0xf8 , 0x0159 },
+{ 0xf9 , 0x016F },
+{ 0xfa , 0x00FA },
+{ 0xfb , 0x0171 },
+{ 0xfc , 0x00FC },
+{ 0xfd , 0x00FD },
+{ 0xfe , 0x0163 },
+{ 0xff , 0x02D9 },
diff --git a/extra/locale/charmaps/ISO-8859-3.pairs b/extra/locale/charmaps/ISO-8859-3.pairs
new file mode 100644
index 000000000..3a8814cde
--- /dev/null
+++ b/extra/locale/charmaps/ISO-8859-3.pairs
@@ -0,0 +1,249 @@
+{ 0x00 , 0x0000 },
+{ 0x01 , 0x0001 },
+{ 0x02 , 0x0002 },
+{ 0x03 , 0x0003 },
+{ 0x04 , 0x0004 },
+{ 0x05 , 0x0005 },
+{ 0x06 , 0x0006 },
+{ 0x07 , 0x0007 },
+{ 0x08 , 0x0008 },
+{ 0x09 , 0x0009 },
+{ 0x0a , 0x000A },
+{ 0x0b , 0x000B },
+{ 0x0c , 0x000C },
+{ 0x0d , 0x000D },
+{ 0x0e , 0x000E },
+{ 0x0f , 0x000F },
+{ 0x10 , 0x0010 },
+{ 0x11 , 0x0011 },
+{ 0x12 , 0x0012 },
+{ 0x13 , 0x0013 },
+{ 0x14 , 0x0014 },
+{ 0x15 , 0x0015 },
+{ 0x16 , 0x0016 },
+{ 0x17 , 0x0017 },
+{ 0x18 , 0x0018 },
+{ 0x19 , 0x0019 },
+{ 0x1a , 0x001A },
+{ 0x1b , 0x001B },
+{ 0x1c , 0x001C },
+{ 0x1d , 0x001D },
+{ 0x1e , 0x001E },
+{ 0x1f , 0x001F },
+{ 0x20 , 0x0020 },
+{ 0x21 , 0x0021 },
+{ 0x22 , 0x0022 },
+{ 0x23 , 0x0023 },
+{ 0x24 , 0x0024 },
+{ 0x25 , 0x0025 },
+{ 0x26 , 0x0026 },
+{ 0x27 , 0x0027 },
+{ 0x28 , 0x0028 },
+{ 0x29 , 0x0029 },
+{ 0x2a , 0x002A },
+{ 0x2b , 0x002B },
+{ 0x2c , 0x002C },
+{ 0x2d , 0x002D },
+{ 0x2e , 0x002E },
+{ 0x2f , 0x002F },
+{ 0x30 , 0x0030 },
+{ 0x31 , 0x0031 },
+{ 0x32 , 0x0032 },
+{ 0x33 , 0x0033 },
+{ 0x34 , 0x0034 },
+{ 0x35 , 0x0035 },
+{ 0x36 , 0x0036 },
+{ 0x37 , 0x0037 },
+{ 0x38 , 0x0038 },
+{ 0x39 , 0x0039 },
+{ 0x3a , 0x003A },
+{ 0x3b , 0x003B },
+{ 0x3c , 0x003C },
+{ 0x3d , 0x003D },
+{ 0x3e , 0x003E },
+{ 0x3f , 0x003F },
+{ 0x40 , 0x0040 },
+{ 0x41 , 0x0041 },
+{ 0x42 , 0x0042 },
+{ 0x43 , 0x0043 },
+{ 0x44 , 0x0044 },
+{ 0x45 , 0x0045 },
+{ 0x46 , 0x0046 },
+{ 0x47 , 0x0047 },
+{ 0x48 , 0x0048 },
+{ 0x49 , 0x0049 },
+{ 0x4a , 0x004A },
+{ 0x4b , 0x004B },
+{ 0x4c , 0x004C },
+{ 0x4d , 0x004D },
+{ 0x4e , 0x004E },
+{ 0x4f , 0x004F },
+{ 0x50 , 0x0050 },
+{ 0x51 , 0x0051 },
+{ 0x52 , 0x0052 },
+{ 0x53 , 0x0053 },
+{ 0x54 , 0x0054 },
+{ 0x55 , 0x0055 },
+{ 0x56 , 0x0056 },
+{ 0x57 , 0x0057 },
+{ 0x58 , 0x0058 },
+{ 0x59 , 0x0059 },
+{ 0x5a , 0x005A },
+{ 0x5b , 0x005B },
+{ 0x5c , 0x005C },
+{ 0x5d , 0x005D },
+{ 0x5e , 0x005E },
+{ 0x5f , 0x005F },
+{ 0x60 , 0x0060 },
+{ 0x61 , 0x0061 },
+{ 0x62 , 0x0062 },
+{ 0x63 , 0x0063 },
+{ 0x64 , 0x0064 },
+{ 0x65 , 0x0065 },
+{ 0x66 , 0x0066 },
+{ 0x67 , 0x0067 },
+{ 0x68 , 0x0068 },
+{ 0x69 , 0x0069 },
+{ 0x6a , 0x006A },
+{ 0x6b , 0x006B },
+{ 0x6c , 0x006C },
+{ 0x6d , 0x006D },
+{ 0x6e , 0x006E },
+{ 0x6f , 0x006F },
+{ 0x70 , 0x0070 },
+{ 0x71 , 0x0071 },
+{ 0x72 , 0x0072 },
+{ 0x73 , 0x0073 },
+{ 0x74 , 0x0074 },
+{ 0x75 , 0x0075 },
+{ 0x76 , 0x0076 },
+{ 0x77 , 0x0077 },
+{ 0x78 , 0x0078 },
+{ 0x79 , 0x0079 },
+{ 0x7a , 0x007A },
+{ 0x7b , 0x007B },
+{ 0x7c , 0x007C },
+{ 0x7d , 0x007D },
+{ 0x7e , 0x007E },
+{ 0x7f , 0x007F },
+{ 0x80 , 0x0080 },
+{ 0x81 , 0x0081 },
+{ 0x82 , 0x0082 },
+{ 0x83 , 0x0083 },
+{ 0x84 , 0x0084 },
+{ 0x85 , 0x0085 },
+{ 0x86 , 0x0086 },
+{ 0x87 , 0x0087 },
+{ 0x88 , 0x0088 },
+{ 0x89 , 0x0089 },
+{ 0x8a , 0x008A },
+{ 0x8b , 0x008B },
+{ 0x8c , 0x008C },
+{ 0x8d , 0x008D },
+{ 0x8e , 0x008E },
+{ 0x8f , 0x008F },
+{ 0x90 , 0x0090 },
+{ 0x91 , 0x0091 },
+{ 0x92 , 0x0092 },
+{ 0x93 , 0x0093 },
+{ 0x94 , 0x0094 },
+{ 0x95 , 0x0095 },
+{ 0x96 , 0x0096 },
+{ 0x97 , 0x0097 },
+{ 0x98 , 0x0098 },
+{ 0x99 , 0x0099 },
+{ 0x9a , 0x009A },
+{ 0x9b , 0x009B },
+{ 0x9c , 0x009C },
+{ 0x9d , 0x009D },
+{ 0x9e , 0x009E },
+{ 0x9f , 0x009F },
+{ 0xa0 , 0x00A0 },
+{ 0xa1 , 0x0126 },
+{ 0xa2 , 0x02D8 },
+{ 0xa3 , 0x00A3 },
+{ 0xa4 , 0x00A4 },
+{ 0xa6 , 0x0124 },
+{ 0xa7 , 0x00A7 },
+{ 0xa8 , 0x00A8 },
+{ 0xa9 , 0x0130 },
+{ 0xaa , 0x015E },
+{ 0xab , 0x011E },
+{ 0xac , 0x0134 },
+{ 0xad , 0x00AD },
+{ 0xaf , 0x017B },
+{ 0xb0 , 0x00B0 },
+{ 0xb1 , 0x0127 },
+{ 0xb2 , 0x00B2 },
+{ 0xb3 , 0x00B3 },
+{ 0xb4 , 0x00B4 },
+{ 0xb5 , 0x00B5 },
+{ 0xb6 , 0x0125 },
+{ 0xb7 , 0x00B7 },
+{ 0xb8 , 0x00B8 },
+{ 0xb9 , 0x0131 },
+{ 0xba , 0x015F },
+{ 0xbb , 0x011F },
+{ 0xbc , 0x0135 },
+{ 0xbd , 0x00BD },
+{ 0xbf , 0x017C },
+{ 0xc0 , 0x00C0 },
+{ 0xc1 , 0x00C1 },
+{ 0xc2 , 0x00C2 },
+{ 0xc4 , 0x00C4 },
+{ 0xc5 , 0x010A },
+{ 0xc6 , 0x0108 },
+{ 0xc7 , 0x00C7 },
+{ 0xc8 , 0x00C8 },
+{ 0xc9 , 0x00C9 },
+{ 0xca , 0x00CA },
+{ 0xcb , 0x00CB },
+{ 0xcc , 0x00CC },
+{ 0xcd , 0x00CD },
+{ 0xce , 0x00CE },
+{ 0xcf , 0x00CF },
+{ 0xd1 , 0x00D1 },
+{ 0xd2 , 0x00D2 },
+{ 0xd3 , 0x00D3 },
+{ 0xd4 , 0x00D4 },
+{ 0xd5 , 0x0120 },
+{ 0xd6 , 0x00D6 },
+{ 0xd7 , 0x00D7 },
+{ 0xd8 , 0x011C },
+{ 0xd9 , 0x00D9 },
+{ 0xda , 0x00DA },
+{ 0xdb , 0x00DB },
+{ 0xdc , 0x00DC },
+{ 0xdd , 0x016C },
+{ 0xde , 0x015C },
+{ 0xdf , 0x00DF },
+{ 0xe0 , 0x00E0 },
+{ 0xe1 , 0x00E1 },
+{ 0xe2 , 0x00E2 },
+{ 0xe4 , 0x00E4 },
+{ 0xe5 , 0x010B },
+{ 0xe6 , 0x0109 },
+{ 0xe7 , 0x00E7 },
+{ 0xe8 , 0x00E8 },
+{ 0xe9 , 0x00E9 },
+{ 0xea , 0x00EA },
+{ 0xeb , 0x00EB },
+{ 0xec , 0x00EC },
+{ 0xed , 0x00ED },
+{ 0xee , 0x00EE },
+{ 0xef , 0x00EF },
+{ 0xf1 , 0x00F1 },
+{ 0xf2 , 0x00F2 },
+{ 0xf3 , 0x00F3 },
+{ 0xf4 , 0x00F4 },
+{ 0xf5 , 0x0121 },
+{ 0xf6 , 0x00F6 },
+{ 0xf7 , 0x00F7 },
+{ 0xf8 , 0x011D },
+{ 0xf9 , 0x00F9 },
+{ 0xfa , 0x00FA },
+{ 0xfb , 0x00FB },
+{ 0xfc , 0x00FC },
+{ 0xfd , 0x016D },
+{ 0xfe , 0x015D },
+{ 0xff , 0x02D9 },
diff --git a/extra/locale/charmaps/ISO-8859-4.pairs b/extra/locale/charmaps/ISO-8859-4.pairs
new file mode 100644
index 000000000..dda865c3c
--- /dev/null
+++ b/extra/locale/charmaps/ISO-8859-4.pairs
@@ -0,0 +1,256 @@
+{ 0x00 , 0x0000 },
+{ 0x01 , 0x0001 },
+{ 0x02 , 0x0002 },
+{ 0x03 , 0x0003 },
+{ 0x04 , 0x0004 },
+{ 0x05 , 0x0005 },
+{ 0x06 , 0x0006 },
+{ 0x07 , 0x0007 },
+{ 0x08 , 0x0008 },
+{ 0x09 , 0x0009 },
+{ 0x0a , 0x000A },
+{ 0x0b , 0x000B },
+{ 0x0c , 0x000C },
+{ 0x0d , 0x000D },
+{ 0x0e , 0x000E },
+{ 0x0f , 0x000F },
+{ 0x10 , 0x0010 },
+{ 0x11 , 0x0011 },
+{ 0x12 , 0x0012 },
+{ 0x13 , 0x0013 },
+{ 0x14 , 0x0014 },
+{ 0x15 , 0x0015 },
+{ 0x16 , 0x0016 },
+{ 0x17 , 0x0017 },
+{ 0x18 , 0x0018 },
+{ 0x19 , 0x0019 },
+{ 0x1a , 0x001A },
+{ 0x1b , 0x001B },
+{ 0x1c , 0x001C },
+{ 0x1d , 0x001D },
+{ 0x1e , 0x001E },
+{ 0x1f , 0x001F },
+{ 0x20 , 0x0020 },
+{ 0x21 , 0x0021 },
+{ 0x22 , 0x0022 },
+{ 0x23 , 0x0023 },
+{ 0x24 , 0x0024 },
+{ 0x25 , 0x0025 },
+{ 0x26 , 0x0026 },
+{ 0x27 , 0x0027 },
+{ 0x28 , 0x0028 },
+{ 0x29 , 0x0029 },
+{ 0x2a , 0x002A },
+{ 0x2b , 0x002B },
+{ 0x2c , 0x002C },
+{ 0x2d , 0x002D },
+{ 0x2e , 0x002E },
+{ 0x2f , 0x002F },
+{ 0x30 , 0x0030 },
+{ 0x31 , 0x0031 },
+{ 0x32 , 0x0032 },
+{ 0x33 , 0x0033 },
+{ 0x34 , 0x0034 },
+{ 0x35 , 0x0035 },
+{ 0x36 , 0x0036 },
+{ 0x37 , 0x0037 },
+{ 0x38 , 0x0038 },
+{ 0x39 , 0x0039 },
+{ 0x3a , 0x003A },
+{ 0x3b , 0x003B },
+{ 0x3c , 0x003C },
+{ 0x3d , 0x003D },
+{ 0x3e , 0x003E },
+{ 0x3f , 0x003F },
+{ 0x40 , 0x0040 },
+{ 0x41 , 0x0041 },
+{ 0x42 , 0x0042 },
+{ 0x43 , 0x0043 },
+{ 0x44 , 0x0044 },
+{ 0x45 , 0x0045 },
+{ 0x46 , 0x0046 },
+{ 0x47 , 0x0047 },
+{ 0x48 , 0x0048 },
+{ 0x49 , 0x0049 },
+{ 0x4a , 0x004A },
+{ 0x4b , 0x004B },
+{ 0x4c , 0x004C },
+{ 0x4d , 0x004D },
+{ 0x4e , 0x004E },
+{ 0x4f , 0x004F },
+{ 0x50 , 0x0050 },
+{ 0x51 , 0x0051 },
+{ 0x52 , 0x0052 },
+{ 0x53 , 0x0053 },
+{ 0x54 , 0x0054 },
+{ 0x55 , 0x0055 },
+{ 0x56 , 0x0056 },
+{ 0x57 , 0x0057 },
+{ 0x58 , 0x0058 },
+{ 0x59 , 0x0059 },
+{ 0x5a , 0x005A },
+{ 0x5b , 0x005B },
+{ 0x5c , 0x005C },
+{ 0x5d , 0x005D },
+{ 0x5e , 0x005E },
+{ 0x5f , 0x005F },
+{ 0x60 , 0x0060 },
+{ 0x61 , 0x0061 },
+{ 0x62 , 0x0062 },
+{ 0x63 , 0x0063 },
+{ 0x64 , 0x0064 },
+{ 0x65 , 0x0065 },
+{ 0x66 , 0x0066 },
+{ 0x67 , 0x0067 },
+{ 0x68 , 0x0068 },
+{ 0x69 , 0x0069 },
+{ 0x6a , 0x006A },
+{ 0x6b , 0x006B },
+{ 0x6c , 0x006C },
+{ 0x6d , 0x006D },
+{ 0x6e , 0x006E },
+{ 0x6f , 0x006F },
+{ 0x70 , 0x0070 },
+{ 0x71 , 0x0071 },
+{ 0x72 , 0x0072 },
+{ 0x73 , 0x0073 },
+{ 0x74 , 0x0074 },
+{ 0x75 , 0x0075 },
+{ 0x76 , 0x0076 },
+{ 0x77 , 0x0077 },
+{ 0x78 , 0x0078 },
+{ 0x79 , 0x0079 },
+{ 0x7a , 0x007A },
+{ 0x7b , 0x007B },
+{ 0x7c , 0x007C },
+{ 0x7d , 0x007D },
+{ 0x7e , 0x007E },
+{ 0x7f , 0x007F },
+{ 0x80 , 0x0080 },
+{ 0x81 , 0x0081 },
+{ 0x82 , 0x0082 },
+{ 0x83 , 0x0083 },
+{ 0x84 , 0x0084 },
+{ 0x85 , 0x0085 },
+{ 0x86 , 0x0086 },
+{ 0x87 , 0x0087 },
+{ 0x88 , 0x0088 },
+{ 0x89 , 0x0089 },
+{ 0x8a , 0x008A },
+{ 0x8b , 0x008B },
+{ 0x8c , 0x008C },
+{ 0x8d , 0x008D },
+{ 0x8e , 0x008E },
+{ 0x8f , 0x008F },
+{ 0x90 , 0x0090 },
+{ 0x91 , 0x0091 },
+{ 0x92 , 0x0092 },
+{ 0x93 , 0x0093 },
+{ 0x94 , 0x0094 },
+{ 0x95 , 0x0095 },
+{ 0x96 , 0x0096 },
+{ 0x97 , 0x0097 },
+{ 0x98 , 0x0098 },
+{ 0x99 , 0x0099 },
+{ 0x9a , 0x009A },
+{ 0x9b , 0x009B },
+{ 0x9c , 0x009C },
+{ 0x9d , 0x009D },
+{ 0x9e , 0x009E },
+{ 0x9f , 0x009F },
+{ 0xa0 , 0x00A0 },
+{ 0xa1 , 0x0104 },
+{ 0xa2 , 0x0138 },
+{ 0xa3 , 0x0156 },
+{ 0xa4 , 0x00A4 },
+{ 0xa5 , 0x0128 },
+{ 0xa6 , 0x013B },
+{ 0xa7 , 0x00A7 },
+{ 0xa8 , 0x00A8 },
+{ 0xa9 , 0x0160 },
+{ 0xaa , 0x0112 },
+{ 0xab , 0x0122 },
+{ 0xac , 0x0166 },
+{ 0xad , 0x00AD },
+{ 0xae , 0x017D },
+{ 0xaf , 0x00AF },
+{ 0xb0 , 0x00B0 },
+{ 0xb1 , 0x0105 },
+{ 0xb2 , 0x02DB },
+{ 0xb3 , 0x0157 },
+{ 0xb4 , 0x00B4 },
+{ 0xb5 , 0x0129 },
+{ 0xb6 , 0x013C },
+{ 0xb7 , 0x02C7 },
+{ 0xb8 , 0x00B8 },
+{ 0xb9 , 0x0161 },
+{ 0xba , 0x0113 },
+{ 0xbb , 0x0123 },
+{ 0xbc , 0x0167 },
+{ 0xbd , 0x014A },
+{ 0xbe , 0x017E },
+{ 0xbf , 0x014B },
+{ 0xc0 , 0x0100 },
+{ 0xc1 , 0x00C1 },
+{ 0xc2 , 0x00C2 },
+{ 0xc3 , 0x00C3 },
+{ 0xc4 , 0x00C4 },
+{ 0xc5 , 0x00C5 },
+{ 0xc6 , 0x00C6 },
+{ 0xc7 , 0x012E },
+{ 0xc8 , 0x010C },
+{ 0xc9 , 0x00C9 },
+{ 0xca , 0x0118 },
+{ 0xcb , 0x00CB },
+{ 0xcc , 0x0116 },
+{ 0xcd , 0x00CD },
+{ 0xce , 0x00CE },
+{ 0xcf , 0x012A },
+{ 0xd0 , 0x0110 },
+{ 0xd1 , 0x0145 },
+{ 0xd2 , 0x014C },
+{ 0xd3 , 0x0136 },
+{ 0xd4 , 0x00D4 },
+{ 0xd5 , 0x00D5 },
+{ 0xd6 , 0x00D6 },
+{ 0xd7 , 0x00D7 },
+{ 0xd8 , 0x00D8 },
+{ 0xd9 , 0x0172 },
+{ 0xda , 0x00DA },
+{ 0xdb , 0x00DB },
+{ 0xdc , 0x00DC },
+{ 0xdd , 0x0168 },
+{ 0xde , 0x016A },
+{ 0xdf , 0x00DF },
+{ 0xe0 , 0x0101 },
+{ 0xe1 , 0x00E1 },
+{ 0xe2 , 0x00E2 },
+{ 0xe3 , 0x00E3 },
+{ 0xe4 , 0x00E4 },
+{ 0xe5 , 0x00E5 },
+{ 0xe6 , 0x00E6 },
+{ 0xe7 , 0x012F },
+{ 0xe8 , 0x010D },
+{ 0xe9 , 0x00E9 },
+{ 0xea , 0x0119 },
+{ 0xeb , 0x00EB },
+{ 0xec , 0x0117 },
+{ 0xed , 0x00ED },
+{ 0xee , 0x00EE },
+{ 0xef , 0x012B },
+{ 0xf0 , 0x0111 },
+{ 0xf1 , 0x0146 },
+{ 0xf2 , 0x014D },
+{ 0xf3 , 0x0137 },
+{ 0xf4 , 0x00F4 },
+{ 0xf5 , 0x00F5 },
+{ 0xf6 , 0x00F6 },
+{ 0xf7 , 0x00F7 },
+{ 0xf8 , 0x00F8 },
+{ 0xf9 , 0x0173 },
+{ 0xfa , 0x00FA },
+{ 0xfb , 0x00FB },
+{ 0xfc , 0x00FC },
+{ 0xfd , 0x0169 },
+{ 0xfe , 0x016B },
+{ 0xff , 0x02D9 },
diff --git a/extra/locale/charmaps/ISO-8859-5.pairs b/extra/locale/charmaps/ISO-8859-5.pairs
new file mode 100644
index 000000000..08ebcae49
--- /dev/null
+++ b/extra/locale/charmaps/ISO-8859-5.pairs
@@ -0,0 +1,256 @@
+{ 0x00 , 0x0000 },
+{ 0x01 , 0x0001 },
+{ 0x02 , 0x0002 },
+{ 0x03 , 0x0003 },
+{ 0x04 , 0x0004 },
+{ 0x05 , 0x0005 },
+{ 0x06 , 0x0006 },
+{ 0x07 , 0x0007 },
+{ 0x08 , 0x0008 },
+{ 0x09 , 0x0009 },
+{ 0x0a , 0x000A },
+{ 0x0b , 0x000B },
+{ 0x0c , 0x000C },
+{ 0x0d , 0x000D },
+{ 0x0e , 0x000E },
+{ 0x0f , 0x000F },
+{ 0x10 , 0x0010 },
+{ 0x11 , 0x0011 },
+{ 0x12 , 0x0012 },
+{ 0x13 , 0x0013 },
+{ 0x14 , 0x0014 },
+{ 0x15 , 0x0015 },
+{ 0x16 , 0x0016 },
+{ 0x17 , 0x0017 },
+{ 0x18 , 0x0018 },
+{ 0x19 , 0x0019 },
+{ 0x1a , 0x001A },
+{ 0x1b , 0x001B },
+{ 0x1c , 0x001C },
+{ 0x1d , 0x001D },
+{ 0x1e , 0x001E },
+{ 0x1f , 0x001F },
+{ 0x20 , 0x0020 },
+{ 0x21 , 0x0021 },
+{ 0x22 , 0x0022 },
+{ 0x23 , 0x0023 },
+{ 0x24 , 0x0024 },
+{ 0x25 , 0x0025 },
+{ 0x26 , 0x0026 },
+{ 0x27 , 0x0027 },
+{ 0x28 , 0x0028 },
+{ 0x29 , 0x0029 },
+{ 0x2a , 0x002A },
+{ 0x2b , 0x002B },
+{ 0x2c , 0x002C },
+{ 0x2d , 0x002D },
+{ 0x2e , 0x002E },
+{ 0x2f , 0x002F },
+{ 0x30 , 0x0030 },
+{ 0x31 , 0x0031 },
+{ 0x32 , 0x0032 },
+{ 0x33 , 0x0033 },
+{ 0x34 , 0x0034 },
+{ 0x35 , 0x0035 },
+{ 0x36 , 0x0036 },
+{ 0x37 , 0x0037 },
+{ 0x38 , 0x0038 },
+{ 0x39 , 0x0039 },
+{ 0x3a , 0x003A },
+{ 0x3b , 0x003B },
+{ 0x3c , 0x003C },
+{ 0x3d , 0x003D },
+{ 0x3e , 0x003E },
+{ 0x3f , 0x003F },
+{ 0x40 , 0x0040 },
+{ 0x41 , 0x0041 },
+{ 0x42 , 0x0042 },
+{ 0x43 , 0x0043 },
+{ 0x44 , 0x0044 },
+{ 0x45 , 0x0045 },
+{ 0x46 , 0x0046 },
+{ 0x47 , 0x0047 },
+{ 0x48 , 0x0048 },
+{ 0x49 , 0x0049 },
+{ 0x4a , 0x004A },
+{ 0x4b , 0x004B },
+{ 0x4c , 0x004C },
+{ 0x4d , 0x004D },
+{ 0x4e , 0x004E },
+{ 0x4f , 0x004F },
+{ 0x50 , 0x0050 },
+{ 0x51 , 0x0051 },
+{ 0x52 , 0x0052 },
+{ 0x53 , 0x0053 },
+{ 0x54 , 0x0054 },
+{ 0x55 , 0x0055 },
+{ 0x56 , 0x0056 },
+{ 0x57 , 0x0057 },
+{ 0x58 , 0x0058 },
+{ 0x59 , 0x0059 },
+{ 0x5a , 0x005A },
+{ 0x5b , 0x005B },
+{ 0x5c , 0x005C },
+{ 0x5d , 0x005D },
+{ 0x5e , 0x005E },
+{ 0x5f , 0x005F },
+{ 0x60 , 0x0060 },
+{ 0x61 , 0x0061 },
+{ 0x62 , 0x0062 },
+{ 0x63 , 0x0063 },
+{ 0x64 , 0x0064 },
+{ 0x65 , 0x0065 },
+{ 0x66 , 0x0066 },
+{ 0x67 , 0x0067 },
+{ 0x68 , 0x0068 },
+{ 0x69 , 0x0069 },
+{ 0x6a , 0x006A },
+{ 0x6b , 0x006B },
+{ 0x6c , 0x006C },
+{ 0x6d , 0x006D },
+{ 0x6e , 0x006E },
+{ 0x6f , 0x006F },
+{ 0x70 , 0x0070 },
+{ 0x71 , 0x0071 },
+{ 0x72 , 0x0072 },
+{ 0x73 , 0x0073 },
+{ 0x74 , 0x0074 },
+{ 0x75 , 0x0075 },
+{ 0x76 , 0x0076 },
+{ 0x77 , 0x0077 },
+{ 0x78 , 0x0078 },
+{ 0x79 , 0x0079 },
+{ 0x7a , 0x007A },
+{ 0x7b , 0x007B },
+{ 0x7c , 0x007C },
+{ 0x7d , 0x007D },
+{ 0x7e , 0x007E },
+{ 0x7f , 0x007F },
+{ 0x80 , 0x0080 },
+{ 0x81 , 0x0081 },
+{ 0x82 , 0x0082 },
+{ 0x83 , 0x0083 },
+{ 0x84 , 0x0084 },
+{ 0x85 , 0x0085 },
+{ 0x86 , 0x0086 },
+{ 0x87 , 0x0087 },
+{ 0x88 , 0x0088 },
+{ 0x89 , 0x0089 },
+{ 0x8a , 0x008A },
+{ 0x8b , 0x008B },
+{ 0x8c , 0x008C },
+{ 0x8d , 0x008D },
+{ 0x8e , 0x008E },
+{ 0x8f , 0x008F },
+{ 0x90 , 0x0090 },
+{ 0x91 , 0x0091 },
+{ 0x92 , 0x0092 },
+{ 0x93 , 0x0093 },
+{ 0x94 , 0x0094 },
+{ 0x95 , 0x0095 },
+{ 0x96 , 0x0096 },
+{ 0x97 , 0x0097 },
+{ 0x98 , 0x0098 },
+{ 0x99 , 0x0099 },
+{ 0x9a , 0x009A },
+{ 0x9b , 0x009B },
+{ 0x9c , 0x009C },
+{ 0x9d , 0x009D },
+{ 0x9e , 0x009E },
+{ 0x9f , 0x009F },
+{ 0xa0 , 0x00A0 },
+{ 0xa1 , 0x0401 },
+{ 0xa2 , 0x0402 },
+{ 0xa3 , 0x0403 },
+{ 0xa4 , 0x0404 },
+{ 0xa5 , 0x0405 },
+{ 0xa6 , 0x0406 },
+{ 0xa7 , 0x0407 },
+{ 0xa8 , 0x0408 },
+{ 0xa9 , 0x0409 },
+{ 0xaa , 0x040A },
+{ 0xab , 0x040B },
+{ 0xac , 0x040C },
+{ 0xad , 0x00AD },
+{ 0xae , 0x040E },
+{ 0xaf , 0x040F },
+{ 0xb0 , 0x0410 },
+{ 0xb1 , 0x0411 },
+{ 0xb2 , 0x0412 },
+{ 0xb3 , 0x0413 },
+{ 0xb4 , 0x0414 },
+{ 0xb5 , 0x0415 },
+{ 0xb6 , 0x0416 },
+{ 0xb7 , 0x0417 },
+{ 0xb8 , 0x0418 },
+{ 0xb9 , 0x0419 },
+{ 0xba , 0x041A },
+{ 0xbb , 0x041B },
+{ 0xbc , 0x041C },
+{ 0xbd , 0x041D },
+{ 0xbe , 0x041E },
+{ 0xbf , 0x041F },
+{ 0xc0 , 0x0420 },
+{ 0xc1 , 0x0421 },
+{ 0xc2 , 0x0422 },
+{ 0xc3 , 0x0423 },
+{ 0xc4 , 0x0424 },
+{ 0xc5 , 0x0425 },
+{ 0xc6 , 0x0426 },
+{ 0xc7 , 0x0427 },
+{ 0xc8 , 0x0428 },
+{ 0xc9 , 0x0429 },
+{ 0xca , 0x042A },
+{ 0xcb , 0x042B },
+{ 0xcc , 0x042C },
+{ 0xcd , 0x042D },
+{ 0xce , 0x042E },
+{ 0xcf , 0x042F },
+{ 0xd0 , 0x0430 },
+{ 0xd1 , 0x0431 },
+{ 0xd2 , 0x0432 },
+{ 0xd3 , 0x0433 },
+{ 0xd4 , 0x0434 },
+{ 0xd5 , 0x0435 },
+{ 0xd6 , 0x0436 },
+{ 0xd7 , 0x0437 },
+{ 0xd8 , 0x0438 },
+{ 0xd9 , 0x0439 },
+{ 0xda , 0x043A },
+{ 0xdb , 0x043B },
+{ 0xdc , 0x043C },
+{ 0xdd , 0x043D },
+{ 0xde , 0x043E },
+{ 0xdf , 0x043F },
+{ 0xe0 , 0x0440 },
+{ 0xe1 , 0x0441 },
+{ 0xe2 , 0x0442 },
+{ 0xe3 , 0x0443 },
+{ 0xe4 , 0x0444 },
+{ 0xe5 , 0x0445 },
+{ 0xe6 , 0x0446 },
+{ 0xe7 , 0x0447 },
+{ 0xe8 , 0x0448 },
+{ 0xe9 , 0x0449 },
+{ 0xea , 0x044A },
+{ 0xeb , 0x044B },
+{ 0xec , 0x044C },
+{ 0xed , 0x044D },
+{ 0xee , 0x044E },
+{ 0xef , 0x044F },
+{ 0xf0 , 0x2116 },
+{ 0xf1 , 0x0451 },
+{ 0xf2 , 0x0452 },
+{ 0xf3 , 0x0453 },
+{ 0xf4 , 0x0454 },
+{ 0xf5 , 0x0455 },
+{ 0xf6 , 0x0456 },
+{ 0xf7 , 0x0457 },
+{ 0xf8 , 0x0458 },
+{ 0xf9 , 0x0459 },
+{ 0xfa , 0x045A },
+{ 0xfb , 0x045B },
+{ 0xfc , 0x045C },
+{ 0xfd , 0x00A7 },
+{ 0xfe , 0x045E },
+{ 0xff , 0x045F },
diff --git a/extra/locale/charmaps/ISO-8859-6.pairs b/extra/locale/charmaps/ISO-8859-6.pairs
new file mode 100644
index 000000000..ccba9f907
--- /dev/null
+++ b/extra/locale/charmaps/ISO-8859-6.pairs
@@ -0,0 +1,211 @@
+{ 0x00 , 0x0000 },
+{ 0x01 , 0x0001 },
+{ 0x02 , 0x0002 },
+{ 0x03 , 0x0003 },
+{ 0x04 , 0x0004 },
+{ 0x05 , 0x0005 },
+{ 0x06 , 0x0006 },
+{ 0x07 , 0x0007 },
+{ 0x08 , 0x0008 },
+{ 0x09 , 0x0009 },
+{ 0x0a , 0x000A },
+{ 0x0b , 0x000B },
+{ 0x0c , 0x000C },
+{ 0x0d , 0x000D },
+{ 0x0e , 0x000E },
+{ 0x0f , 0x000F },
+{ 0x10 , 0x0010 },
+{ 0x11 , 0x0011 },
+{ 0x12 , 0x0012 },
+{ 0x13 , 0x0013 },
+{ 0x14 , 0x0014 },
+{ 0x15 , 0x0015 },
+{ 0x16 , 0x0016 },
+{ 0x17 , 0x0017 },
+{ 0x18 , 0x0018 },
+{ 0x19 , 0x0019 },
+{ 0x1a , 0x001A },
+{ 0x1b , 0x001B },
+{ 0x1c , 0x001C },
+{ 0x1d , 0x001D },
+{ 0x1e , 0x001E },
+{ 0x1f , 0x001F },
+{ 0x20 , 0x0020 },
+{ 0x21 , 0x0021 },
+{ 0x22 , 0x0022 },
+{ 0x23 , 0x0023 },
+{ 0x24 , 0x0024 },
+{ 0x25 , 0x0025 },
+{ 0x26 , 0x0026 },
+{ 0x27 , 0x0027 },
+{ 0x28 , 0x0028 },
+{ 0x29 , 0x0029 },
+{ 0x2a , 0x002A },
+{ 0x2b , 0x002B },
+{ 0x2c , 0x002C },
+{ 0x2d , 0x002D },
+{ 0x2e , 0x002E },
+{ 0x2f , 0x002F },
+{ 0x30 , 0x0030 },
+{ 0x31 , 0x0031 },
+{ 0x32 , 0x0032 },
+{ 0x33 , 0x0033 },
+{ 0x34 , 0x0034 },
+{ 0x35 , 0x0035 },
+{ 0x36 , 0x0036 },
+{ 0x37 , 0x0037 },
+{ 0x38 , 0x0038 },
+{ 0x39 , 0x0039 },
+{ 0x3a , 0x003A },
+{ 0x3b , 0x003B },
+{ 0x3c , 0x003C },
+{ 0x3d , 0x003D },
+{ 0x3e , 0x003E },
+{ 0x3f , 0x003F },
+{ 0x40 , 0x0040 },
+{ 0x41 , 0x0041 },
+{ 0x42 , 0x0042 },
+{ 0x43 , 0x0043 },
+{ 0x44 , 0x0044 },
+{ 0x45 , 0x0045 },
+{ 0x46 , 0x0046 },
+{ 0x47 , 0x0047 },
+{ 0x48 , 0x0048 },
+{ 0x49 , 0x0049 },
+{ 0x4a , 0x004A },
+{ 0x4b , 0x004B },
+{ 0x4c , 0x004C },
+{ 0x4d , 0x004D },
+{ 0x4e , 0x004E },
+{ 0x4f , 0x004F },
+{ 0x50 , 0x0050 },
+{ 0x51 , 0x0051 },
+{ 0x52 , 0x0052 },
+{ 0x53 , 0x0053 },
+{ 0x54 , 0x0054 },
+{ 0x55 , 0x0055 },
+{ 0x56 , 0x0056 },
+{ 0x57 , 0x0057 },
+{ 0x58 , 0x0058 },
+{ 0x59 , 0x0059 },
+{ 0x5a , 0x005A },
+{ 0x5b , 0x005B },
+{ 0x5c , 0x005C },
+{ 0x5d , 0x005D },
+{ 0x5e , 0x005E },
+{ 0x5f , 0x005F },
+{ 0x60 , 0x0060 },
+{ 0x61 , 0x0061 },
+{ 0x62 , 0x0062 },
+{ 0x63 , 0x0063 },
+{ 0x64 , 0x0064 },
+{ 0x65 , 0x0065 },
+{ 0x66 , 0x0066 },
+{ 0x67 , 0x0067 },
+{ 0x68 , 0x0068 },
+{ 0x69 , 0x0069 },
+{ 0x6a , 0x006A },
+{ 0x6b , 0x006B },
+{ 0x6c , 0x006C },
+{ 0x6d , 0x006D },
+{ 0x6e , 0x006E },
+{ 0x6f , 0x006F },
+{ 0x70 , 0x0070 },
+{ 0x71 , 0x0071 },
+{ 0x72 , 0x0072 },
+{ 0x73 , 0x0073 },
+{ 0x74 , 0x0074 },
+{ 0x75 , 0x0075 },
+{ 0x76 , 0x0076 },
+{ 0x77 , 0x0077 },
+{ 0x78 , 0x0078 },
+{ 0x79 , 0x0079 },
+{ 0x7a , 0x007A },
+{ 0x7b , 0x007B },
+{ 0x7c , 0x007C },
+{ 0x7d , 0x007D },
+{ 0x7e , 0x007E },
+{ 0x7f , 0x007F },
+{ 0x80 , 0x0080 },
+{ 0x81 , 0x0081 },
+{ 0x82 , 0x0082 },
+{ 0x83 , 0x0083 },
+{ 0x84 , 0x0084 },
+{ 0x85 , 0x0085 },
+{ 0x86 , 0x0086 },
+{ 0x87 , 0x0087 },
+{ 0x88 , 0x0088 },
+{ 0x89 , 0x0089 },
+{ 0x8a , 0x008A },
+{ 0x8b , 0x008B },
+{ 0x8c , 0x008C },
+{ 0x8d , 0x008D },
+{ 0x8e , 0x008E },
+{ 0x8f , 0x008F },
+{ 0x90 , 0x0090 },
+{ 0x91 , 0x0091 },
+{ 0x92 , 0x0092 },
+{ 0x93 , 0x0093 },
+{ 0x94 , 0x0094 },
+{ 0x95 , 0x0095 },
+{ 0x96 , 0x0096 },
+{ 0x97 , 0x0097 },
+{ 0x98 , 0x0098 },
+{ 0x99 , 0x0099 },
+{ 0x9a , 0x009A },
+{ 0x9b , 0x009B },
+{ 0x9c , 0x009C },
+{ 0x9d , 0x009D },
+{ 0x9e , 0x009E },
+{ 0x9f , 0x009F },
+{ 0xa0 , 0x00A0 },
+{ 0xa4 , 0x00A4 },
+{ 0xac , 0x060C },
+{ 0xad , 0x00AD },
+{ 0xbb , 0x061B },
+{ 0xbf , 0x061F },
+{ 0xc1 , 0x0621 },
+{ 0xc2 , 0x0622 },
+{ 0xc3 , 0x0623 },
+{ 0xc4 , 0x0624 },
+{ 0xc5 , 0x0625 },
+{ 0xc6 , 0x0626 },
+{ 0xc7 , 0x0627 },
+{ 0xc8 , 0x0628 },
+{ 0xc9 , 0x0629 },
+{ 0xca , 0x062A },
+{ 0xcb , 0x062B },
+{ 0xcc , 0x062C },
+{ 0xcd , 0x062D },
+{ 0xce , 0x062E },
+{ 0xcf , 0x062F },
+{ 0xd0 , 0x0630 },
+{ 0xd1 , 0x0631 },
+{ 0xd2 , 0x0632 },
+{ 0xd3 , 0x0633 },
+{ 0xd4 , 0x0634 },
+{ 0xd5 , 0x0635 },
+{ 0xd6 , 0x0636 },
+{ 0xd7 , 0x0637 },
+{ 0xd8 , 0x0638 },
+{ 0xd9 , 0x0639 },
+{ 0xda , 0x063A },
+{ 0xe0 , 0x0640 },
+{ 0xe1 , 0x0641 },
+{ 0xe2 , 0x0642 },
+{ 0xe3 , 0x0643 },
+{ 0xe4 , 0x0644 },
+{ 0xe5 , 0x0645 },
+{ 0xe6 , 0x0646 },
+{ 0xe7 , 0x0647 },
+{ 0xe8 , 0x0648 },
+{ 0xe9 , 0x0649 },
+{ 0xea , 0x064A },
+{ 0xeb , 0x064B },
+{ 0xec , 0x064C },
+{ 0xed , 0x064D },
+{ 0xee , 0x064E },
+{ 0xef , 0x064F },
+{ 0xf0 , 0x0650 },
+{ 0xf1 , 0x0651 },
+{ 0xf2 , 0x0652 },
diff --git a/extra/locale/charmaps/ISO-8859-7.pairs b/extra/locale/charmaps/ISO-8859-7.pairs
new file mode 100644
index 000000000..36b8ba24c
--- /dev/null
+++ b/extra/locale/charmaps/ISO-8859-7.pairs
@@ -0,0 +1,250 @@
+{ 0x00 , 0x0000 },
+{ 0x01 , 0x0001 },
+{ 0x02 , 0x0002 },
+{ 0x03 , 0x0003 },
+{ 0x04 , 0x0004 },
+{ 0x05 , 0x0005 },
+{ 0x06 , 0x0006 },
+{ 0x07 , 0x0007 },
+{ 0x08 , 0x0008 },
+{ 0x09 , 0x0009 },
+{ 0x0a , 0x000A },
+{ 0x0b , 0x000B },
+{ 0x0c , 0x000C },
+{ 0x0d , 0x000D },
+{ 0x0e , 0x000E },
+{ 0x0f , 0x000F },
+{ 0x10 , 0x0010 },
+{ 0x11 , 0x0011 },
+{ 0x12 , 0x0012 },
+{ 0x13 , 0x0013 },
+{ 0x14 , 0x0014 },
+{ 0x15 , 0x0015 },
+{ 0x16 , 0x0016 },
+{ 0x17 , 0x0017 },
+{ 0x18 , 0x0018 },
+{ 0x19 , 0x0019 },
+{ 0x1a , 0x001A },
+{ 0x1b , 0x001B },
+{ 0x1c , 0x001C },
+{ 0x1d , 0x001D },
+{ 0x1e , 0x001E },
+{ 0x1f , 0x001F },
+{ 0x20 , 0x0020 },
+{ 0x21 , 0x0021 },
+{ 0x22 , 0x0022 },
+{ 0x23 , 0x0023 },
+{ 0x24 , 0x0024 },
+{ 0x25 , 0x0025 },
+{ 0x26 , 0x0026 },
+{ 0x27 , 0x0027 },
+{ 0x28 , 0x0028 },
+{ 0x29 , 0x0029 },
+{ 0x2a , 0x002A },
+{ 0x2b , 0x002B },
+{ 0x2c , 0x002C },
+{ 0x2d , 0x002D },
+{ 0x2e , 0x002E },
+{ 0x2f , 0x002F },
+{ 0x30 , 0x0030 },
+{ 0x31 , 0x0031 },
+{ 0x32 , 0x0032 },
+{ 0x33 , 0x0033 },
+{ 0x34 , 0x0034 },
+{ 0x35 , 0x0035 },
+{ 0x36 , 0x0036 },
+{ 0x37 , 0x0037 },
+{ 0x38 , 0x0038 },
+{ 0x39 , 0x0039 },
+{ 0x3a , 0x003A },
+{ 0x3b , 0x003B },
+{ 0x3c , 0x003C },
+{ 0x3d , 0x003D },
+{ 0x3e , 0x003E },
+{ 0x3f , 0x003F },
+{ 0x40 , 0x0040 },
+{ 0x41 , 0x0041 },
+{ 0x42 , 0x0042 },
+{ 0x43 , 0x0043 },
+{ 0x44 , 0x0044 },
+{ 0x45 , 0x0045 },
+{ 0x46 , 0x0046 },
+{ 0x47 , 0x0047 },
+{ 0x48 , 0x0048 },
+{ 0x49 , 0x0049 },
+{ 0x4a , 0x004A },
+{ 0x4b , 0x004B },
+{ 0x4c , 0x004C },
+{ 0x4d , 0x004D },
+{ 0x4e , 0x004E },
+{ 0x4f , 0x004F },
+{ 0x50 , 0x0050 },
+{ 0x51 , 0x0051 },
+{ 0x52 , 0x0052 },
+{ 0x53 , 0x0053 },
+{ 0x54 , 0x0054 },
+{ 0x55 , 0x0055 },
+{ 0x56 , 0x0056 },
+{ 0x57 , 0x0057 },
+{ 0x58 , 0x0058 },
+{ 0x59 , 0x0059 },
+{ 0x5a , 0x005A },
+{ 0x5b , 0x005B },
+{ 0x5c , 0x005C },
+{ 0x5d , 0x005D },
+{ 0x5e , 0x005E },
+{ 0x5f , 0x005F },
+{ 0x60 , 0x0060 },
+{ 0x61 , 0x0061 },
+{ 0x62 , 0x0062 },
+{ 0x63 , 0x0063 },
+{ 0x64 , 0x0064 },
+{ 0x65 , 0x0065 },
+{ 0x66 , 0x0066 },
+{ 0x67 , 0x0067 },
+{ 0x68 , 0x0068 },
+{ 0x69 , 0x0069 },
+{ 0x6a , 0x006A },
+{ 0x6b , 0x006B },
+{ 0x6c , 0x006C },
+{ 0x6d , 0x006D },
+{ 0x6e , 0x006E },
+{ 0x6f , 0x006F },
+{ 0x70 , 0x0070 },
+{ 0x71 , 0x0071 },
+{ 0x72 , 0x0072 },
+{ 0x73 , 0x0073 },
+{ 0x74 , 0x0074 },
+{ 0x75 , 0x0075 },
+{ 0x76 , 0x0076 },
+{ 0x77 , 0x0077 },
+{ 0x78 , 0x0078 },
+{ 0x79 , 0x0079 },
+{ 0x7a , 0x007A },
+{ 0x7b , 0x007B },
+{ 0x7c , 0x007C },
+{ 0x7d , 0x007D },
+{ 0x7e , 0x007E },
+{ 0x7f , 0x007F },
+{ 0x80 , 0x0080 },
+{ 0x81 , 0x0081 },
+{ 0x82 , 0x0082 },
+{ 0x83 , 0x0083 },
+{ 0x84 , 0x0084 },
+{ 0x85 , 0x0085 },
+{ 0x86 , 0x0086 },
+{ 0x87 , 0x0087 },
+{ 0x88 , 0x0088 },
+{ 0x89 , 0x0089 },
+{ 0x8a , 0x008A },
+{ 0x8b , 0x008B },
+{ 0x8c , 0x008C },
+{ 0x8d , 0x008D },
+{ 0x8e , 0x008E },
+{ 0x8f , 0x008F },
+{ 0x90 , 0x0090 },
+{ 0x91 , 0x0091 },
+{ 0x92 , 0x0092 },
+{ 0x93 , 0x0093 },
+{ 0x94 , 0x0094 },
+{ 0x95 , 0x0095 },
+{ 0x96 , 0x0096 },
+{ 0x97 , 0x0097 },
+{ 0x98 , 0x0098 },
+{ 0x99 , 0x0099 },
+{ 0x9a , 0x009A },
+{ 0x9b , 0x009B },
+{ 0x9c , 0x009C },
+{ 0x9d , 0x009D },
+{ 0x9e , 0x009E },
+{ 0x9f , 0x009F },
+{ 0xa0 , 0x00A0 },
+{ 0xa1 , 0x2018 },
+{ 0xa2 , 0x2019 },
+{ 0xa3 , 0x00A3 },
+{ 0xa6 , 0x00A6 },
+{ 0xa7 , 0x00A7 },
+{ 0xa8 , 0x00A8 },
+{ 0xa9 , 0x00A9 },
+{ 0xab , 0x00AB },
+{ 0xac , 0x00AC },
+{ 0xad , 0x00AD },
+{ 0xaf , 0x2015 },
+{ 0xb0 , 0x00B0 },
+{ 0xb1 , 0x00B1 },
+{ 0xb2 , 0x00B2 },
+{ 0xb3 , 0x00B3 },
+{ 0xb4 , 0x0384 },
+{ 0xb5 , 0x0385 },
+{ 0xb6 , 0x0386 },
+{ 0xb7 , 0x00B7 },
+{ 0xb8 , 0x0388 },
+{ 0xb9 , 0x0389 },
+{ 0xba , 0x038A },
+{ 0xbb , 0x00BB },
+{ 0xbc , 0x038C },
+{ 0xbd , 0x00BD },
+{ 0xbe , 0x038E },
+{ 0xbf , 0x038F },
+{ 0xc0 , 0x0390 },
+{ 0xc1 , 0x0391 },
+{ 0xc2 , 0x0392 },
+{ 0xc3 , 0x0393 },
+{ 0xc4 , 0x0394 },
+{ 0xc5 , 0x0395 },
+{ 0xc6 , 0x0396 },
+{ 0xc7 , 0x0397 },
+{ 0xc8 , 0x0398 },
+{ 0xc9 , 0x0399 },
+{ 0xca , 0x039A },
+{ 0xcb , 0x039B },
+{ 0xcc , 0x039C },
+{ 0xcd , 0x039D },
+{ 0xce , 0x039E },
+{ 0xcf , 0x039F },
+{ 0xd0 , 0x03A0 },
+{ 0xd1 , 0x03A1 },
+{ 0xd3 , 0x03A3 },
+{ 0xd4 , 0x03A4 },
+{ 0xd5 , 0x03A5 },
+{ 0xd6 , 0x03A6 },
+{ 0xd7 , 0x03A7 },
+{ 0xd8 , 0x03A8 },
+{ 0xd9 , 0x03A9 },
+{ 0xda , 0x03AA },
+{ 0xdb , 0x03AB },
+{ 0xdc , 0x03AC },
+{ 0xdd , 0x03AD },
+{ 0xde , 0x03AE },
+{ 0xdf , 0x03AF },
+{ 0xe0 , 0x03B0 },
+{ 0xe1 , 0x03B1 },
+{ 0xe2 , 0x03B2 },
+{ 0xe3 , 0x03B3 },
+{ 0xe4 , 0x03B4 },
+{ 0xe5 , 0x03B5 },
+{ 0xe6 , 0x03B6 },
+{ 0xe7 , 0x03B7 },
+{ 0xe8 , 0x03B8 },
+{ 0xe9 , 0x03B9 },
+{ 0xea , 0x03BA },
+{ 0xeb , 0x03BB },
+{ 0xec , 0x03BC },
+{ 0xed , 0x03BD },
+{ 0xee , 0x03BE },
+{ 0xef , 0x03BF },
+{ 0xf0 , 0x03C0 },
+{ 0xf1 , 0x03C1 },
+{ 0xf2 , 0x03C2 },
+{ 0xf3 , 0x03C3 },
+{ 0xf4 , 0x03C4 },
+{ 0xf5 , 0x03C5 },
+{ 0xf6 , 0x03C6 },
+{ 0xf7 , 0x03C7 },
+{ 0xf8 , 0x03C8 },
+{ 0xf9 , 0x03C9 },
+{ 0xfa , 0x03CA },
+{ 0xfb , 0x03CB },
+{ 0xfc , 0x03CC },
+{ 0xfd , 0x03CD },
+{ 0xfe , 0x03CE },
diff --git a/extra/locale/charmaps/ISO-8859-8.pairs b/extra/locale/charmaps/ISO-8859-8.pairs
new file mode 100644
index 000000000..8d32dd859
--- /dev/null
+++ b/extra/locale/charmaps/ISO-8859-8.pairs
@@ -0,0 +1,220 @@
+{ 0x00 , 0x0000 },
+{ 0x01 , 0x0001 },
+{ 0x02 , 0x0002 },
+{ 0x03 , 0x0003 },
+{ 0x04 , 0x0004 },
+{ 0x05 , 0x0005 },
+{ 0x06 , 0x0006 },
+{ 0x07 , 0x0007 },
+{ 0x08 , 0x0008 },
+{ 0x09 , 0x0009 },
+{ 0x0a , 0x000A },
+{ 0x0b , 0x000B },
+{ 0x0c , 0x000C },
+{ 0x0d , 0x000D },
+{ 0x0e , 0x000E },
+{ 0x0f , 0x000F },
+{ 0x10 , 0x0010 },
+{ 0x11 , 0x0011 },
+{ 0x12 , 0x0012 },
+{ 0x13 , 0x0013 },
+{ 0x14 , 0x0014 },
+{ 0x15 , 0x0015 },
+{ 0x16 , 0x0016 },
+{ 0x17 , 0x0017 },
+{ 0x18 , 0x0018 },
+{ 0x19 , 0x0019 },
+{ 0x1a , 0x001A },
+{ 0x1b , 0x001B },
+{ 0x1c , 0x001C },
+{ 0x1d , 0x001D },
+{ 0x1e , 0x001E },
+{ 0x1f , 0x001F },
+{ 0x20 , 0x0020 },
+{ 0x21 , 0x0021 },
+{ 0x22 , 0x0022 },
+{ 0x23 , 0x0023 },
+{ 0x24 , 0x0024 },
+{ 0x25 , 0x0025 },
+{ 0x26 , 0x0026 },
+{ 0x27 , 0x0027 },
+{ 0x28 , 0x0028 },
+{ 0x29 , 0x0029 },
+{ 0x2a , 0x002A },
+{ 0x2b , 0x002B },
+{ 0x2c , 0x002C },
+{ 0x2d , 0x002D },
+{ 0x2e , 0x002E },
+{ 0x2f , 0x002F },
+{ 0x30 , 0x0030 },
+{ 0x31 , 0x0031 },
+{ 0x32 , 0x0032 },
+{ 0x33 , 0x0033 },
+{ 0x34 , 0x0034 },
+{ 0x35 , 0x0035 },
+{ 0x36 , 0x0036 },
+{ 0x37 , 0x0037 },
+{ 0x38 , 0x0038 },
+{ 0x39 , 0x0039 },
+{ 0x3a , 0x003A },
+{ 0x3b , 0x003B },
+{ 0x3c , 0x003C },
+{ 0x3d , 0x003D },
+{ 0x3e , 0x003E },
+{ 0x3f , 0x003F },
+{ 0x40 , 0x0040 },
+{ 0x41 , 0x0041 },
+{ 0x42 , 0x0042 },
+{ 0x43 , 0x0043 },
+{ 0x44 , 0x0044 },
+{ 0x45 , 0x0045 },
+{ 0x46 , 0x0046 },
+{ 0x47 , 0x0047 },
+{ 0x48 , 0x0048 },
+{ 0x49 , 0x0049 },
+{ 0x4a , 0x004A },
+{ 0x4b , 0x004B },
+{ 0x4c , 0x004C },
+{ 0x4d , 0x004D },
+{ 0x4e , 0x004E },
+{ 0x4f , 0x004F },
+{ 0x50 , 0x0050 },
+{ 0x51 , 0x0051 },
+{ 0x52 , 0x0052 },
+{ 0x53 , 0x0053 },
+{ 0x54 , 0x0054 },
+{ 0x55 , 0x0055 },
+{ 0x56 , 0x0056 },
+{ 0x57 , 0x0057 },
+{ 0x58 , 0x0058 },
+{ 0x59 , 0x0059 },
+{ 0x5a , 0x005A },
+{ 0x5b , 0x005B },
+{ 0x5c , 0x005C },
+{ 0x5d , 0x005D },
+{ 0x5e , 0x005E },
+{ 0x5f , 0x005F },
+{ 0x60 , 0x0060 },
+{ 0x61 , 0x0061 },
+{ 0x62 , 0x0062 },
+{ 0x63 , 0x0063 },
+{ 0x64 , 0x0064 },
+{ 0x65 , 0x0065 },
+{ 0x66 , 0x0066 },
+{ 0x67 , 0x0067 },
+{ 0x68 , 0x0068 },
+{ 0x69 , 0x0069 },
+{ 0x6a , 0x006A },
+{ 0x6b , 0x006B },
+{ 0x6c , 0x006C },
+{ 0x6d , 0x006D },
+{ 0x6e , 0x006E },
+{ 0x6f , 0x006F },
+{ 0x70 , 0x0070 },
+{ 0x71 , 0x0071 },
+{ 0x72 , 0x0072 },
+{ 0x73 , 0x0073 },
+{ 0x74 , 0x0074 },
+{ 0x75 , 0x0075 },
+{ 0x76 , 0x0076 },
+{ 0x77 , 0x0077 },
+{ 0x78 , 0x0078 },
+{ 0x79 , 0x0079 },
+{ 0x7a , 0x007A },
+{ 0x7b , 0x007B },
+{ 0x7c , 0x007C },
+{ 0x7d , 0x007D },
+{ 0x7e , 0x007E },
+{ 0x7f , 0x007F },
+{ 0x80 , 0x0080 },
+{ 0x81 , 0x0081 },
+{ 0x82 , 0x0082 },
+{ 0x83 , 0x0083 },
+{ 0x84 , 0x0084 },
+{ 0x85 , 0x0085 },
+{ 0x86 , 0x0086 },
+{ 0x87 , 0x0087 },
+{ 0x88 , 0x0088 },
+{ 0x89 , 0x0089 },
+{ 0x8a , 0x008A },
+{ 0x8b , 0x008B },
+{ 0x8c , 0x008C },
+{ 0x8d , 0x008D },
+{ 0x8e , 0x008E },
+{ 0x8f , 0x008F },
+{ 0x90 , 0x0090 },
+{ 0x91 , 0x0091 },
+{ 0x92 , 0x0092 },
+{ 0x93 , 0x0093 },
+{ 0x94 , 0x0094 },
+{ 0x95 , 0x0095 },
+{ 0x96 , 0x0096 },
+{ 0x97 , 0x0097 },
+{ 0x98 , 0x0098 },
+{ 0x99 , 0x0099 },
+{ 0x9a , 0x009A },
+{ 0x9b , 0x009B },
+{ 0x9c , 0x009C },
+{ 0x9d , 0x009D },
+{ 0x9e , 0x009E },
+{ 0x9f , 0x009F },
+{ 0xa0 , 0x00A0 },
+{ 0xa2 , 0x00A2 },
+{ 0xa3 , 0x00A3 },
+{ 0xa4 , 0x00A4 },
+{ 0xa5 , 0x00A5 },
+{ 0xa6 , 0x00A6 },
+{ 0xa7 , 0x00A7 },
+{ 0xa8 , 0x00A8 },
+{ 0xa9 , 0x00A9 },
+{ 0xaa , 0x00D7 },
+{ 0xab , 0x00AB },
+{ 0xac , 0x00AC },
+{ 0xad , 0x00AD },
+{ 0xae , 0x00AE },
+{ 0xaf , 0x00AF },
+{ 0xb0 , 0x00B0 },
+{ 0xb1 , 0x00B1 },
+{ 0xb2 , 0x00B2 },
+{ 0xb3 , 0x00B3 },
+{ 0xb4 , 0x00B4 },
+{ 0xb5 , 0x00B5 },
+{ 0xb6 , 0x00B6 },
+{ 0xb7 , 0x00B7 },
+{ 0xb8 , 0x00B8 },
+{ 0xb9 , 0x00B9 },
+{ 0xba , 0x00F7 },
+{ 0xbb , 0x00BB },
+{ 0xbc , 0x00BC },
+{ 0xbd , 0x00BD },
+{ 0xbe , 0x00BE },
+{ 0xdf , 0x2017 },
+{ 0xe0 , 0x05D0 },
+{ 0xe1 , 0x05D1 },
+{ 0xe2 , 0x05D2 },
+{ 0xe3 , 0x05D3 },
+{ 0xe4 , 0x05D4 },
+{ 0xe5 , 0x05D5 },
+{ 0xe6 , 0x05D6 },
+{ 0xe7 , 0x05D7 },
+{ 0xe8 , 0x05D8 },
+{ 0xe9 , 0x05D9 },
+{ 0xea , 0x05DA },
+{ 0xeb , 0x05DB },
+{ 0xec , 0x05DC },
+{ 0xed , 0x05DD },
+{ 0xee , 0x05DE },
+{ 0xef , 0x05DF },
+{ 0xf0 , 0x05E0 },
+{ 0xf1 , 0x05E1 },
+{ 0xf2 , 0x05E2 },
+{ 0xf3 , 0x05E3 },
+{ 0xf4 , 0x05E4 },
+{ 0xf5 , 0x05E5 },
+{ 0xf6 , 0x05E6 },
+{ 0xf7 , 0x05E7 },
+{ 0xf8 , 0x05E8 },
+{ 0xf9 , 0x05E9 },
+{ 0xfa , 0x05EA },
+{ 0xfd , 0x200E },
+{ 0xfe , 0x200F },
diff --git a/extra/locale/charmaps/ISO-8859-9.pairs b/extra/locale/charmaps/ISO-8859-9.pairs
new file mode 100644
index 000000000..a5748cda7
--- /dev/null
+++ b/extra/locale/charmaps/ISO-8859-9.pairs
@@ -0,0 +1,256 @@
+{ 0x00 , 0x0000 },
+{ 0x01 , 0x0001 },
+{ 0x02 , 0x0002 },
+{ 0x03 , 0x0003 },
+{ 0x04 , 0x0004 },
+{ 0x05 , 0x0005 },
+{ 0x06 , 0x0006 },
+{ 0x07 , 0x0007 },
+{ 0x08 , 0x0008 },
+{ 0x09 , 0x0009 },
+{ 0x0a , 0x000A },
+{ 0x0b , 0x000B },
+{ 0x0c , 0x000C },
+{ 0x0d , 0x000D },
+{ 0x0e , 0x000E },
+{ 0x0f , 0x000F },
+{ 0x10 , 0x0010 },
+{ 0x11 , 0x0011 },
+{ 0x12 , 0x0012 },
+{ 0x13 , 0x0013 },
+{ 0x14 , 0x0014 },
+{ 0x15 , 0x0015 },
+{ 0x16 , 0x0016 },
+{ 0x17 , 0x0017 },
+{ 0x18 , 0x0018 },
+{ 0x19 , 0x0019 },
+{ 0x1a , 0x001A },
+{ 0x1b , 0x001B },
+{ 0x1c , 0x001C },
+{ 0x1d , 0x001D },
+{ 0x1e , 0x001E },
+{ 0x1f , 0x001F },
+{ 0x20 , 0x0020 },
+{ 0x21 , 0x0021 },
+{ 0x22 , 0x0022 },
+{ 0x23 , 0x0023 },
+{ 0x24 , 0x0024 },
+{ 0x25 , 0x0025 },
+{ 0x26 , 0x0026 },
+{ 0x27 , 0x0027 },
+{ 0x28 , 0x0028 },
+{ 0x29 , 0x0029 },
+{ 0x2a , 0x002A },
+{ 0x2b , 0x002B },
+{ 0x2c , 0x002C },
+{ 0x2d , 0x002D },
+{ 0x2e , 0x002E },
+{ 0x2f , 0x002F },
+{ 0x30 , 0x0030 },
+{ 0x31 , 0x0031 },
+{ 0x32 , 0x0032 },
+{ 0x33 , 0x0033 },
+{ 0x34 , 0x0034 },
+{ 0x35 , 0x0035 },
+{ 0x36 , 0x0036 },
+{ 0x37 , 0x0037 },
+{ 0x38 , 0x0038 },
+{ 0x39 , 0x0039 },
+{ 0x3a , 0x003A },
+{ 0x3b , 0x003B },
+{ 0x3c , 0x003C },
+{ 0x3d , 0x003D },
+{ 0x3e , 0x003E },
+{ 0x3f , 0x003F },
+{ 0x40 , 0x0040 },
+{ 0x41 , 0x0041 },
+{ 0x42 , 0x0042 },
+{ 0x43 , 0x0043 },
+{ 0x44 , 0x0044 },
+{ 0x45 , 0x0045 },
+{ 0x46 , 0x0046 },
+{ 0x47 , 0x0047 },
+{ 0x48 , 0x0048 },
+{ 0x49 , 0x0049 },
+{ 0x4a , 0x004A },
+{ 0x4b , 0x004B },
+{ 0x4c , 0x004C },
+{ 0x4d , 0x004D },
+{ 0x4e , 0x004E },
+{ 0x4f , 0x004F },
+{ 0x50 , 0x0050 },
+{ 0x51 , 0x0051 },
+{ 0x52 , 0x0052 },
+{ 0x53 , 0x0053 },
+{ 0x54 , 0x0054 },
+{ 0x55 , 0x0055 },
+{ 0x56 , 0x0056 },
+{ 0x57 , 0x0057 },
+{ 0x58 , 0x0058 },
+{ 0x59 , 0x0059 },
+{ 0x5a , 0x005A },
+{ 0x5b , 0x005B },
+{ 0x5c , 0x005C },
+{ 0x5d , 0x005D },
+{ 0x5e , 0x005E },
+{ 0x5f , 0x005F },
+{ 0x60 , 0x0060 },
+{ 0x61 , 0x0061 },
+{ 0x62 , 0x0062 },
+{ 0x63 , 0x0063 },
+{ 0x64 , 0x0064 },
+{ 0x65 , 0x0065 },
+{ 0x66 , 0x0066 },
+{ 0x67 , 0x0067 },
+{ 0x68 , 0x0068 },
+{ 0x69 , 0x0069 },
+{ 0x6a , 0x006A },
+{ 0x6b , 0x006B },
+{ 0x6c , 0x006C },
+{ 0x6d , 0x006D },
+{ 0x6e , 0x006E },
+{ 0x6f , 0x006F },
+{ 0x70 , 0x0070 },
+{ 0x71 , 0x0071 },
+{ 0x72 , 0x0072 },
+{ 0x73 , 0x0073 },
+{ 0x74 , 0x0074 },
+{ 0x75 , 0x0075 },
+{ 0x76 , 0x0076 },
+{ 0x77 , 0x0077 },
+{ 0x78 , 0x0078 },
+{ 0x79 , 0x0079 },
+{ 0x7a , 0x007A },
+{ 0x7b , 0x007B },
+{ 0x7c , 0x007C },
+{ 0x7d , 0x007D },
+{ 0x7e , 0x007E },
+{ 0x7f , 0x007F },
+{ 0x80 , 0x0080 },
+{ 0x81 , 0x0081 },
+{ 0x82 , 0x0082 },
+{ 0x83 , 0x0083 },
+{ 0x84 , 0x0084 },
+{ 0x85 , 0x0085 },
+{ 0x86 , 0x0086 },
+{ 0x87 , 0x0087 },
+{ 0x88 , 0x0088 },
+{ 0x89 , 0x0089 },
+{ 0x8a , 0x008A },
+{ 0x8b , 0x008B },
+{ 0x8c , 0x008C },
+{ 0x8d , 0x008D },
+{ 0x8e , 0x008E },
+{ 0x8f , 0x008F },
+{ 0x90 , 0x0090 },
+{ 0x91 , 0x0091 },
+{ 0x92 , 0x0092 },
+{ 0x93 , 0x0093 },
+{ 0x94 , 0x0094 },
+{ 0x95 , 0x0095 },
+{ 0x96 , 0x0096 },
+{ 0x97 , 0x0097 },
+{ 0x98 , 0x0098 },
+{ 0x99 , 0x0099 },
+{ 0x9a , 0x009A },
+{ 0x9b , 0x009B },
+{ 0x9c , 0x009C },
+{ 0x9d , 0x009D },
+{ 0x9e , 0x009E },
+{ 0x9f , 0x009F },
+{ 0xa0 , 0x00A0 },
+{ 0xa1 , 0x00A1 },
+{ 0xa2 , 0x00A2 },
+{ 0xa3 , 0x00A3 },
+{ 0xa4 , 0x00A4 },
+{ 0xa5 , 0x00A5 },
+{ 0xa6 , 0x00A6 },
+{ 0xa7 , 0x00A7 },
+{ 0xa8 , 0x00A8 },
+{ 0xa9 , 0x00A9 },
+{ 0xaa , 0x00AA },
+{ 0xab , 0x00AB },
+{ 0xac , 0x00AC },
+{ 0xad , 0x00AD },
+{ 0xae , 0x00AE },
+{ 0xaf , 0x00AF },
+{ 0xb0 , 0x00B0 },
+{ 0xb1 , 0x00B1 },
+{ 0xb2 , 0x00B2 },
+{ 0xb3 , 0x00B3 },
+{ 0xb4 , 0x00B4 },
+{ 0xb5 , 0x00B5 },
+{ 0xb6 , 0x00B6 },
+{ 0xb7 , 0x00B7 },
+{ 0xb8 , 0x00B8 },
+{ 0xb9 , 0x00B9 },
+{ 0xba , 0x00BA },
+{ 0xbb , 0x00BB },
+{ 0xbc , 0x00BC },
+{ 0xbd , 0x00BD },
+{ 0xbe , 0x00BE },
+{ 0xbf , 0x00BF },
+{ 0xc0 , 0x00C0 },
+{ 0xc1 , 0x00C1 },
+{ 0xc2 , 0x00C2 },
+{ 0xc3 , 0x00C3 },
+{ 0xc4 , 0x00C4 },
+{ 0xc5 , 0x00C5 },
+{ 0xc6 , 0x00C6 },
+{ 0xc7 , 0x00C7 },
+{ 0xc8 , 0x00C8 },
+{ 0xc9 , 0x00C9 },
+{ 0xca , 0x00CA },
+{ 0xcb , 0x00CB },
+{ 0xcc , 0x00CC },
+{ 0xcd , 0x00CD },
+{ 0xce , 0x00CE },
+{ 0xcf , 0x00CF },
+{ 0xd0 , 0x011E },
+{ 0xd1 , 0x00D1 },
+{ 0xd2 , 0x00D2 },
+{ 0xd3 , 0x00D3 },
+{ 0xd4 , 0x00D4 },
+{ 0xd5 , 0x00D5 },
+{ 0xd6 , 0x00D6 },
+{ 0xd7 , 0x00D7 },
+{ 0xd8 , 0x00D8 },
+{ 0xd9 , 0x00D9 },
+{ 0xda , 0x00DA },
+{ 0xdb , 0x00DB },
+{ 0xdc , 0x00DC },
+{ 0xdd , 0x0130 },
+{ 0xde , 0x015E },
+{ 0xdf , 0x00DF },
+{ 0xe0 , 0x00E0 },
+{ 0xe1 , 0x00E1 },
+{ 0xe2 , 0x00E2 },
+{ 0xe3 , 0x00E3 },
+{ 0xe4 , 0x00E4 },
+{ 0xe5 , 0x00E5 },
+{ 0xe6 , 0x00E6 },
+{ 0xe7 , 0x00E7 },
+{ 0xe8 , 0x00E8 },
+{ 0xe9 , 0x00E9 },
+{ 0xea , 0x00EA },
+{ 0xeb , 0x00EB },
+{ 0xec , 0x00EC },
+{ 0xed , 0x00ED },
+{ 0xee , 0x00EE },
+{ 0xef , 0x00EF },
+{ 0xf0 , 0x011F },
+{ 0xf1 , 0x00F1 },
+{ 0xf2 , 0x00F2 },
+{ 0xf3 , 0x00F3 },
+{ 0xf4 , 0x00F4 },
+{ 0xf5 , 0x00F5 },
+{ 0xf6 , 0x00F6 },
+{ 0xf7 , 0x00F7 },
+{ 0xf8 , 0x00F8 },
+{ 0xf9 , 0x00F9 },
+{ 0xfa , 0x00FA },
+{ 0xfb , 0x00FB },
+{ 0xfc , 0x00FC },
+{ 0xfd , 0x0131 },
+{ 0xfe , 0x015F },
+{ 0xff , 0x00FF },
diff --git a/extra/locale/charmaps/KOI8-R.pairs b/extra/locale/charmaps/KOI8-R.pairs
new file mode 100644
index 000000000..f34053d95
--- /dev/null
+++ b/extra/locale/charmaps/KOI8-R.pairs
@@ -0,0 +1,256 @@
+{ 0x00 , 0x0000 },
+{ 0x01 , 0x0001 },
+{ 0x02 , 0x0002 },
+{ 0x03 , 0x0003 },
+{ 0x04 , 0x0004 },
+{ 0x05 , 0x0005 },
+{ 0x06 , 0x0006 },
+{ 0x07 , 0x0007 },
+{ 0x08 , 0x0008 },
+{ 0x09 , 0x0009 },
+{ 0x0a , 0x000A },
+{ 0x0b , 0x000B },
+{ 0x0c , 0x000C },
+{ 0x0d , 0x000D },
+{ 0x0e , 0x000E },
+{ 0x0f , 0x000F },
+{ 0x10 , 0x0010 },
+{ 0x11 , 0x0011 },
+{ 0x12 , 0x0012 },
+{ 0x13 , 0x0013 },
+{ 0x14 , 0x0014 },
+{ 0x15 , 0x0015 },
+{ 0x16 , 0x0016 },
+{ 0x17 , 0x0017 },
+{ 0x18 , 0x0018 },
+{ 0x19 , 0x0019 },
+{ 0x1a , 0x001A },
+{ 0x1b , 0x001B },
+{ 0x1c , 0x001C },
+{ 0x1d , 0x001D },
+{ 0x1e , 0x001E },
+{ 0x1f , 0x001F },
+{ 0x20 , 0x0020 },
+{ 0x21 , 0x0021 },
+{ 0x22 , 0x0022 },
+{ 0x23 , 0x0023 },
+{ 0x24 , 0x0024 },
+{ 0x25 , 0x0025 },
+{ 0x26 , 0x0026 },
+{ 0x27 , 0x0027 },
+{ 0x28 , 0x0028 },
+{ 0x29 , 0x0029 },
+{ 0x2a , 0x002A },
+{ 0x2b , 0x002B },
+{ 0x2c , 0x002C },
+{ 0x2d , 0x002D },
+{ 0x2e , 0x002E },
+{ 0x2f , 0x002F },
+{ 0x30 , 0x0030 },
+{ 0x31 , 0x0031 },
+{ 0x32 , 0x0032 },
+{ 0x33 , 0x0033 },
+{ 0x34 , 0x0034 },
+{ 0x35 , 0x0035 },
+{ 0x36 , 0x0036 },
+{ 0x37 , 0x0037 },
+{ 0x38 , 0x0038 },
+{ 0x39 , 0x0039 },
+{ 0x3a , 0x003A },
+{ 0x3b , 0x003B },
+{ 0x3c , 0x003C },
+{ 0x3d , 0x003D },
+{ 0x3e , 0x003E },
+{ 0x3f , 0x003F },
+{ 0x40 , 0x0040 },
+{ 0x41 , 0x0041 },
+{ 0x42 , 0x0042 },
+{ 0x43 , 0x0043 },
+{ 0x44 , 0x0044 },
+{ 0x45 , 0x0045 },
+{ 0x46 , 0x0046 },
+{ 0x47 , 0x0047 },
+{ 0x48 , 0x0048 },
+{ 0x49 , 0x0049 },
+{ 0x4a , 0x004A },
+{ 0x4b , 0x004B },
+{ 0x4c , 0x004C },
+{ 0x4d , 0x004D },
+{ 0x4e , 0x004E },
+{ 0x4f , 0x004F },
+{ 0x50 , 0x0050 },
+{ 0x51 , 0x0051 },
+{ 0x52 , 0x0052 },
+{ 0x53 , 0x0053 },
+{ 0x54 , 0x0054 },
+{ 0x55 , 0x0055 },
+{ 0x56 , 0x0056 },
+{ 0x57 , 0x0057 },
+{ 0x58 , 0x0058 },
+{ 0x59 , 0x0059 },
+{ 0x5a , 0x005A },
+{ 0x5b , 0x005B },
+{ 0x5c , 0x005C },
+{ 0x5d , 0x005D },
+{ 0x5e , 0x005E },
+{ 0x5f , 0x005F },
+{ 0x60 , 0x0060 },
+{ 0x61 , 0x0061 },
+{ 0x62 , 0x0062 },
+{ 0x63 , 0x0063 },
+{ 0x64 , 0x0064 },
+{ 0x65 , 0x0065 },
+{ 0x66 , 0x0066 },
+{ 0x67 , 0x0067 },
+{ 0x68 , 0x0068 },
+{ 0x69 , 0x0069 },
+{ 0x6a , 0x006A },
+{ 0x6b , 0x006B },
+{ 0x6c , 0x006C },
+{ 0x6d , 0x006D },
+{ 0x6e , 0x006E },
+{ 0x6f , 0x006F },
+{ 0x70 , 0x0070 },
+{ 0x71 , 0x0071 },
+{ 0x72 , 0x0072 },
+{ 0x73 , 0x0073 },
+{ 0x74 , 0x0074 },
+{ 0x75 , 0x0075 },
+{ 0x76 , 0x0076 },
+{ 0x77 , 0x0077 },
+{ 0x78 , 0x0078 },
+{ 0x79 , 0x0079 },
+{ 0x7a , 0x007A },
+{ 0x7b , 0x007B },
+{ 0x7c , 0x007C },
+{ 0x7d , 0x007D },
+{ 0x7e , 0x007E },
+{ 0x7f , 0x007F },
+{ 0x80 , 0x2500 },
+{ 0x81 , 0x2502 },
+{ 0x82 , 0x250C },
+{ 0x83 , 0x2510 },
+{ 0x84 , 0x2514 },
+{ 0x85 , 0x2518 },
+{ 0x86 , 0x251C },
+{ 0x87 , 0x2524 },
+{ 0x88 , 0x252C },
+{ 0x89 , 0x2534 },
+{ 0x8a , 0x253C },
+{ 0x8b , 0x2580 },
+{ 0x8c , 0x2584 },
+{ 0x8d , 0x2588 },
+{ 0x8e , 0x258C },
+{ 0x8f , 0x2590 },
+{ 0x90 , 0x2591 },
+{ 0x91 , 0x2592 },
+{ 0x92 , 0x2593 },
+{ 0x93 , 0x2320 },
+{ 0x94 , 0x25A0 },
+{ 0x95 , 0x2219 },
+{ 0x96 , 0x221A },
+{ 0x97 , 0x2248 },
+{ 0x98 , 0x2264 },
+{ 0x99 , 0x2265 },
+{ 0x9a , 0x00A0 },
+{ 0x9b , 0x2321 },
+{ 0x9c , 0x00B0 },
+{ 0x9d , 0x00B2 },
+{ 0x9e , 0x00B7 },
+{ 0x9f , 0x00F7 },
+{ 0xa0 , 0x2550 },
+{ 0xa1 , 0x2551 },
+{ 0xa2 , 0x2552 },
+{ 0xa3 , 0x0451 },
+{ 0xa4 , 0x2553 },
+{ 0xa5 , 0x2554 },
+{ 0xa6 , 0x2555 },
+{ 0xa7 , 0x2556 },
+{ 0xa8 , 0x2557 },
+{ 0xa9 , 0x2558 },
+{ 0xaa , 0x2559 },
+{ 0xab , 0x255A },
+{ 0xac , 0x255B },
+{ 0xad , 0x255C },
+{ 0xae , 0x255D },
+{ 0xaf , 0x255E },
+{ 0xb0 , 0x255F },
+{ 0xb1 , 0x2560 },
+{ 0xb2 , 0x2561 },
+{ 0xb3 , 0x0401 },
+{ 0xb4 , 0x2562 },
+{ 0xb5 , 0x2563 },
+{ 0xb6 , 0x2564 },
+{ 0xb7 , 0x2565 },
+{ 0xb8 , 0x2566 },
+{ 0xb9 , 0x2567 },
+{ 0xba , 0x2568 },
+{ 0xbb , 0x2569 },
+{ 0xbc , 0x256A },
+{ 0xbd , 0x256B },
+{ 0xbe , 0x256C },
+{ 0xbf , 0x00A9 },
+{ 0xc0 , 0x044E },
+{ 0xc1 , 0x0430 },
+{ 0xc2 , 0x0431 },
+{ 0xc3 , 0x0446 },
+{ 0xc4 , 0x0434 },
+{ 0xc5 , 0x0435 },
+{ 0xc6 , 0x0444 },
+{ 0xc7 , 0x0433 },
+{ 0xc8 , 0x0445 },
+{ 0xc9 , 0x0438 },
+{ 0xca , 0x0439 },
+{ 0xcb , 0x043A },
+{ 0xcc , 0x043B },
+{ 0xcd , 0x043C },
+{ 0xce , 0x043D },
+{ 0xcf , 0x043E },
+{ 0xd0 , 0x043F },
+{ 0xd1 , 0x044F },
+{ 0xd2 , 0x0440 },
+{ 0xd3 , 0x0441 },
+{ 0xd4 , 0x0442 },
+{ 0xd5 , 0x0443 },
+{ 0xd6 , 0x0436 },
+{ 0xd7 , 0x0432 },
+{ 0xd8 , 0x044C },
+{ 0xd9 , 0x044B },
+{ 0xda , 0x0437 },
+{ 0xdb , 0x0448 },
+{ 0xdc , 0x044D },
+{ 0xdd , 0x0449 },
+{ 0xde , 0x0447 },
+{ 0xdf , 0x044A },
+{ 0xe0 , 0x042E },
+{ 0xe1 , 0x0410 },
+{ 0xe2 , 0x0411 },
+{ 0xe3 , 0x0426 },
+{ 0xe4 , 0x0414 },
+{ 0xe5 , 0x0415 },
+{ 0xe6 , 0x0424 },
+{ 0xe7 , 0x0413 },
+{ 0xe8 , 0x0425 },
+{ 0xe9 , 0x0418 },
+{ 0xea , 0x0419 },
+{ 0xeb , 0x041A },
+{ 0xec , 0x041B },
+{ 0xed , 0x041C },
+{ 0xee , 0x041D },
+{ 0xef , 0x041E },
+{ 0xf0 , 0x041F },
+{ 0xf1 , 0x042F },
+{ 0xf2 , 0x0420 },
+{ 0xf3 , 0x0421 },
+{ 0xf4 , 0x0422 },
+{ 0xf5 , 0x0423 },
+{ 0xf6 , 0x0416 },
+{ 0xf7 , 0x0412 },
+{ 0xf8 , 0x042C },
+{ 0xf9 , 0x042B },
+{ 0xfa , 0x0417 },
+{ 0xfb , 0x0428 },
+{ 0xfc , 0x042D },
+{ 0xfd , 0x0429 },
+{ 0xfe , 0x0427 },
+{ 0xff , 0x042A },
diff --git a/extra/locale/charmaps/KOI8-U.pairs b/extra/locale/charmaps/KOI8-U.pairs
new file mode 100644
index 000000000..ff05ed24f
--- /dev/null
+++ b/extra/locale/charmaps/KOI8-U.pairs
@@ -0,0 +1,256 @@
+{ 0x00 , 0x0000 },
+{ 0x01 , 0x0001 },
+{ 0x02 , 0x0002 },
+{ 0x03 , 0x0003 },
+{ 0x04 , 0x0004 },
+{ 0x05 , 0x0005 },
+{ 0x06 , 0x0006 },
+{ 0x07 , 0x0007 },
+{ 0x08 , 0x0008 },
+{ 0x09 , 0x0009 },
+{ 0x0a , 0x000A },
+{ 0x0b , 0x000B },
+{ 0x0c , 0x000C },
+{ 0x0d , 0x000D },
+{ 0x0e , 0x000E },
+{ 0x0f , 0x000F },
+{ 0x10 , 0x0010 },
+{ 0x11 , 0x0011 },
+{ 0x12 , 0x0012 },
+{ 0x13 , 0x0013 },
+{ 0x14 , 0x0014 },
+{ 0x15 , 0x0015 },
+{ 0x16 , 0x0016 },
+{ 0x17 , 0x0017 },
+{ 0x18 , 0x0018 },
+{ 0x19 , 0x0019 },
+{ 0x1a , 0x001A },
+{ 0x1b , 0x001B },
+{ 0x1c , 0x001C },
+{ 0x1d , 0x001D },
+{ 0x1e , 0x001E },
+{ 0x1f , 0x001F },
+{ 0x20 , 0x0020 },
+{ 0x21 , 0x0021 },
+{ 0x22 , 0x0022 },
+{ 0x23 , 0x0023 },
+{ 0x24 , 0x0024 },
+{ 0x25 , 0x0025 },
+{ 0x26 , 0x0026 },
+{ 0x27 , 0x0027 },
+{ 0x28 , 0x0028 },
+{ 0x29 , 0x0029 },
+{ 0x2a , 0x002A },
+{ 0x2b , 0x002B },
+{ 0x2c , 0x002C },
+{ 0x2d , 0x002D },
+{ 0x2e , 0x002E },
+{ 0x2f , 0x002F },
+{ 0x30 , 0x0030 },
+{ 0x31 , 0x0031 },
+{ 0x32 , 0x0032 },
+{ 0x33 , 0x0033 },
+{ 0x34 , 0x0034 },
+{ 0x35 , 0x0035 },
+{ 0x36 , 0x0036 },
+{ 0x37 , 0x0037 },
+{ 0x38 , 0x0038 },
+{ 0x39 , 0x0039 },
+{ 0x3a , 0x003A },
+{ 0x3b , 0x003B },
+{ 0x3c , 0x003C },
+{ 0x3d , 0x003D },
+{ 0x3e , 0x003E },
+{ 0x3f , 0x003F },
+{ 0x40 , 0x0040 },
+{ 0x41 , 0x0041 },
+{ 0x42 , 0x0042 },
+{ 0x43 , 0x0043 },
+{ 0x44 , 0x0044 },
+{ 0x45 , 0x0045 },
+{ 0x46 , 0x0046 },
+{ 0x47 , 0x0047 },
+{ 0x48 , 0x0048 },
+{ 0x49 , 0x0049 },
+{ 0x4a , 0x004A },
+{ 0x4b , 0x004B },
+{ 0x4c , 0x004C },
+{ 0x4d , 0x004D },
+{ 0x4e , 0x004E },
+{ 0x4f , 0x004F },
+{ 0x50 , 0x0050 },
+{ 0x51 , 0x0051 },
+{ 0x52 , 0x0052 },
+{ 0x53 , 0x0053 },
+{ 0x54 , 0x0054 },
+{ 0x55 , 0x0055 },
+{ 0x56 , 0x0056 },
+{ 0x57 , 0x0057 },
+{ 0x58 , 0x0058 },
+{ 0x59 , 0x0059 },
+{ 0x5a , 0x005A },
+{ 0x5b , 0x005B },
+{ 0x5c , 0x005C },
+{ 0x5d , 0x005D },
+{ 0x5e , 0x005E },
+{ 0x5f , 0x005F },
+{ 0x60 , 0x0060 },
+{ 0x61 , 0x0061 },
+{ 0x62 , 0x0062 },
+{ 0x63 , 0x0063 },
+{ 0x64 , 0x0064 },
+{ 0x65 , 0x0065 },
+{ 0x66 , 0x0066 },
+{ 0x67 , 0x0067 },
+{ 0x68 , 0x0068 },
+{ 0x69 , 0x0069 },
+{ 0x6a , 0x006A },
+{ 0x6b , 0x006B },
+{ 0x6c , 0x006C },
+{ 0x6d , 0x006D },
+{ 0x6e , 0x006E },
+{ 0x6f , 0x006F },
+{ 0x70 , 0x0070 },
+{ 0x71 , 0x0071 },
+{ 0x72 , 0x0072 },
+{ 0x73 , 0x0073 },
+{ 0x74 , 0x0074 },
+{ 0x75 , 0x0075 },
+{ 0x76 , 0x0076 },
+{ 0x77 , 0x0077 },
+{ 0x78 , 0x0078 },
+{ 0x79 , 0x0079 },
+{ 0x7a , 0x007A },
+{ 0x7b , 0x007B },
+{ 0x7c , 0x007C },
+{ 0x7d , 0x007D },
+{ 0x7e , 0x007E },
+{ 0x7f , 0x007F },
+{ 0x80 , 0x2500 },
+{ 0x81 , 0x2502 },
+{ 0x82 , 0x250C },
+{ 0x83 , 0x2510 },
+{ 0x84 , 0x2514 },
+{ 0x85 , 0x2518 },
+{ 0x86 , 0x251C },
+{ 0x87 , 0x2524 },
+{ 0x88 , 0x252C },
+{ 0x89 , 0x2534 },
+{ 0x8a , 0x253C },
+{ 0x8b , 0x2580 },
+{ 0x8c , 0x2584 },
+{ 0x8d , 0x2588 },
+{ 0x8e , 0x258C },
+{ 0x8f , 0x2590 },
+{ 0x90 , 0x2591 },
+{ 0x91 , 0x2592 },
+{ 0x92 , 0x2593 },
+{ 0x93 , 0x2320 },
+{ 0x94 , 0x25A0 },
+{ 0x95 , 0x2219 },
+{ 0x96 , 0x221A },
+{ 0x97 , 0x2248 },
+{ 0x98 , 0x2264 },
+{ 0x99 , 0x2265 },
+{ 0x9a , 0x00A0 },
+{ 0x9b , 0x2321 },
+{ 0x9c , 0x00B0 },
+{ 0x9d , 0x00B2 },
+{ 0x9e , 0x00B7 },
+{ 0x9f , 0x00F7 },
+{ 0xa0 , 0x2550 },
+{ 0xa1 , 0x2551 },
+{ 0xa2 , 0x2552 },
+{ 0xa3 , 0x0451 },
+{ 0xa4 , 0x0454 },
+{ 0xa5 , 0x2554 },
+{ 0xa6 , 0x0456 },
+{ 0xa7 , 0x0457 },
+{ 0xa8 , 0x2557 },
+{ 0xa9 , 0x2558 },
+{ 0xaa , 0x2559 },
+{ 0xab , 0x255A },
+{ 0xac , 0x255B },
+{ 0xad , 0x0491 },
+{ 0xae , 0x255D },
+{ 0xaf , 0x255E },
+{ 0xb0 , 0x255F },
+{ 0xb1 , 0x2560 },
+{ 0xb2 , 0x2561 },
+{ 0xb3 , 0x0401 },
+{ 0xb4 , 0x0404 },
+{ 0xb5 , 0x2563 },
+{ 0xb6 , 0x0406 },
+{ 0xb7 , 0x0407 },
+{ 0xb8 , 0x2566 },
+{ 0xb9 , 0x2567 },
+{ 0xba , 0x2568 },
+{ 0xbb , 0x2569 },
+{ 0xbc , 0x256A },
+{ 0xbd , 0x0490 },
+{ 0xbe , 0x256C },
+{ 0xbf , 0x00A9 },
+{ 0xc0 , 0x044E },
+{ 0xc1 , 0x0430 },
+{ 0xc2 , 0x0431 },
+{ 0xc3 , 0x0446 },
+{ 0xc4 , 0x0434 },
+{ 0xc5 , 0x0435 },
+{ 0xc6 , 0x0444 },
+{ 0xc7 , 0x0433 },
+{ 0xc8 , 0x0445 },
+{ 0xc9 , 0x0438 },
+{ 0xca , 0x0439 },
+{ 0xcb , 0x043A },
+{ 0xcc , 0x043B },
+{ 0xcd , 0x043C },
+{ 0xce , 0x043D },
+{ 0xcf , 0x043E },
+{ 0xd0 , 0x043F },
+{ 0xd1 , 0x044F },
+{ 0xd2 , 0x0440 },
+{ 0xd3 , 0x0441 },
+{ 0xd4 , 0x0442 },
+{ 0xd5 , 0x0443 },
+{ 0xd6 , 0x0436 },
+{ 0xd7 , 0x0432 },
+{ 0xd8 , 0x044C },
+{ 0xd9 , 0x044B },
+{ 0xda , 0x0437 },
+{ 0xdb , 0x0448 },
+{ 0xdc , 0x044D },
+{ 0xdd , 0x0449 },
+{ 0xde , 0x0447 },
+{ 0xdf , 0x044A },
+{ 0xe0 , 0x042E },
+{ 0xe1 , 0x0410 },
+{ 0xe2 , 0x0411 },
+{ 0xe3 , 0x0426 },
+{ 0xe4 , 0x0414 },
+{ 0xe5 , 0x0415 },
+{ 0xe6 , 0x0424 },
+{ 0xe7 , 0x0413 },
+{ 0xe8 , 0x0425 },
+{ 0xe9 , 0x0418 },
+{ 0xea , 0x0419 },
+{ 0xeb , 0x041A },
+{ 0xec , 0x041B },
+{ 0xed , 0x041C },
+{ 0xee , 0x041D },
+{ 0xef , 0x041E },
+{ 0xf0 , 0x041F },
+{ 0xf1 , 0x042F },
+{ 0xf2 , 0x0420 },
+{ 0xf3 , 0x0421 },
+{ 0xf4 , 0x0422 },
+{ 0xf5 , 0x0423 },
+{ 0xf6 , 0x0416 },
+{ 0xf7 , 0x0412 },
+{ 0xf8 , 0x042C },
+{ 0xf9 , 0x042B },
+{ 0xfa , 0x0417 },
+{ 0xfb , 0x0428 },
+{ 0xfc , 0x042D },
+{ 0xfd , 0x0429 },
+{ 0xfe , 0x0427 },
+{ 0xff , 0x042A },
diff --git a/extra/locale/charmaps/TIS-620.pairs b/extra/locale/charmaps/TIS-620.pairs
new file mode 100644
index 000000000..618a3be98
--- /dev/null
+++ b/extra/locale/charmaps/TIS-620.pairs
@@ -0,0 +1,215 @@
+{ 0x00 , 0x0000 },
+{ 0x01 , 0x0001 },
+{ 0x02 , 0x0002 },
+{ 0x03 , 0x0003 },
+{ 0x04 , 0x0004 },
+{ 0x05 , 0x0005 },
+{ 0x06 , 0x0006 },
+{ 0x07 , 0x0007 },
+{ 0x08 , 0x0008 },
+{ 0x09 , 0x0009 },
+{ 0x0a , 0x000A },
+{ 0x0b , 0x000B },
+{ 0x0c , 0x000C },
+{ 0x0d , 0x000D },
+{ 0x0e , 0x000E },
+{ 0x0f , 0x000F },
+{ 0x10 , 0x0010 },
+{ 0x11 , 0x0011 },
+{ 0x12 , 0x0012 },
+{ 0x13 , 0x0013 },
+{ 0x14 , 0x0014 },
+{ 0x15 , 0x0015 },
+{ 0x16 , 0x0016 },
+{ 0x17 , 0x0017 },
+{ 0x18 , 0x0018 },
+{ 0x19 , 0x0019 },
+{ 0x1a , 0x001A },
+{ 0x1b , 0x001B },
+{ 0x1c , 0x001C },
+{ 0x1d , 0x001D },
+{ 0x1e , 0x001E },
+{ 0x1f , 0x001F },
+{ 0x20 , 0x0020 },
+{ 0x21 , 0x0021 },
+{ 0x22 , 0x0022 },
+{ 0x23 , 0x0023 },
+{ 0x24 , 0x0024 },
+{ 0x25 , 0x0025 },
+{ 0x26 , 0x0026 },
+{ 0x27 , 0x0027 },
+{ 0x28 , 0x0028 },
+{ 0x29 , 0x0029 },
+{ 0x2a , 0x002A },
+{ 0x2b , 0x002B },
+{ 0x2c , 0x002C },
+{ 0x2d , 0x002D },
+{ 0x2e , 0x002E },
+{ 0x2f , 0x002F },
+{ 0x30 , 0x0030 },
+{ 0x31 , 0x0031 },
+{ 0x32 , 0x0032 },
+{ 0x33 , 0x0033 },
+{ 0x34 , 0x0034 },
+{ 0x35 , 0x0035 },
+{ 0x36 , 0x0036 },
+{ 0x37 , 0x0037 },
+{ 0x38 , 0x0038 },
+{ 0x39 , 0x0039 },
+{ 0x3a , 0x003A },
+{ 0x3b , 0x003B },
+{ 0x3c , 0x003C },
+{ 0x3d , 0x003D },
+{ 0x3e , 0x003E },
+{ 0x3f , 0x003F },
+{ 0x40 , 0x0040 },
+{ 0x41 , 0x0041 },
+{ 0x42 , 0x0042 },
+{ 0x43 , 0x0043 },
+{ 0x44 , 0x0044 },
+{ 0x45 , 0x0045 },
+{ 0x46 , 0x0046 },
+{ 0x47 , 0x0047 },
+{ 0x48 , 0x0048 },
+{ 0x49 , 0x0049 },
+{ 0x4a , 0x004A },
+{ 0x4b , 0x004B },
+{ 0x4c , 0x004C },
+{ 0x4d , 0x004D },
+{ 0x4e , 0x004E },
+{ 0x4f , 0x004F },
+{ 0x50 , 0x0050 },
+{ 0x51 , 0x0051 },
+{ 0x52 , 0x0052 },
+{ 0x53 , 0x0053 },
+{ 0x54 , 0x0054 },
+{ 0x55 , 0x0055 },
+{ 0x56 , 0x0056 },
+{ 0x57 , 0x0057 },
+{ 0x58 , 0x0058 },
+{ 0x59 , 0x0059 },
+{ 0x5a , 0x005A },
+{ 0x5b , 0x005B },
+{ 0x5c , 0x005C },
+{ 0x5d , 0x005D },
+{ 0x5e , 0x005E },
+{ 0x5f , 0x005F },
+{ 0x60 , 0x0060 },
+{ 0x61 , 0x0061 },
+{ 0x62 , 0x0062 },
+{ 0x63 , 0x0063 },
+{ 0x64 , 0x0064 },
+{ 0x65 , 0x0065 },
+{ 0x66 , 0x0066 },
+{ 0x67 , 0x0067 },
+{ 0x68 , 0x0068 },
+{ 0x69 , 0x0069 },
+{ 0x6a , 0x006A },
+{ 0x6b , 0x006B },
+{ 0x6c , 0x006C },
+{ 0x6d , 0x006D },
+{ 0x6e , 0x006E },
+{ 0x6f , 0x006F },
+{ 0x70 , 0x0070 },
+{ 0x71 , 0x0071 },
+{ 0x72 , 0x0072 },
+{ 0x73 , 0x0073 },
+{ 0x74 , 0x0074 },
+{ 0x75 , 0x0075 },
+{ 0x76 , 0x0076 },
+{ 0x77 , 0x0077 },
+{ 0x78 , 0x0078 },
+{ 0x79 , 0x0079 },
+{ 0x7a , 0x007A },
+{ 0x7b , 0x007B },
+{ 0x7c , 0x007C },
+{ 0x7d , 0x007D },
+{ 0x7e , 0x007E },
+{ 0x7f , 0x007F },
+{ 0xa1 , 0x0E01 },
+{ 0xa2 , 0x0E02 },
+{ 0xa3 , 0x0E03 },
+{ 0xa4 , 0x0E04 },
+{ 0xa5 , 0x0E05 },
+{ 0xa6 , 0x0E06 },
+{ 0xa7 , 0x0E07 },
+{ 0xa8 , 0x0E08 },
+{ 0xa9 , 0x0E09 },
+{ 0xaa , 0x0E0A },
+{ 0xab , 0x0E0B },
+{ 0xac , 0x0E0C },
+{ 0xad , 0x0E0D },
+{ 0xae , 0x0E0E },
+{ 0xaf , 0x0E0F },
+{ 0xb0 , 0x0E10 },
+{ 0xb1 , 0x0E11 },
+{ 0xb2 , 0x0E12 },
+{ 0xb3 , 0x0E13 },
+{ 0xb4 , 0x0E14 },
+{ 0xb5 , 0x0E15 },
+{ 0xb6 , 0x0E16 },
+{ 0xb7 , 0x0E17 },
+{ 0xb8 , 0x0E18 },
+{ 0xb9 , 0x0E19 },
+{ 0xba , 0x0E1A },
+{ 0xbb , 0x0E1B },
+{ 0xbc , 0x0E1C },
+{ 0xbd , 0x0E1D },
+{ 0xbe , 0x0E1E },
+{ 0xbf , 0x0E1F },
+{ 0xc0 , 0x0E20 },
+{ 0xc1 , 0x0E21 },
+{ 0xc2 , 0x0E22 },
+{ 0xc3 , 0x0E23 },
+{ 0xc4 , 0x0E24 },
+{ 0xc5 , 0x0E25 },
+{ 0xc6 , 0x0E26 },
+{ 0xc7 , 0x0E27 },
+{ 0xc8 , 0x0E28 },
+{ 0xc9 , 0x0E29 },
+{ 0xca , 0x0E2A },
+{ 0xcb , 0x0E2B },
+{ 0xcc , 0x0E2C },
+{ 0xcd , 0x0E2D },
+{ 0xce , 0x0E2E },
+{ 0xcf , 0x0E2F },
+{ 0xd0 , 0x0E30 },
+{ 0xd1 , 0x0E31 },
+{ 0xd2 , 0x0E32 },
+{ 0xd3 , 0x0E33 },
+{ 0xd4 , 0x0E34 },
+{ 0xd5 , 0x0E35 },
+{ 0xd6 , 0x0E36 },
+{ 0xd7 , 0x0E37 },
+{ 0xd8 , 0x0E38 },
+{ 0xd9 , 0x0E39 },
+{ 0xda , 0x0E3A },
+{ 0xdf , 0x0E3F },
+{ 0xe0 , 0x0E40 },
+{ 0xe1 , 0x0E41 },
+{ 0xe2 , 0x0E42 },
+{ 0xe3 , 0x0E43 },
+{ 0xe4 , 0x0E44 },
+{ 0xe5 , 0x0E45 },
+{ 0xe6 , 0x0E46 },
+{ 0xe7 , 0x0E47 },
+{ 0xe8 , 0x0E48 },
+{ 0xe9 , 0x0E49 },
+{ 0xea , 0x0E4A },
+{ 0xeb , 0x0E4B },
+{ 0xec , 0x0E4C },
+{ 0xed , 0x0E4D },
+{ 0xee , 0x0E4E },
+{ 0xef , 0x0E4F },
+{ 0xf0 , 0x0E50 },
+{ 0xf1 , 0x0E51 },
+{ 0xf2 , 0x0E52 },
+{ 0xf3 , 0x0E53 },
+{ 0xf4 , 0x0E54 },
+{ 0xf5 , 0x0E55 },
+{ 0xf6 , 0x0E56 },
+{ 0xf7 , 0x0E57 },
+{ 0xf8 , 0x0E58 },
+{ 0xf9 , 0x0E59 },
+{ 0xfa , 0x0E5A },
+{ 0xfb , 0x0E5B },
diff --git a/extra/locale/gen_collate_from_glibc.c b/extra/locale/gen_collate_from_glibc.c
deleted file mode 100644
index f6d12ba9c..000000000
--- a/extra/locale/gen_collate_from_glibc.c
+++ /dev/null
@@ -1,207 +0,0 @@
-/*
- * Generator collate table from glibc special for Uclibc.
- * Author Vladimir Oleynik. vodz@usa.net (c) 2001
- *
- * Require setuped work non-C LC_COLLATE
- * This programm created ./LOCALE/LC_COLLATE file for Uclibc
- * setlocale() and strcoll().
- * Without argument this programm used setlocale(LC_COLLATE, "") -
- * equivalent result setlocale(LC_COLLATE, getenv("LC_XXX"))
- *
- * Also, this programm have russian koi8 collate for test
- * working Uclibc ;-)
- *
- */
-
-#include <ctype.h>
-#include <string.h>
-#include <locale.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <unistd.h>
-#include <sys/stat.h> /* mkdir() */
-#include <errno.h>
-
-
-/* For strong test russian locale LC_COLLATE="ru_RU.KOI8-R" */
-static const unsigned char koi8_weights[256] = {
- 0, 99, 100, 101, 102, 103, 104, 105,
-106, 2, 5, 3, 6, 4, 107, 108,
-109, 110, 111, 112, 113, 114, 115, 116,
-117, 118, 119, 120, 121, 122, 123, 124,
- 1, 12, 21, 34, 30, 35, 33, 20,
- 22, 23, 31, 36, 9, 8, 15, 14,
-127, 128, 129, 131, 132, 133, 134, 135,
-136, 137, 11, 10, 38, 40, 42, 13,
- 29, 138, 140, 142, 144, 146, 148, 150,
-152, 154, 156, 158, 160, 162, 164, 166,
-168, 170, 172, 174, 176, 178, 180, 182,
-184, 186, 188, 24, 32, 25, 17, 7,
- 16, 139, 141, 143, 145, 147, 149, 151,
-153, 155, 157, 159, 161, 163, 165, 167,
-169, 171, 173, 175, 177, 179, 181, 183,
-185, 187, 189, 26, 43, 27, 18, 125,
- 50, 52, 54, 58, 62, 66, 70, 74,
- 78, 82, 86, 90, 91, 92, 93, 94,
- 95, 96, 97, 48, 98, 45, 46, 47,
- 39, 41, 126, 49, 44, 130, 19, 37,
- 51, 53, 55, 203, 56, 57, 59, 60,
- 61, 63, 64, 65, 67, 68, 69, 71,
- 72, 73, 75, 202, 76, 77, 79, 80,
- 81, 83, 84, 85, 87, 88, 89, 28,
-253, 191, 193, 237, 199, 201, 233, 197,
-235, 209, 211, 213, 215, 217, 219, 221,
-223, 255, 225, 227, 229, 231, 205, 195,
-249, 247, 207, 241, 251, 243, 239, 245,
-252, 190, 192, 236, 198, 200, 232, 196,
-234, 208, 210, 212, 214, 216, 218, 220,
-222, 254, 224, 226, 228, 230, 204, 194,
-248, 246, 206, 240, 250, 242, 238, 244
-};
-
-int gen_weights(const char *collate)
-{
- int weights[256];
- int i,j;
- char probe_str1[2];
- char probe_str2[2];
- char print_buf[16];
- int retcode = 0;
- unsigned char out_weights[256];
- FILE *out;
-
- memset(weights, 0, sizeof(weights));
- probe_str1[1]=probe_str2[1]=0;
-
- for(i=0; i<256; i++) {
- probe_str1[0] = i;
- for(j=0; j<256; j++) {
- probe_str2[0] = j;
- if(strcoll(probe_str1, probe_str2)>0) {
- weights[i]++;
- if(i==j) {
- fprintf(stderr, "\
-\nWarning! c1=%d == c2, but strcoll returned greater zero\n", i);
- retcode++;
- }
- }
- }
- }
- for(i=0; i<256; ) {
- if(isprint(i))
- sprintf(print_buf, " '%c'", i);
- else {
- if(i=='\0')
- strcpy(print_buf, "'\\0'");
- else if(i=='\a')
- strcpy(print_buf, "'\\a'");
- else if(i=='\b')
- strcpy(print_buf, "'\\b'");
- else if(i=='\f')
- strcpy(print_buf, "'\\f'");
- else if(i=='\r')
- strcpy(print_buf, "'\\r'");
- else if(i=='\t')
- strcpy(print_buf, "'\\t'");
- else sprintf(print_buf, " x%02X", i);
- }
- printf("weights[%s] = %3d ", print_buf, weights[i]);
- i++;
- if( (i%4) == 0)
- printf("\n");
- }
-
- for(i=0; i<256; i++) {
- if(weights[i]<0 || weights[i]>=256) {
- fprintf(stderr, "Hmm, weights[%d]=%d\n", i, weights[i]);
- retcode++;
- }
- for(j=0; j<256; j++) {
- if(i==j)
- continue;
- if(weights[i]==weights[j]) {
- fprintf(stderr, "\
-Warning! c1=%d c2=%d and strcoll returned equivalent weight\n", i, j);
- retcode++;
- }
- }
- }
- if(retcode)
- return 1;
-
- if(strcasecmp(collate, "ru_RU.KOI8-R")==0 ||
- strcmp(collate, "ru_RU")==0 ||
- strcmp(collate, "koi8-r")==0) {
- for(i=0; i<256; i++)
- if(weights[i]!=koi8_weights[i]) {
- fprintf(stderr, "\
-Error koi8-r collate compare, glibc weights[%d]=%d but current generation %d\n",
- i, koi8_weights[i], weights[i]);
- retcode++;
- }
- if(retcode)
- return 5;
- }
- for(i=0; i<256; i++)
- out_weights[i] = weights[i];
- out = fopen("LC_COLLATE", "w");
- if(out == NULL) {
- fprintf(stderr, "Can`t create ./%s/LC_COLLATE file\n", collate);
- return 10;
- }
- if(fwrite(out_weights, 1, 256, out)!=256) {
- fprintf(stderr, "IO error in process write ./%s/LC_COLLATE file\n", collate);
- return 11;
- }
- return 0;
-}
-
-int main(int argc, char **argv)
-{
- char *locale;
- char *slr;
- char *collate;
-
- if(argc<1 || argc>2) {
- fprintf(stderr, "Usage: %s [locale]\n", argv[0]);
- }
- locale = argc==1 ? "" : argv[1];
-
- collate = setlocale(LC_COLLATE, locale);
- fprintf(stderr, "setlocale(LC_COLLATE, \"%s\") returned %s\n", locale, collate);
- if(collate==0) {
- fprintf(stderr, "Can`t set LC_COLLATE\n");
- return 2;
- }
- if(strcmp(collate, "C")==0) {
- fprintf(stderr, "\
-LC_COLLATE=\"C\" is trivial and not interesting for this programm\n");
- return 3;
- }
- slr = setlocale(LC_CTYPE, locale);
- fprintf(stderr, "setlocale(LC_CTYPE, \"%s\") returned %s\n", locale, slr);
- if(slr==0) {
- slr = setlocale(LC_CTYPE, "POSIX");
- if(slr==0) {
- fprintf(stderr, "Hmm, can`t set setlocale(LC_CTYPE, \"POSIX\")\n");
- return 4;
- }
- }
- if(mkdir(collate, 0755)!=0 && errno!=EEXIST) {
- fprintf(stderr, "Can`t make directory %s\n", collate);
- return 6;
- }
- if(chdir(collate)) {
- fprintf(stderr, "Hmm, can`t change directory to %s\n", collate);
- return 7;
- }
- if(gen_weights(collate)) {
- if(chdir("..")) {
- fprintf(stderr, "Hmm, can`t change to current directory\n");
- return 7;
- }
- rmdir(collate);
- return 1;
- }
- return 0;
-}
diff --git a/extra/locale/gen_ctype_from_glibc.c b/extra/locale/gen_ctype_from_glibc.c
deleted file mode 100644
index 0488048cd..000000000
--- a/extra/locale/gen_ctype_from_glibc.c
+++ /dev/null
@@ -1,277 +0,0 @@
-/*
- * Generator locale ctype tables
- * You must have already setuped locale for worked libc (libc5 or glibc)
- *
- * This programm scan /usr/share/locale directories and write
- * ./LOCALE/LC_CTYPE files for system with uclibc
- *
- * Written by Vladimir Oleynik <vodz@usa.net> 2001
- * Base on ideas Nickolay Saukh <nms@ussr.EU.net>
- *
- */
-
-#include <locale.h>
-#include <ctype.h>
-#include <stdio.h>
-#include <dirent.h>
-#include <sys/stat.h>
-#include <stdlib.h>
-#include <string.h>
-#include <getopt.h>
-#include <unistd.h>
-#include <errno.h>
-
-#include "../../libc/misc/locale/_locale.h"
-
-
-#define DEFAULT_LOCALE_DIR "/usr/share/locale/"
-
-#define DEF_OUT_NAME "LC_CTYPE"
-
-#define CURRENT_SUPPORT_MAX (LOCALE_BUF_SIZE/2)
-
-
-unsigned char x2type[CURRENT_SUPPORT_MAX];
-
-unsigned char x2trans[CURRENT_SUPPORT_MAX];
-
-
-
-int
-write_out (outname)
- unsigned char *outname;
-{
- FILE *ofp = fopen (outname, "w");
-
- if (ofp == NULL) {
- fprintf (stderr, "Can`t write `%s\n", outname);
- return 1;
- }
-
- fwrite (x2type, sizeof (x2type), 1, ofp);
- fwrite (x2trans, sizeof (x2trans), 1, ofp);
- fclose (ofp);
- return 0;
-}
-
-
-typedef struct bbits_ {
- int bbits;
- char *bb_name;
- } bbits_t;
-
-
-bbits_t basic_bits[] =
-{
- {ISprint , "ISprint" },
- {ISupper , "ISupper" },
- {ISlower , "ISlower" },
- {IScntrl , "IScntrl" },
- {ISspace , "ISspace" },
- {ISpunct , "ISpunct" },
- {ISalpha , "ISalpha" },
- {ISxdigit, "ISxdigit"},
- {0, NULL}
-};
-
-
-void
-ctab_out (char *oun)
-{
- int i;
- char *outname;
- FILE *fout;
-
- outname = alloca(strlen(oun)+strlen("ctype_.c")+1);
- if(outname==0) {
- perror("");
- exit(1);
- }
- strcpy(outname, "ctype_");
- strcat(outname, oun);
- strcat(outname, ".c");
-
- fout = fopen (outname, "w");
-
- if (fout == NULL)
- {
- perror ("");
- return;
- }
-
- fprintf (fout, "const unsigned char _uc_ctype_b_C[LOCALE_BUF_SIZE] = {\n");
-
- for (i = 0; i < CURRENT_SUPPORT_MAX; i++)
- {
- if(i)
- fprintf (fout, ",\n");
- fprintf (fout, "\t/* 0x%02x, %d, 0%o */\t", i, i, i);
- if (x2type[i])
- {
- int dirty = 0;
- bbits_t *tb = basic_bits;
-
- while (tb->bbits)
- {
- if (x2type[i] & tb->bbits)
- {
- if (dirty)
- fputs ("|", fout);
- fputs (tb->bb_name, fout);
- dirty = 1;
- }
- tb++;
- }
- }
- else
- fputs ("0", fout);
- }
-
- fputs (",\n\n", fout);
-
- fprintf (fout, "/* _uc_ctype_trans_C */\n\n");
-
- for (i = 0; i < CURRENT_SUPPORT_MAX; i++)
- {
- if(i)
- fprintf (fout, ",\n");
- fprintf (fout, "\t/* 0x%02x, %d, 0%o */\t0x%02x", i, i, i, x2trans[i]);
- }
- fputs ("\n};\n", fout);
-
- (void) fclose (fout);
-}
-
-
-int
-main (int argc, char *argv[])
-{
- int i,l;
- char *outname = DEF_OUT_NAME;
- char *search_dir = DEFAULT_LOCALE_DIR;
- char *full_path = 0;
- DIR *dir;
- struct dirent *next;
- char *t;
- int err=0;
- char *ln;
- int generate_c_code = 1;
-
- while ((i = getopt (argc, argv, "d:o:c")) != EOF) {
- switch (i) {
- case 'o':
- outname = optarg;
- break;
- case 'd':
- search_dir = optarg;
- break;
- case 'c':
- generate_c_code = 0;
- break;
- default:
- optind = i = -1;
- break;
- }
- if(i<0)
- break;
- }
-
- if (argc > optind) {
- fprintf (stderr,
-"Usage: %s [-d search_dir] [-o output_name] [-c]\n\
-Defaults:\n\
- search_dir : " DEFAULT_LOCALE_DIR "\n\
- output_name : " DEF_OUT_NAME "\n\
- -c : no generate c-code for other locale exept C-locale.\n"
- , argv[0]);
- return 3;
- }
-
- l = strlen(search_dir);
- if(l == 0) {
- search_dir = "./";
- l = 2;
- } else {
- if(search_dir[l-1]!='/') {
-
- t = malloc(l+2);
- if(t==0) {
- fprintf (stderr, "Can`t get %d bytes memory\n", l+2);
- return 4;
- }
- search_dir = strcat(strcpy(t, search_dir), "/");
- l++;
- }
- }
-
- dir = opendir(search_dir);
- if (!dir) {
- fprintf (stderr, "Can`t open directory `%s' load all locales\n", search_dir);
- return 2;
- }
-
- while ((next = readdir(dir)) != NULL) {
-
- struct stat st;
- if(strcmp(next->d_name, ".")==0)
- ln = "C";
- else if(strcmp(next->d_name, "..")==0)
- continue;
- else {
- ln = next->d_name;
- full_path = realloc(full_path, l+strlen(ln)+1);
- strcat(strcpy(full_path, search_dir), ln);
- if (lstat(full_path, &st) < 0)
- continue;
- if(S_ISDIR(st.st_mode)==0)
- continue;
- }
- t = setlocale(LC_CTYPE, ln);
- printf("setlocale(LC_CTYPE, %s) returned %s\n", ln, t);
- if(t==0)
- continue;
- if(mkdir(ln, 0755)!=0 && errno!=EEXIST) {
- fprintf(stderr, "Can`t create directory `%s'\n", ln);
- continue;
- }
- if(chdir(ln)) {
- fprintf(stderr, "Can`t change directory to `%s'\n", ln);
- continue;
- }
-
- for (i = 0; i < CURRENT_SUPPORT_MAX; i++) {
-
- if(isprint(i))
- x2type[i] |= ISprint;
- if(isupper(i))
- x2type[i] |= ISupper;
- if(islower(i))
- x2type[i] |= ISlower;
- if(isspace(i))
- x2type[i] |= ISspace;
- if(isalpha(i))
- x2type[i] |= ISalpha;
- if(iscntrl(i))
- x2type[i] |= IScntrl;
- if(ispunct(i))
- x2type[i] |= ISpunct;
- if(isxdigit(i))
- x2type[i] |= ISxdigit;
- x2trans[i] = i;
- if(toupper(x2trans[i]) != x2trans[i])
- x2trans[i] = toupper(x2trans[i]);
- else if(tolower(x2trans[i]) != x2trans[i])
- x2trans[i] = tolower(x2trans[i]);
- }
- err += write_out(outname);
- if(chdir("..")) {
- fprintf(stderr, "Can`t change directory to `..'\n");
- return 1;
- }
- if(strcmp(ln, "C")==0 || generate_c_code!=0)
- ctab_out(ln);
- for (i = 0; i < CURRENT_SUPPORT_MAX; i++)
- x2type[i] = x2trans[i] = 0;
- }
- return err ? 1 : 0;
-}
diff --git a/extra/locale/gen_locale.c b/extra/locale/gen_locale.c
new file mode 100644
index 000000000..9b2576e50
--- /dev/null
+++ b/extra/locale/gen_locale.c
@@ -0,0 +1,1254 @@
+#define _GNU_SOURCE
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <ctype.h>
+#include <assert.h>
+#include <locale.h>
+#include <langinfo.h>
+#include <nl_types.h>
+#include <stdint.h>
+
+#define __CTYPE_HAS_8_BIT_LOCALES
+#include "c8tables.h"
+
+
+#define CATEGORIES 6
+
+/* must agree with ordering of gen_mmap! */
+static const unsigned char *lc_names[] = {
+ "LC_CTYPE",
+ "LC_NUMERIC",
+ "LC_MONETARY",
+ "LC_TIME",
+ "LC_COLLATE",
+ "LC_MESSAGES",
+#if CATEGORIES == 12
+ "LC_PAPER",
+ "LC_NAME",
+ "LC_ADDRESS",
+ "LC_TELEPHONE",
+ "LC_MEASUREMENT",
+ "LC_IDENTIFICATION",
+#elif CATEGORIES != 6
+#error unsupported CATEGORIES value!
+#endif
+};
+
+
+
+typedef struct {
+ char *glibc_name;
+ char name[5];
+ char dot_cs; /* 0 if no codeset specified */
+ char cs;
+ unsigned char idx_name;
+ unsigned char lc_time_row;
+ unsigned char lc_numeric_row;
+ unsigned char lc_monetary_row;
+ unsigned char lc_messages_row;
+#if CATEGORIES != 6
+#error unsupported CATEGORIES value
+#endif
+} locale_entry;
+
+static void read_at_mappings(void);
+static void read_enable_disable(void);
+static void read_locale_list(void);
+
+static int find_codeset_num(const char *cs);
+static int find_at_string_num(const char *as);
+static int le_cmp(const void *, const void *);
+static void dump_table8(const char *name, const char *tbl, int len);
+static void dump_table8c(const char *name, const char *tbl, int len);
+static void dump_table16(const char *name, const int *tbl, int len);
+
+static void do_lc_time(void);
+static void do_lc_numeric(void);
+static void do_lc_monetary(void);
+
+static void do_lc_messages(void);
+
+
+static FILE *fp;
+static FILE *ofp;
+static char line_buf[80];
+static char at_mappings[256];
+static char at_mapto[256];
+static char at_strings[1024];
+static char *at_strings_end;
+static locale_entry locales[700];
+static char glibc_locale_names[60000];
+
+static int num_locales;
+
+static int default_utf8;
+static int default_8bit;
+
+static int total_size;
+static int null_count;
+
+static void do_locale_names(void)
+{
+ /* "C" locale name is handled specially by the setlocale code. */
+ int uniq = 0;
+ int i;
+
+ if (num_locales <= 1) {
+/* printf("error - only C locale?\n"); */
+/* exit(EXIT_FAILURE); */
+ fprintf(ofp, "static const unsigned char __locales[%d];\n", (3 + CATEGORIES));
+ fprintf(ofp, "static const unsigned char __locale_names5[5];\n");
+ } else {
+ fprintf(ofp, "#define CATEGORIES\t\t\t%d\n", CATEGORIES);
+ fprintf(ofp, "#define WIDTH_LOCALES\t\t\t%d\n", 3+CATEGORIES);
+ fprintf(ofp, "#define NUM_LOCALES\t\t\t%d\n", num_locales);
+ fprintf(ofp, "static const unsigned char __locales[%d] = {\n",
+ (num_locales) * (3 + CATEGORIES));
+ for (i=0 ; i < num_locales ; i++) {
+ if (memcmp(locales[i].name, locales[i-1].name, 5) != 0) {
+ locales[i].idx_name = uniq;
+ ++uniq;
+ } else {
+ locales[i].idx_name = uniq - 1;
+ }
+ fprintf(ofp, "\t%#4x, ", (int)((unsigned char) locales[i].idx_name));
+ fprintf(ofp, "\t%#4x, ", (int)((unsigned char) locales[i].dot_cs));
+ fprintf(ofp, "\t%#4x, ", (int)((unsigned char) locales[i].cs));
+ /* lc_ctype would store translit flags and turkish up/low flag. */
+ fprintf(ofp, "%#4x, ", 0); /* place holder for lc_ctype */
+ fprintf(ofp, "%#4x, ", (int)((unsigned char) locales[i].lc_numeric_row));
+ fprintf(ofp, "%#4x, ", (int)((unsigned char) locales[i].lc_monetary_row));
+ fprintf(ofp, "%#4x, ", (int)((unsigned char) locales[i].lc_time_row));
+ fprintf(ofp, "%#4x, ", 0); /* place holder for lc_collate */
+ fprintf(ofp, "%#4x, ", (int)((unsigned char) locales[i].lc_messages_row));
+ fprintf(ofp, "\t/* %s */\n", locales[i].glibc_name);
+ }
+ fprintf(ofp, "};\n\n");
+
+ fprintf(ofp, "#define NUM_LOCALE_NAMES\t\t%d\n", uniq );
+ fprintf(ofp, "static const unsigned char __locale_names5[%d] = \n\t", uniq * 5);
+ uniq = 0;
+ for (i=1 ; i < num_locales ; i++) {
+ if (memcmp(locales[i].name, locales[i-1].name, 5) != 0) {
+ fprintf(ofp, "\"%5.5s\" ", locales[i].name);
+ ++uniq;
+ if ((uniq % 8) == 0) {
+ fprintf(ofp, "\n\t");
+ }
+ }
+ }
+ fprintf(ofp,";\n\n");
+
+ if (at_strings_end > at_strings) {
+ int i, j;
+ char *p;
+ i = 0;
+ p = at_strings;
+ while (*p) {
+ ++i;
+ p += 1 + (unsigned char) *p;
+ }
+ /* len, char, string\0 */
+ fprintf(ofp, "#define LOCALE_AT_MODIFIERS_LENGTH\t\t%d\n",
+ i + (at_strings_end - at_strings));
+ fprintf(ofp, "static const unsigned char __locale_at_modifiers[%d] = {",
+ i + (at_strings_end - at_strings));
+ i = 0;
+ p = at_strings;
+ while (*p) {
+ fprintf(ofp, "\n\t%4d, '%c',",
+ (unsigned char) *p, /* len of string\0 */
+ at_mapto[i]);
+ for (j=1 ; j < ((unsigned char) *p) ; j++) {
+ fprintf(ofp, " '%c',", p[j]);
+ }
+ fprintf(ofp, " 0,");
+ ++i;
+ p += 1 + (unsigned char) *p;
+ }
+ fprintf(ofp, "\n};\n\n");
+ }
+
+ {
+ int pos[CATEGORIES];
+ pos[0] = CATEGORIES;
+ for (i=0 ; i < CATEGORIES ; i++) {
+ fprintf(ofp, "#define __%s\t\t%d\n", lc_names[i], i);
+ if (i + 1 < CATEGORIES) {
+ pos[i+1] = 1 + strlen(lc_names[i]) + pos[i];
+ }
+ }
+ if (pos[CATEGORIES-1] > 255) {
+ printf("error - lc_names is too big (%d)\n", pos[CATEGORIES-1]);
+ exit(EXIT_FAILURE);
+ }
+ fprintf(ofp, "#define __LC_ALL\t\t%d\n\n", i);
+
+ fprintf(ofp, "#define lc_names_LEN\t\t%d\n",
+ pos[CATEGORIES-1] + strlen(lc_names[CATEGORIES-1]) + 1);
+ total_size += pos[CATEGORIES-1] + strlen(lc_names[CATEGORIES-1]) + 1;
+
+ fprintf(ofp, "static unsigned const char lc_names[%d] =\n",
+ pos[CATEGORIES-1] + strlen(lc_names[CATEGORIES-1]) + 1);
+ fprintf(ofp, "\t\"");
+ for (i=0 ; i < CATEGORIES ; i++) {
+ fprintf(ofp, "\\x%02x", (unsigned char) pos[i]);
+ }
+ fprintf(ofp, "\"");
+ for (i=0 ; i < CATEGORIES ; i++) {
+ fprintf(ofp, "\n\t\"%s\\0\"", lc_names[i]);
+ }
+ fprintf(ofp, ";\n\n");
+ }
+
+ printf("locale data = %d name data = %d for %d uniq\n",
+ num_locales * (3 + CATEGORIES), uniq * 5, uniq);
+
+ total_size += num_locales * (3 + CATEGORIES) + uniq * 5;
+ }
+
+}
+
+int main(int argc, char **argv)
+{
+ if ((argc != 2) || (!(fp = fopen(*++argv, "r")))) {
+ printf("error: missing filename or file!\n");
+ return EXIT_FAILURE;
+ }
+
+ at_strings_end = at_strings;
+
+ read_at_mappings();
+ read_enable_disable();
+ read_locale_list();
+
+ fclose(fp);
+
+ /* handle C locale specially */
+ qsort(locales+1, num_locales-1, sizeof(locale_entry), le_cmp);
+
+#if 0
+ for (i=0 ; i < num_locales ; i++) {
+ printf("locale: %5.5s %2d %2d %s\n",
+ locales[i].name,
+ locales[i].cs,
+ locales[i].dot_cs,
+ locales[i].glibc_name
+ );
+ }
+#endif
+
+ if (!(ofp = fopen("locale_tables.h", "w"))) {
+ printf("error: can not open locale_tables.h for writing!\n");
+ return EXIT_FAILURE;
+ }
+
+ do_lc_time();
+ do_lc_numeric();
+ do_lc_monetary();
+ do_lc_messages();
+
+ do_locale_names();
+
+ fclose(ofp);
+
+ printf("total data size = %d\n", total_size);
+ printf("null count = %d\n", null_count);
+
+ return EXIT_SUCCESS;
+}
+
+static char *idx[10000];
+static char buf[100000];
+static char *last;
+static int uniq;
+
+static int addstring(const char *s)
+{
+ int j;
+ size_t l;
+
+ if (!s) {
+ ++null_count;
+ return 0;
+ }
+
+ for (j=0 ; j < uniq ; j++) {
+ if (!strcmp(s, idx[j])) {
+ return idx[j] - buf;
+ }
+ }
+ if (uniq >= sizeof(idx)) {
+ printf("too many uniq strings!\n");
+ exit(EXIT_FAILURE);
+ }
+ l = strlen(s) + 1;
+ if (last + l >= buf + sizeof(buf)) {
+ printf("need to increase size of buf!\n");
+ exit(EXIT_FAILURE);
+ }
+
+ idx[uniq] = last;
+ ++uniq;
+ strcpy(last, s);
+ last += l;
+ return idx[uniq - 1] - buf;
+}
+
+static int buf16[50*256];
+
+
+static int lc_time_item[50][256];
+static int lc_time_count[50];
+static unsigned char lc_time_uniq_50[700][50];
+static int lc_time_uniq;
+
+#define DO_NL_S(X) lc_time_S(X, k++)
+
+static void lc_time_S(int X, int k)
+{
+ int j, m;
+ j = addstring(nl_langinfo(X));
+ for (m=0 ; m < lc_time_count[k] ; m++) {
+ if (lc_time_item[k][m] == j) {
+ break;
+ }
+ }
+ if (m == lc_time_count[k]) { /* new for this nl_item */
+ if (m > 255) {
+ printf("too many nl_item %d entries in lc_time\n", k);
+ exit(EXIT_FAILURE);
+ }
+ lc_time_item[k][m] = j;
+ ++lc_time_count[k];
+ }
+/* printf("\\x%02x", m); */
+ lc_time_uniq_50[lc_time_uniq][k] = m;
+}
+
+static void do_lc_time(void)
+{
+ int i, k, m;
+
+ last = buf+1;
+ uniq = 1;
+ *buf = 0;
+ *idx = buf;
+
+/* printf("processing lc_time..."); */
+ for (i=0 ; i < num_locales ; i++) {
+ k = 0;
+
+/* printf(" %d", i); fflush(stdout); */
+ if (!setlocale(LC_ALL, locales[i].glibc_name)) {
+ printf("setlocale(LC_ALL,%s) failed!\n",
+ locales[i].glibc_name);
+ }
+
+ DO_NL_S(ABDAY_1);
+ DO_NL_S(ABDAY_2);
+ DO_NL_S(ABDAY_3);
+ DO_NL_S(ABDAY_4);
+ DO_NL_S(ABDAY_5);
+ DO_NL_S(ABDAY_6);
+ DO_NL_S(ABDAY_7);
+
+ DO_NL_S(DAY_1);
+ DO_NL_S(DAY_2);
+ DO_NL_S(DAY_3);
+ DO_NL_S(DAY_4);
+ DO_NL_S(DAY_5);
+ DO_NL_S(DAY_6);
+ DO_NL_S(DAY_7);
+
+ DO_NL_S(ABMON_1);
+ DO_NL_S(ABMON_2);
+ DO_NL_S(ABMON_3);
+ DO_NL_S(ABMON_4);
+ DO_NL_S(ABMON_5);
+ DO_NL_S(ABMON_6);
+ DO_NL_S(ABMON_7);
+ DO_NL_S(ABMON_8);
+ DO_NL_S(ABMON_9);
+ DO_NL_S(ABMON_10);
+ DO_NL_S(ABMON_11);
+ DO_NL_S(ABMON_12);
+
+ DO_NL_S(MON_1);
+ DO_NL_S(MON_2);
+ DO_NL_S(MON_3);
+ DO_NL_S(MON_4);
+ DO_NL_S(MON_5);
+ DO_NL_S(MON_6);
+ DO_NL_S(MON_7);
+ DO_NL_S(MON_8);
+ DO_NL_S(MON_9);
+ DO_NL_S(MON_10);
+ DO_NL_S(MON_11);
+ DO_NL_S(MON_12);
+
+ DO_NL_S(AM_STR);
+ DO_NL_S(PM_STR);
+
+ DO_NL_S(D_T_FMT);
+ DO_NL_S(D_FMT);
+ DO_NL_S(T_FMT);
+ DO_NL_S(T_FMT_AMPM);
+ DO_NL_S(ERA);
+
+ DO_NL_S(ERA_YEAR); /* non SuSv3 */
+ DO_NL_S(ERA_D_FMT);
+ DO_NL_S(ALT_DIGITS);
+ DO_NL_S(ERA_D_T_FMT);
+ DO_NL_S(ERA_T_FMT);
+
+ if (k > 50) {
+ printf("error -- lc_time nl_item count > 50!\n");
+ exit(EXIT_FAILURE);
+ }
+
+ {
+ int r;
+ for (r=0 ; r < lc_time_uniq ; r++) {
+ if (!memcmp(lc_time_uniq_50[lc_time_uniq],
+ lc_time_uniq_50[r], 50)) {
+ break;
+ }
+ }
+ if (r == lc_time_uniq) { /* new locale row */
+ ++lc_time_uniq;
+ if (lc_time_uniq > 255) {
+ printf("too many unique lc_time rows!\n");
+ exit(EXIT_FAILURE);
+ }
+ }
+ locales[i].lc_time_row = r;
+ }
+/* printf("\n"); */
+ }
+/* printf(" done\n"); */
+
+ m = k = 0;
+ for (i=0 ; i < 50 ; i++) {
+ k += lc_time_count[i];
+ if (m < lc_time_count[i]) {
+ m = lc_time_count[i];
+ }
+ }
+ printf("buf-size=%d uniq=%d item_offsets=%d max=%d rows=%d\n",
+ (int)(last - buf), uniq, k, m, lc_time_uniq);
+/* printf("total = %d * 50 + 2 * (50 + %d) + %d = %d\n", */
+/* num_locales, k, (int)(last - buf), */
+/* num_locales*50 + 2*(50 + k) + (int)(last - buf)); */
+ printf("total = %d + %d * 50 + 2 * (50 + %d) + %d = %d\n",
+ num_locales, lc_time_uniq, k, (int)(last - buf),
+ i = num_locales + lc_time_uniq*50 + 2*(50 + k) + (int)(last - buf));
+ total_size += i;
+
+ dump_table8c("__lc_time_data", buf, (int)(last - buf));
+
+ for (i=0 ; i < lc_time_uniq ; i++) {
+ m = locales[i].lc_time_row;
+ for (k=0 ; k < 50 ; k++) {
+ buf[50*i + k] = (char)((unsigned char) lc_time_uniq_50[i][k]);
+ }
+ }
+ dump_table8("__lc_time_rows", buf, lc_time_uniq * 50);
+
+ buf16[0] =0;
+ for (i=0 ; i < 50 - 1 ; i++) {
+ buf16[i+1] = buf16[i] + lc_time_count[i];
+ }
+ dump_table16("__lc_time_item_offsets", buf16, 50);
+
+ m = 0;
+ for (k=0 ; k < 50 ; k++) {
+ for (i=0 ; i < lc_time_count[k] ; i++) {
+ buf16[m] = lc_time_item[k][i];
+ ++m;
+ }
+ }
+ dump_table16("__lc_time_item_idx", buf16, m);
+}
+
+static void dump_table8(const char *name, const char *tbl, int len)
+{
+ int i;
+
+ fprintf(ofp, "#define %s_LEN\t\t%d\n", name, len);
+ fprintf(ofp, "static const unsigned char %s[%d] = {", name, len);
+ for (i=0 ; i < len ; i++) {
+ if ((i % 12) == 0) {
+ fprintf(ofp, "\n\t");
+ }
+ fprintf(ofp, "%#4x, ", (int)((unsigned char) tbl[i]));
+ }
+ fprintf(ofp, "\n};\n\n");
+}
+
+#define __C_isdigit(c) \
+ ((sizeof(c) == sizeof(char)) \
+ ? (((unsigned char)((c) - '0')) < 10) \
+ : (((unsigned int)((c) - '0')) < 10))
+#define __C_isalpha(c) \
+ ((sizeof(c) == sizeof(char)) \
+ ? (((unsigned char)(((c) | 0x20) - 'a')) < 26) \
+ : (((unsigned int)(((c) | 0x20) - 'a')) < 26))
+#define __C_isalnum(c) (__C_isalpha(c) || __C_isdigit(c))
+
+static void dump_table8c(const char *name, const char *tbl, int len)
+{
+ int i;
+
+ fprintf(ofp, "#define %s_LEN\t\t%d\n", name, len);
+ fprintf(ofp, "static const unsigned char %s[%d] = {", name, len);
+ for (i=0 ; i < len ; i++) {
+ if ((i % 12) == 0) {
+ fprintf(ofp, "\n\t");
+ }
+ if (__C_isalnum(tbl[i]) || (tbl[i] == ' ')) {
+ fprintf(ofp, " '%c', ", (int)((unsigned char) tbl[i]));
+ } else {
+ fprintf(ofp, "%#4x, ", (int)((unsigned char) tbl[i]));
+ }
+ }
+ fprintf(ofp, "\n};\n\n");
+}
+
+static void dump_table16(const char *name, const int *tbl, int len)
+{
+ int i;
+
+ fprintf(ofp, "#define %s_LEN\t\t%d\n", name, len);
+ fprintf(ofp, "static const uint16_t %s[%d] = {", name, len);
+ for (i=0 ; i < len ; i++) {
+ if ((i % 8) == 0) {
+ fprintf(ofp, "\n\t");
+ }
+ if (tbl[i] != (uint16_t) tbl[i]) {
+ printf("error - falls outside uint16 range!\n");
+ exit(EXIT_FAILURE);
+ }
+ fprintf(ofp, "%#6x, ", tbl[i]);
+ }
+ fprintf(ofp, "\n};\n\n");
+}
+
+#undef DO_NL_S
+
+static int lc_numeric_item[3][256];
+static int lc_numeric_count[3];
+static unsigned char lc_numeric_uniq_3[700][3];
+static int lc_numeric_uniq;
+
+#define DO_NL_S(X) lc_numeric_S(X, k++)
+
+static void lc_numeric_S(int X, int k)
+{
+ int j, m;
+ j = addstring(nl_langinfo(X));
+ for (m=0 ; m < lc_numeric_count[k] ; m++) {
+ if (lc_numeric_item[k][m] == j) {
+ break;
+ }
+ }
+ if (m == lc_numeric_count[k]) { /* new for this nl_item */
+ if (m > 255) {
+ printf("too many nl_item %d entries in lc_numeric\n", k);
+ exit(EXIT_FAILURE);
+ }
+ lc_numeric_item[k][m] = j;
+ ++lc_numeric_count[k];
+ }
+/* printf("\\x%02x", m); */
+ lc_numeric_uniq_3[lc_numeric_uniq][k] = m;
+}
+
+static void do_lc_numeric(void)
+{
+ int i, k, m;
+
+ last = buf+1;
+ uniq = 1;
+ *buf = 0;
+ *idx = buf;
+
+ for (i=0 ; i < num_locales ; i++) {
+ k = 0;
+
+ if (!setlocale(LC_ALL, locales[i].glibc_name)) {
+ printf("setlocale(LC_ALL,%s) failed!\n",
+ locales[i].glibc_name);
+ }
+
+ DO_NL_S(RADIXCHAR); /* DECIMAL_POINT */
+ DO_NL_S(THOUSEP); /* THOUSANDS_SEP */
+ DO_NL_S(GROUPING);
+
+ if (k > 3) {
+ printf("error -- lc_numeric nl_item count > 3!\n");
+ exit(EXIT_FAILURE);
+ }
+
+ {
+ int r;
+ for (r=0 ; r < lc_numeric_uniq ; r++) {
+ if (!memcmp(lc_numeric_uniq_3[lc_numeric_uniq],
+ lc_numeric_uniq_3[r], 3)) {
+ break;
+ }
+ }
+ if (r == lc_numeric_uniq) { /* new locale row */
+ ++lc_numeric_uniq;
+ if (lc_numeric_uniq > 255) {
+ printf("too many unique lc_numeric rows!\n");
+ exit(EXIT_FAILURE);
+ }
+ }
+ locales[i].lc_numeric_row = r;
+ }
+ }
+
+ printf("buf-size=%d uniq=%d rows=%d\n",
+ (int)(last - buf), uniq, lc_numeric_uniq);
+ printf("total = %d + %d * 3 + %d = %d\n",
+ num_locales, lc_numeric_uniq, (int)(last - buf),
+ i = num_locales + lc_numeric_uniq*3 + (int)(last - buf));
+ total_size += i;
+
+/* printf("buf-size=%d uniq=%d\n", (int)(last - buf), uniq); */
+
+ dump_table8c("__lc_numeric_data", buf, (int)(last - buf));
+
+
+ for (i=0 ; i < lc_numeric_uniq ; i++) {
+ m = locales[i].lc_numeric_row;
+ for (k=0 ; k < 3 ; k++) {
+ buf[3*i + k] = (char)((unsigned char) lc_numeric_uniq_3[i][k]);
+ }
+ }
+ dump_table8("__lc_numeric_rows", buf, lc_numeric_uniq * 3);
+
+ buf16[0] =0;
+ for (i=0 ; i < 3 - 1 ; i++) {
+ buf16[i+1] = buf16[i] + lc_numeric_count[i];
+ }
+ dump_table16("__lc_numeric_item_offsets", buf16, 3);
+
+ m = 0;
+ for (k=0 ; k < 3 ; k++) {
+ for (i=0 ; i < lc_numeric_count[k] ; i++) {
+ buf16[m] = lc_numeric_item[k][i];
+ ++m;
+ }
+ }
+ dump_table16("__lc_numeric_item_idx", buf16, m);
+}
+
+#undef DO_NL_S
+
+/* #define NUM_NL_MONETARY 7 */
+#define NUM_NL_MONETARY (7+14+1)
+
+static int lc_monetary_item[NUM_NL_MONETARY][256];
+static int lc_monetary_count[NUM_NL_MONETARY];
+static unsigned char lc_monetary_uniq_X[700][NUM_NL_MONETARY];
+static int lc_monetary_uniq;
+
+#define DO_NL_S(X) lc_monetary_S(X, k++)
+
+/* #define DO_NL_C(X) printf("%#02x", (int)(unsigned char)(*nl_langinfo(X))); */
+#define DO_NL_C(X) lc_monetary_C(X, k++)
+
+static void lc_monetary_C(int X, int k)
+{
+ int j, m;
+ char c_buf[2];
+
+#warning fix the char entries for monetary... target signedness of char may be different!
+
+ c_buf[1] = 0;
+ c_buf[0] = *nl_langinfo(X);
+ j = addstring(c_buf);
+ for (m=0 ; m < lc_monetary_count[k] ; m++) {
+ if (lc_monetary_item[k][m] == j) {
+ break;
+ }
+ }
+ if (m == lc_monetary_count[k]) { /* new for this nl_item */
+ if (m > 255) {
+ printf("too many nl_item %d entries in lc_monetary\n", k);
+ exit(EXIT_FAILURE);
+ }
+ lc_monetary_item[k][m] = j;
+ ++lc_monetary_count[k];
+ }
+/* printf("\\x%02x", m); */
+ lc_monetary_uniq_X[lc_monetary_uniq][k] = m;
+}
+
+
+static void lc_monetary_S(int X, int k)
+{
+ int j, m;
+ j = addstring(nl_langinfo(X));
+ for (m=0 ; m < lc_monetary_count[k] ; m++) {
+ if (lc_monetary_item[k][m] == j) {
+ break;
+ }
+ }
+ if (m == lc_monetary_count[k]) { /* new for this nl_item */
+ if (m > 255) {
+ printf("too many nl_item %d entries in lc_monetary\n", k);
+ exit(EXIT_FAILURE);
+ }
+ lc_monetary_item[k][m] = j;
+ ++lc_monetary_count[k];
+ }
+/* printf("\\x%02x", m); */
+ lc_monetary_uniq_X[lc_monetary_uniq][k] = m;
+}
+
+static void do_lc_monetary(void)
+{
+ int i, k, m;
+
+ last = buf+1;
+ uniq = 1;
+ *buf = 0;
+ *idx = buf;
+
+ for (i=0 ; i < num_locales ; i++) {
+ k = 0;
+
+ if (!setlocale(LC_ALL, locales[i].glibc_name)) {
+ printf("setlocale(LC_ALL,%s) failed!\n",
+ locales[i].glibc_name);
+ }
+
+
+ /* non SUSv3 */
+ DO_NL_S(INT_CURR_SYMBOL);
+ DO_NL_S(CURRENCY_SYMBOL);
+ DO_NL_S(MON_DECIMAL_POINT);
+ DO_NL_S(MON_THOUSANDS_SEP);
+ DO_NL_S(MON_GROUPING);
+ DO_NL_S(POSITIVE_SIGN);
+ DO_NL_S(NEGATIVE_SIGN);
+ DO_NL_C(INT_FRAC_DIGITS);
+ DO_NL_C(FRAC_DIGITS);
+ DO_NL_C(P_CS_PRECEDES);
+ DO_NL_C(P_SEP_BY_SPACE);
+ DO_NL_C(N_CS_PRECEDES);
+ DO_NL_C(N_SEP_BY_SPACE);
+ DO_NL_C(P_SIGN_POSN);
+ DO_NL_C(N_SIGN_POSN);
+ DO_NL_C(INT_P_CS_PRECEDES);
+ DO_NL_C(INT_P_SEP_BY_SPACE);
+ DO_NL_C(INT_N_CS_PRECEDES);
+ DO_NL_C(INT_N_SEP_BY_SPACE);
+ DO_NL_C(INT_P_SIGN_POSN);
+ DO_NL_C(INT_N_SIGN_POSN);
+
+ DO_NL_S(CRNCYSTR); /* CURRENCY_SYMBOL */
+
+/* printf("\n"); */
+
+ if (k > NUM_NL_MONETARY) {
+ printf("error -- lc_monetary nl_item count > %d!\n", NUM_NL_MONETARY);
+ exit(EXIT_FAILURE);
+ }
+
+ {
+ int r;
+ for (r=0 ; r < lc_monetary_uniq ; r++) {
+ if (!memcmp(lc_monetary_uniq_X[lc_monetary_uniq],
+ lc_monetary_uniq_X[r], NUM_NL_MONETARY)) {
+ break;
+ }
+ }
+ if (r == lc_monetary_uniq) { /* new locale row */
+ ++lc_monetary_uniq;
+ if (lc_monetary_uniq > 255) {
+ printf("too many unique lc_monetary rows!\n");
+ exit(EXIT_FAILURE);
+ }
+ }
+ locales[i].lc_monetary_row = r;
+ }
+ }
+
+ printf("buf-size=%d uniq=%d rows=%d\n",
+ (int)(last - buf), uniq, lc_monetary_uniq);
+ printf("total = %d + %d * %d + %d = %d\n",
+ num_locales, lc_monetary_uniq, NUM_NL_MONETARY, (int)(last - buf),
+ i = num_locales + lc_monetary_uniq*NUM_NL_MONETARY + (int)(last - buf));
+ total_size += i;
+
+ dump_table8c("__lc_monetary_data", buf, (int)(last - buf));
+
+ for (i=0 ; i < lc_monetary_uniq ; i++) {
+ m = locales[i].lc_monetary_row;
+ for (k=0 ; k < NUM_NL_MONETARY ; k++) {
+ buf[NUM_NL_MONETARY*i + k] = (char)((unsigned char) lc_monetary_uniq_X[i][k]);
+ }
+ }
+ dump_table8("__lc_monetary_rows", buf, lc_monetary_uniq * NUM_NL_MONETARY);
+
+ buf16[0] =0;
+ for (i=0 ; i < NUM_NL_MONETARY - 1 ; i++) {
+ buf16[i+1] = buf16[i] + lc_monetary_count[i];
+ }
+ dump_table16("__lc_monetary_item_offsets", buf16, NUM_NL_MONETARY);
+
+ m = 0;
+ for (k=0 ; k < NUM_NL_MONETARY ; k++) {
+ for (i=0 ; i < lc_monetary_count[k] ; i++) {
+ buf16[m] = lc_monetary_item[k][i];
+ ++m;
+ }
+ }
+ dump_table16("__lc_monetary_item_idx", buf16, m);
+}
+
+
+#undef DO_NL_S
+
+static int lc_messages_item[2][256];
+static int lc_messages_count[2];
+static unsigned char lc_messages_uniq_2[700][2];
+static int lc_messages_uniq;
+
+#define DO_NL_S(X) lc_messages_S(X, k++)
+
+static void lc_messages_S(int X, int k)
+{
+ int j, m;
+ j = addstring(nl_langinfo(X));
+ for (m=0 ; m < lc_messages_count[k] ; m++) {
+ if (lc_messages_item[k][m] == j) {
+ break;
+ }
+ }
+ if (m == lc_messages_count[k]) { /* new for this nl_item */
+ if (m > 255) {
+ printf("too many nl_item %d entries in lc_messages\n", k);
+ exit(EXIT_FAILURE);
+ }
+ lc_messages_item[k][m] = j;
+ ++lc_messages_count[k];
+ }
+/* printf("\\x%02x", m); */
+ lc_messages_uniq_2[lc_messages_uniq][k] = m;
+}
+
+static void do_lc_messages(void)
+{
+ int i, k, m;
+
+ last = buf+1;
+ uniq = 1;
+ *buf = 0;
+ *idx = buf;
+
+ for (i=0 ; i < num_locales ; i++) {
+ k = 0;
+
+ if (!setlocale(LC_ALL, locales[i].glibc_name)) {
+ printf("setlocale(LC_ALL,%s) failed!\n",
+ locales[i].glibc_name);
+ }
+
+ DO_NL_S(YESEXPR);
+ DO_NL_S(NOEXPR);
+
+ if (k > 2) {
+ printf("error -- lc_messages nl_item count > 2!\n");
+ exit(EXIT_FAILURE);
+ }
+
+ {
+ int r;
+ for (r=0 ; r < lc_messages_uniq ; r++) {
+ if (!memcmp(lc_messages_uniq_2[lc_messages_uniq],
+ lc_messages_uniq_2[r], 2)) {
+ break;
+ }
+ }
+ if (r == lc_messages_uniq) { /* new locale row */
+ ++lc_messages_uniq;
+ if (lc_messages_uniq > 255) {
+ printf("too many unique lc_messages rows!\n");
+ exit(EXIT_FAILURE);
+ }
+ }
+ locales[i].lc_messages_row = r;
+ }
+ }
+
+ printf("buf-size=%d uniq=%d rows=%d\n",
+ (int)(last - buf), uniq, lc_messages_uniq);
+ printf("total = %d + %d * 2 + %d = %d\n",
+ num_locales, lc_messages_uniq, (int)(last - buf),
+ i = num_locales + lc_messages_uniq*2 + (int)(last - buf));
+ total_size += i;
+
+/* printf("buf-size=%d uniq=%d\n", (int)(last - buf), uniq); */
+
+ dump_table8c("__lc_messages_data", buf, (int)(last - buf));
+
+ for (i=0 ; i < lc_messages_uniq ; i++) {
+ m = locales[i].lc_messages_row;
+ for (k=0 ; k < 2 ; k++) {
+ buf[2*i + k] = (char)((unsigned char) lc_messages_uniq_2[i][k]);
+ }
+ }
+ dump_table8("__lc_messages_rows", buf, lc_messages_uniq * 2);
+
+ buf16[0] =0;
+ for (i=0 ; i < 2 - 1 ; i++) {
+ buf16[i+1] = buf16[i] + lc_messages_count[i];
+ }
+ dump_table16("__lc_messages_item_offsets", buf16, 2);
+
+ m = 0;
+ for (k=0 ; k < 2 ; k++) {
+ for (i=0 ; i < lc_messages_count[k] ; i++) {
+ buf16[m] = lc_messages_item[k][i];
+ ++m;
+ }
+ }
+ dump_table16("__lc_messages_item_idx", buf16, m);
+}
+
+
+
+
+
+static void read_at_mappings(void)
+{
+ char *p;
+ char *m;
+ int mc = 0;
+
+ do {
+ if (!(p = strtok(line_buf, " \t\n")) || (*p == '#')) {
+ if (!fgets(line_buf, sizeof(line_buf), fp)) {
+ if (ferror(fp)) {
+ printf("error reading file\n");
+ exit(EXIT_FAILURE);
+ }
+ return; /* EOF */
+ }
+ if ((*line_buf == '#') && (line_buf[1] == '-')) {
+ break;
+ }
+ continue;
+ }
+ if (*p == '@') {
+ if (p[1] == 0) {
+ printf("error: missing @modifier name\n");
+ exit(EXIT_FAILURE);
+ }
+ m = p; /* save the modifier name */
+ if (!(p = strtok(NULL, " \t\n")) || p[1] || (((unsigned char) *p) > 0x7f)) {
+ printf("error: missing or illegal @modifier mapping char\n");
+ exit(EXIT_FAILURE);
+ }
+ if (at_mappings[(int)((unsigned char) *p)]) {
+ printf("error: reused @modifier mapping char\n");
+ exit(EXIT_FAILURE);
+ }
+ at_mappings[(int)((unsigned char) *p)] = 1;
+ at_mapto[mc] = *p;
+ ++mc;
+ *at_strings_end = (char)( (unsigned char) (strlen(m)) );
+ strcpy(++at_strings_end, m+1);
+ at_strings_end += (unsigned char) at_strings_end[-1];
+
+ printf("@mapping: \"%s\" to '%c'\n", m, *p);
+
+ if (((p = strtok(NULL, " \t\n")) != NULL) && (*p != '#')) {
+ printf("ignoring trailing text: %s...\n", p);
+ }
+ *line_buf = 0;
+ continue;
+ }
+ break;
+ } while (1);
+
+#if 0
+ {
+ p = at_strings;
+
+ if (!*p) {
+ printf("no @ strings\n");
+ return;
+ }
+
+ do {
+ printf("%s\n", p+1);
+ p += 1 + (unsigned char) *p;
+ } while (*p);
+ }
+#endif
+}
+
+static void read_enable_disable(void)
+{
+ char *p;
+
+ do {
+ if (!(p = strtok(line_buf, " =\t\n")) || (*p == '#')) {
+ if (!fgets(line_buf, sizeof(line_buf), fp)) {
+ if (ferror(fp)) {
+ printf("error reading file\n");
+ exit(EXIT_FAILURE);
+ }
+ return; /* EOF */
+ }
+ if ((*line_buf == '#') && (line_buf[1] == '-')) {
+ break;
+ }
+ continue;
+ }
+ if (!strcmp(p, "UTF-8")) {
+ if (!(p = strtok(NULL, " =\t\n"))
+ || ((toupper(*p) != 'Y') && (toupper(*p) != 'N'))) {
+ printf("error: missing or illegal UTF-8 setting\n");
+ exit(EXIT_FAILURE);
+ }
+ default_utf8 = (toupper(*p) == 'Y');
+ printf("UTF-8 locales are %sabled\n", "dis\0en"+ (default_utf8 << 2));
+ } else if (!strcmp(p, "8-BIT")) {
+ if (!(p = strtok(NULL, " =\t\n"))
+ || ((toupper(*p) != 'Y') && (toupper(*p) != 'N'))) {
+ printf("error: missing or illegal 8-BIT setting\n");
+ exit(EXIT_FAILURE);
+ }
+ default_8bit = (toupper(*p) == 'Y');
+ printf("8-BIT locales are %sabled\n", "dis\0en" + (default_8bit << 2));
+ } else {
+ break;
+ }
+
+ if (((p = strtok(NULL, " \t\n")) != NULL) && (*p != '#')) {
+ printf("ignoring trailing text: %s...\n", p);
+ }
+ *line_buf = 0;
+ continue;
+
+ } while (1);
+}
+
+#ifdef CODESET_LIST
+
+static int find_codeset_num(const char *cs)
+{
+ int r = 2;
+ char *s = CODESET_LIST;
+
+ /* 7-bit is 1, UTF-8 is 2, 8-bits are > 2 */
+
+ if (strcmp(cs, "UTF-8") != 0) {
+ ++r;
+ while (*s && strcmp(CODESET_LIST+ ((unsigned char) *s), cs)) {
+/* printf("tried %s\n", CODESET_LIST + ((unsigned char) *s)); */
+ ++r;
+ ++s;
+ }
+ if (!*s) {
+ printf("error: unsupported codeset %s\n", cs);
+ exit(EXIT_FAILURE);
+ }
+ }
+ return r;
+}
+
+#else
+
+static int find_codeset_num(const char *cs)
+{
+ int r = 2;
+
+ /* 7-bit is 1, UTF-8 is 2, 8-bits are > 2 */
+
+ if (strcmp(cs, "UTF-8") != 0) {
+ printf("error: unsupported codeset %s\n", cs);
+ exit(EXIT_FAILURE);
+ }
+ return r;
+}
+
+#endif
+
+static int find_at_string_num(const char *as)
+{
+ int i = 0;
+ char *p = at_strings;
+
+ while (*p) {
+ if (!strcmp(p+1, as)) {
+ return i;
+ }
+ ++i;
+ p += 1 + (unsigned char) *p;
+ }
+
+ printf("error: unmapped @string %s\n", as);
+ exit(EXIT_FAILURE);
+}
+
+static void read_locale_list(void)
+{
+ char *p;
+ char *s;
+ char *ln; /* locale name */
+ char *ls; /* locale name ll_CC */
+ char *as; /* at string */
+ char *ds; /* dot string */
+ char *cs; /* codeset */
+ int i;
+
+ typedef struct {
+ char *glibc_name;
+ char name[5];
+ char dot_cs; /* 0 if no codeset specified */
+ char cs;
+ } locale_entry;
+
+ /* First the C locale. */
+ locales[0].glibc_name = locales[0].name;
+ strncpy(locales[0].name,"C",5);
+ locales[0].dot_cs = 0;
+ locales[0].cs = 1; /* 7-bit encoding */
+ ++num_locales;
+
+ do {
+ if (!(p = strtok(line_buf, " \t\n")) || (*p == '#')) {
+ if (!fgets(line_buf, sizeof(line_buf), fp)) {
+ if (ferror(fp)) {
+ printf("error reading file\n");
+ exit(EXIT_FAILURE);
+ }
+ return; /* EOF */
+ }
+ if ((*line_buf == '#') && (line_buf[1] == '-')) {
+ break;
+ }
+ continue;
+ }
+
+ s = glibc_locale_names;
+ for (i=0 ; i < num_locales ; i++) {
+ if (!strcmp(s+1, p)) {
+ break;
+ }
+ s += 1 + ((unsigned char) *s);
+ }
+ if (i < num_locales) {
+ printf("ignoring dulplicate locale name: %s", p);
+ *line_buf = 0;
+ continue;
+ }
+
+ /* New locale, but don't increment num until codeset verified! */
+ *s = (char)((unsigned char) (strlen(p) + 1));
+ strcpy(s+1, p);
+ locales[num_locales].glibc_name = s+1;
+ ln = p; /* save locale name */
+
+ if (!(p = strtok(NULL, " \t\n"))) {
+ printf("error: missing codeset for locale %s\n", ln);
+ exit(EXIT_FAILURE);
+ }
+ cs = p;
+ i = find_codeset_num(p);
+ if ((i == 2) && !default_utf8) {
+ printf("ignoring UTF-8 locale %s\n", ln);
+ *line_buf = 0;
+ continue;
+ } else if ((i > 2) && !default_8bit) {
+ printf("ignoring 8-bit codeset locale %s\n", ln);
+ *line_buf = 0;
+ continue;
+ }
+ locales[num_locales].cs = (char)((unsigned char) i);
+
+ if (((p = strtok(NULL, " \t\n")) != NULL) && (*p != '#')) {
+ printf("ignoring trailing text: %s...\n", p);
+ }
+
+ /* Now go back to locale string for .codeset and @modifier */
+ as = strtok(ln, "@");
+ if (as) {
+ as = strtok(NULL, "@");
+ }
+ ds = strtok(ln, ".");
+ if (ds) {
+ ds = strtok(NULL, ".");
+ }
+ ls = ln;
+
+ if ((strlen(ls) != 5) || (ls[2] != '_')) {
+ printf("error: illegal locale name %s\n", ls);
+ exit(EXIT_FAILURE);
+ }
+
+ i = 0; /* value for unspecified codeset */
+ if (ds) {
+ i = find_codeset_num(ds);
+ if ((i == 2) && !default_utf8) {
+ printf("ignoring UTF-8 locale %s\n", ln);
+ *line_buf = 0;
+ continue;
+ } else if ((i > 2) && !default_8bit) {
+ printf("ignoring 8-bit codeset locale %s\n", ln);
+ *line_buf = 0;
+ continue;
+ }
+ }
+ locales[num_locales].dot_cs = (char)((unsigned char) i);
+
+ if (as) {
+ i = find_at_string_num(as);
+ ls[2] = at_mapto[i];
+ }
+ memcpy(locales[num_locales].name, ls, 5);
+/* printf("locale: %5.5s %2d %2d %s\n", */
+/* locales[num_locales].name, */
+/* locales[num_locales].cs, */
+/* locales[num_locales].dot_cs, */
+/* locales[num_locales].glibc_name */
+/* ); */
+ ++num_locales;
+ *line_buf = 0;
+ } while (1);
+}
+
+static int le_cmp(const void *a, const void *b)
+{
+ const locale_entry *p;
+ const locale_entry *q;
+ int r;
+
+ p = (const locale_entry *) a;
+ q = (const locale_entry *) b;
+
+ if (!(r = p->name[0] - q->name[0])
+ && !(r = p->name[1] - q->name[1])
+ && !(r = p->name[3] - q->name[3])
+ && !(r = p->name[4] - q->name[4])
+ && !(r = p->name[2] - q->name[2])
+ && !(r = -(p->cs - q->cs))
+ ) {
+ r = -(p->dot_cs - q->dot_cs);
+ /* Reverse the ordering of the codesets so UTF-8 comes last.
+ * Work-around (hopefully) for glibc bug affecting at least
+ * the euro currency symbol. */
+ }
+
+ return r;
+}
+
diff --git a/extra/locale/gen_mmap.c b/extra/locale/gen_mmap.c
new file mode 100644
index 000000000..ae5d4dff6
--- /dev/null
+++ b/extra/locale/gen_mmap.c
@@ -0,0 +1,237 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <assert.h>
+#include <stdint.h>
+#include <stddef.h>
+
+#define WANT_DATA
+#include "c8tables.h"
+
+/* #define Cctype_TBL_LEN 328 */
+/* #define Cuplow_TBL_LEN 400 */
+/* #define Cc2wc_TBL_LEN 1448 */
+/* #define Cwc2c_TBL_LEN 3744 */
+
+#define WANT_WCctype_data
+#define WANT_WCuplow_data
+#define WANT_WCuplow_diff_data
+#define WANT_WCcomb_data
+/* #define WANT_WCwidth_data */
+#include "wctables.h"
+#undef WANT_WCctype_data
+#undef WANT_WCuplow_data
+#undef WANT_WCuplow_diff_data
+#undef WANT_WCcomb_data
+/* #undef WANT_WCwidth_data */
+
+/* #define WCctype_TBL_LEN (WCctype_II_LEN + WCctype_TI_LEN + WCctype_UT_LEN) */
+/* #define WCuplow_TBL_LEN (WCuplow_II_LEN + WCuplow_TI_LEN + WCuplow_UT_LEN) */
+/* #define WCuplow_diff_TBL_LEN (2 * WCuplow_diffs) */
+/* #define WCcomb_TBL_LEN (WCcomb_II_LEN + WCcomb_TI_LEN + WCcomb_UT_LEN) */
+
+#include "locale_tables.h"
+
+#include "locale_mmap.h"
+
+/* #undef __PASTE2 */
+/* #define __PASTE2(A,B) A ## B */
+/* #undef __PASTE3 */
+/* #define __PASTE3(A,B,C) A ## B ## C */
+
+
+/* #define MAGIC_SIZE 64 */
+
+/* #define COMMON_MMAP(X) \ */
+/* unsigned char __PASTE3(lc_,X,_data)[__PASTE3(__lc_,X,_data_LEN)]; */
+
+/* #define COMMON_MMIDX(X) \ */
+/* unsigned char __PASTE3(lc_,X,_rows)[__PASTE3(__lc_,X,_rows_LEN)]; \ */
+/* uint16_t __PASTE3(lc_,X,_item_offsets)[__PASTE3(__lc_,X,_item_offsets_LEN)]; \ */
+/* uint16_t __PASTE3(lc_,X,_item_idx)[__PASTE3(__lc_,X,_item_idx_LEN)]; \ */
+
+#define WRITE_COMMON_MMAP(X) \
+ fseek(fp, (long) offsetof(__locale_mmap_t, __PASTE3(lc_,X,_data)), SEEK_SET); \
+ for (i=0 ; i < __PASTE3(__lc_,X,_data_LEN) ; i++) { \
+ putc(__PASTE3(__lc_,X,_data)[i], fp); \
+ }
+
+#define WRITE_COMMON_MMIDX(X) \
+ fseek(fp, (long) offsetof(__locale_mmap_t, __PASTE3(lc_,X,_rows)), SEEK_SET); \
+ for (i=0 ; i < __PASTE3(__lc_,X,_rows_LEN) ; i++) { \
+ putc(__PASTE3(__lc_,X,_rows)[i], fp); \
+ } \
+ fseek(fp, (long) offsetof(__locale_mmap_t, __PASTE3(lc_,X,_item_offsets)), SEEK_SET); \
+ for (i=0 ; i < __PASTE3(__lc_,X,_item_offsets_LEN) ; i++) { \
+ putc( ((unsigned char *) &(__PASTE3(__lc_,X,_item_offsets)[i]))[0], fp); \
+ putc( ((unsigned char *) &(__PASTE3(__lc_,X,_item_offsets)[i]))[1], fp); \
+ } \
+ fseek(fp, (long) offsetof(__locale_mmap_t, __PASTE3(lc_,X,_item_idx)), SEEK_SET); \
+ for (i=0 ; i < __PASTE3(__lc_,X,_item_idx_LEN) ; i++) { \
+ putc( ((unsigned char *) &(__PASTE3(__lc_,X,_item_idx)[i]))[0], fp); \
+ putc( ((unsigned char *) &(__PASTE3(__lc_,X,_item_idx)[i]))[1], fp); \
+ }
+
+#define WRITE_WC_DATA(X) \
+ fseek(fp, (long) offsetof(__locale_mmap_t, __PASTE2(tblw,X)), SEEK_SET); \
+ for (i=0 ; i < __PASTE3(WC,X,_TBL_LEN) ; i++) { \
+ putc(__PASTE3(WC,X,_data)[i], fp); \
+ }
+
+#define WRITE_WC_I16_DATA(X) \
+ fseek(fp, (long) offsetof(__locale_mmap_t, __PASTE2(tblw,X)), SEEK_SET); \
+ for (i=0 ; i < __PASTE3(WC,X,_TBL_LEN) ; i++) { \
+ putc( ((unsigned char *) &(__PASTE3(WC,X,_data)[i]))[0], fp); \
+ putc( ((unsigned char *) &(__PASTE3(WC,X,_data)[i]))[1], fp); \
+ }
+
+#define WRITE_C_DATA(X) \
+ fseek(fp, (long) offsetof(__locale_mmap_t, __PASTE2(tbl8,X)), SEEK_SET); \
+ for (i=0 ; i < __PASTE3(C,X,_TBL_LEN) ; i++) { \
+ putc(__PASTE3(C,X,_data)[i], fp); \
+ }
+
+#define WRITE_C_U16_DATA(X) \
+ fseek(fp, (long) offsetof(__locale_mmap_t, __PASTE2(tbl8,X)), SEEK_SET); \
+ for (i=0 ; i < __PASTE3(C,X,_TBL_LEN) ; i++) { \
+ putc( ((unsigned char *) &(__PASTE3(C,X,_data)[i]))[0], fp); \
+ putc( ((unsigned char *) &(__PASTE3(C,X,_data)[i]))[1], fp); \
+ }
+
+/**********************************************************************/
+
+#define COMMON_OFFSETS(X) \
+ offsetof(__locale_mmap_t, __PASTE3(lc_,X,_rows)), \
+ offsetof(__locale_mmap_t, __PASTE3(lc_,X,_item_offsets)), \
+ offsetof(__locale_mmap_t, __PASTE3(lc_,X,_item_idx)), \
+ offsetof(__locale_mmap_t, __PASTE3(lc_,X,_data)) \
+
+
+static const size_t common_tbl_offsets[CATEGORIES*4] = {
+ 0, 0, 0, 0, /* ctype */
+ COMMON_OFFSETS(numeric),
+ COMMON_OFFSETS(monetary),
+ COMMON_OFFSETS(time),
+ 0, 0, 0, 0, /* collate */
+ COMMON_OFFSETS(messages),
+};
+
+
+
+
+
+int main(void)
+{
+ FILE *fp;
+ size_t i;
+ unsigned char *p;
+
+ if (!(fp = fopen("locale.mmap", "w"))) {
+ printf("error - can't open locale.mmap for writing!");
+ return EXIT_FAILURE;
+ }
+
+ for (i=0 ; i < sizeof(__locale_mmap_t) ; i++) {
+ putc(0, fp); /* Zero out the file. */
+ }
+
+#ifdef __CTYPE_HAS_8_BIT_LOCALES
+ WRITE_C_DATA(ctype);
+ WRITE_C_DATA(uplow);
+#ifdef __WCHAR_ENABLED
+ WRITE_C_U16_DATA(c2wc);
+ WRITE_C_DATA(wc2c);
+ /* translit */
+#endif /* __WCHAR_ENABLED */
+#endif /* __CTYPE_HAS_8_BIT_LOCALES */
+
+#ifdef __WCHAR_ENABLED
+ WRITE_WC_DATA(ctype);
+ WRITE_WC_DATA(uplow);
+ WRITE_WC_I16_DATA(uplow_diff);
+ WRITE_WC_DATA(comb);
+ /* width?? */
+#endif /* __WCHAR_ENABLED */
+
+ WRITE_COMMON_MMAP(numeric);
+ WRITE_COMMON_MMAP(monetary);
+ WRITE_COMMON_MMAP(time);
+ /* TODO -- collate*/
+ WRITE_COMMON_MMAP(messages);
+
+#ifdef __CTYPE_HAS_8_BIT_LOCALES
+ fseek(fp, (long) offsetof(__locale_mmap_t, codeset_8_bit), SEEK_SET); \
+ p = (unsigned char *) codeset_8_bit;
+ for (i=0 ; i < sizeof(codeset_8_bit) ; i++) {
+ putc(p[i], fp);
+ }
+#endif /* __CTYPE_HAS_8_BIT_LOCALES */
+
+ WRITE_COMMON_MMIDX(numeric);
+ WRITE_COMMON_MMIDX(monetary);
+ WRITE_COMMON_MMIDX(time);
+ /* TODO -- collate*/
+ WRITE_COMMON_MMIDX(messages);
+
+ fseek(fp, (long) offsetof(__locale_mmap_t, lc_common_item_offsets_LEN), SEEK_SET);
+ putc(1, fp); /* ctype -- (codeset) handled specially */
+ putc(__lc_numeric_item_offsets_LEN, fp);
+ putc(__lc_monetary_item_offsets_LEN, fp);
+ putc(__lc_time_item_offsets_LEN, fp);
+ putc(0, fp); /* collate */
+ putc(__lc_messages_item_offsets_LEN, fp);
+
+ fseek(fp, (long) offsetof(__locale_mmap_t, lc_common_tbl_offsets), SEEK_SET);
+ for (i=0 ; i < sizeof(common_tbl_offsets) ; i++) {
+ putc(((unsigned char *)common_tbl_offsets)[i], fp);
+ }
+
+#ifdef NUM_LOCALES
+ fseek(fp, (long) offsetof(__locale_mmap_t, locales), SEEK_SET);
+ for (i=0 ; i < (NUM_LOCALES * WIDTH_LOCALES) ; i++) {
+ putc(__locales[i], fp);
+ }
+
+ fseek(fp, (long) offsetof(__locale_mmap_t, locale_names5), SEEK_SET);
+ for (i=0 ; i < 5 * NUM_LOCALE_NAMES ; i++) {
+ putc(__locale_names5[i], fp);
+ }
+
+#ifdef LOCALE_AT_MODIFIERS_LENGTH
+ fseek(fp, (long) offsetof(__locale_mmap_t, locale_at_modifiers), SEEK_SET);
+ for (i=0 ; i < LOCALE_AT_MODIFIERS_LENGTH ; i++) {
+ putc(__locale_at_modifiers[i], fp);
+ }
+#endif /* LOCALE_AT_MODIFIERS_LENGTH */
+#endif /* NUM_LOCALES */
+
+ fseek(fp, (long) offsetof(__locale_mmap_t, lc_names), SEEK_SET);
+ for (i=0 ; i < lc_names_LEN ; i++) {
+ putc(lc_names[i], fp);
+ }
+
+#ifdef __CTYPE_HAS_8_BIT_LOCALES
+ fseek(fp, (long) offsetof(__locale_mmap_t, codeset_list), SEEK_SET);
+ for (i=0 ; i < sizeof(CODESET_LIST) ; i++) {
+ putc((unsigned char)(CODESET_LIST[i]), fp);
+ }
+#endif /* __CTYPE_HAS_8_BIT_LOCALES */
+
+
+ if (ferror(fp) || (fclose(fp) == EOF)) {
+ printf("error - stream in error state or fclose failed!");
+ return EXIT_FAILURE;
+ }
+
+ printf("sizeof(__locale_mmap_t) = %zd\n", sizeof(__locale_mmap_t));
+
+ return EXIT_SUCCESS;
+}
+
+/* TODO:
+ * collate data (8-bit weighted single char only)
+ * @ mappings!
+ * codeset list? yes, since we'll want to be able to inspect them...
+ * that means putting some header stuff in magic
+ * fix ctype LEN defines in gen_c8tables
+ */
diff --git a/extra/locale/gen_wc8bit.c b/extra/locale/gen_wc8bit.c
new file mode 100644
index 000000000..785c3f4c0
--- /dev/null
+++ b/extra/locale/gen_wc8bit.c
@@ -0,0 +1,729 @@
+#define _GNU_SOURCE
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <locale.h>
+#include <stddef.h>
+#include <wctype.h>
+#include <limits.h>
+
+/* #define CTYPE_PACKED */
+#define UPLOW_IDX_SHIFT 3
+/* best if 2 unpacked or 3 packed */
+#define CTYPE_IDX_SHIFT 3
+/* 3 or 4 are very similar */
+#define C2WC_IDX_SHIFT 3
+
+#define CTYPE_IDX_LEN (128 >> (CTYPE_IDX_SHIFT))
+#define UPLOW_IDX_LEN (128 >> (UPLOW_IDX_SHIFT))
+#define C2WC_IDX_LEN (128 >> (C2WC_IDX_SHIFT))
+
+/* #ifdef CTYPE_PACKED */
+/* #define CTYPE_ROW_LEN (1 << ((CTYPE_IDX_SHIFT)-1)) */
+/* #else */
+#define CTYPE_ROW_LEN (1 << (CTYPE_IDX_SHIFT))
+/* #endif */
+#define UPLOW_ROW_LEN (1 << (UPLOW_IDX_SHIFT))
+#define C2WC_ROW_LEN (1 << (C2WC_IDX_SHIFT))
+
+
+
+#define MAX_WCHAR (0x2600-1)
+
+static unsigned char ctype_tbl[256 * CTYPE_ROW_LEN];
+static unsigned char uplow_tbl[256 * UPLOW_ROW_LEN];
+#ifdef DO_WIDE_CHAR
+static unsigned short c2wc_tbl[256 * C2WC_ROW_LEN];
+#endif
+static unsigned char tt[MAX_WCHAR+1];
+static unsigned char ti[MAX_WCHAR+1];
+static unsigned char xi[MAX_WCHAR+1];
+
+static int n_ctype_rows;
+static int n_uplow_rows;
+#ifdef DO_WIDE_CHAR
+static int n_c2wc_rows;
+#endif
+static int tt_num;
+static int ti_num;
+
+#define RANGE MAX_WCHAR
+
+#define TT_SHIFT 4
+#define TI_SHIFT 4
+
+#define II_LEN ((MAX_WCHAR+1) >> (TT_SHIFT+TI_SHIFT))
+
+typedef struct {
+ unsigned long c2w[256];
+ unsigned char w2c[MAX_WCHAR];
+ unsigned char ii[II_LEN];
+ unsigned char ctype_idx[CTYPE_IDX_LEN];
+ unsigned char uplow_idx[UPLOW_IDX_LEN];
+ unsigned char c2wc_idx[C2WC_IDX_LEN];
+} charset_data;
+
+/* Taking advantage of the C99 mutual-exclusion guarantees for the various
+ * (w)ctype classes, including the descriptions of printing and control
+ * (w)chars, we can place each in one of the following mutually-exlusive
+ * subsets. Since there are less than 16, we can store the data for
+ * each (w)chars in a nibble. In contrast, glibc uses an unsigned int
+ * per (w)char, with one bit flag for each is* type. While this allows
+ * a simple '&' operation to determine the type vs. a range test and a
+ * little special handling for the "blank" and "xdigit" types in my
+ * approach, it also uses 8 times the space for the tables on the typical
+ * 32-bit archs we supported.*/
+enum {
+ __CTYPE_unclassified = 0,
+ __CTYPE_alpha_nonupper_nonlower,
+ __CTYPE_alpha_lower,
+ __CTYPE_alpha_upper_lower,
+ __CTYPE_alpha_upper,
+ __CTYPE_digit,
+ __CTYPE_punct,
+ __CTYPE_graph,
+ __CTYPE_print_space_nonblank,
+ __CTYPE_print_space_blank,
+ __CTYPE_space_nonblank_noncntrl,
+ __CTYPE_space_blank_noncntrl,
+ __CTYPE_cntrl_space_nonblank,
+ __CTYPE_cntrl_space_blank,
+ __CTYPE_cntrl_nonspace,
+};
+
+int main(int argc, char **argv)
+{
+ FILE *fp;
+ FILE *out;
+ charset_data csd[20];
+ unsigned long max_wchar;
+ unsigned char *p;
+ int numsets;
+ int i;
+ int j;
+ char buf[80];
+ unsigned char row[256];
+#ifdef DO_WIDE_CHAR
+ unsigned short wrow[256];
+#endif
+ char codeset_list[500];
+ char codeset_index[30];
+ int codeset_list_end = 0;
+ int total_size = 0;
+
+ if (!setlocale(LC_CTYPE, "en_US.UTF-8")) {
+ printf("setlocale(LC_CTYPE,\"en_US.UTF-8\") failed!\n");
+ return EXIT_FAILURE;
+ }
+
+ if (!(out = fopen("c8tables.h","w"))) {
+ printf("error: couldn't open file \"c8tables.h\"\n");
+ return EXIT_FAILURE;
+ }
+
+#if 0
+ if (argc == 1) {
+ /* User requested 8-bit codesets, but didn't list any... */
+ /* Allow to build, just so this feature can be left on in config. */
+ fprintf(out, "#ifdef __CTYPE_HAS_8_BIT_LOCALES\n");
+ fprintf(out, "#warning ignoring 8 bit codesets request"
+ " as no codesets specified.\n");
+ fprintf(out, "#endif\n");
+ fprintf(out, "#undef __CTYPE_HAS_8_BIT_LOCALES\n\n");
+
+ fprintf(out, "#define NUM_CODESETS\t\t0\n");
+ fprintf(out, "#define CODESET_LIST\t\t\"\"\n");
+ fclose(out);
+ return EXIT_SUCCESS;
+ }
+
+/* fprintf(out, "#define __CTYPE_HAS_8_BIT_LOCALES\t1\n\n"); */
+ fprintf(out, "#ifdef __CTYPE_HAS_8_BIT_LOCALES\n\n");
+#endif
+
+ if (argc == 1) {
+ fprintf(out, "#undef __CTYPE_HAS_8_BIT_LOCALES\n\n");
+
+ fprintf(out, "#define NUM_CODESETS\t\t0\n");
+ fprintf(out, "#define CODESET_LIST\t\t\"\"\n");
+ } else {
+ fprintf(out, "#define __CTYPE_HAS_8_BIT_LOCALES\t\t1\n\n");
+ }
+
+ fprintf(out, "#define Cctype_IDX_SHIFT\t%d\n", CTYPE_IDX_SHIFT);
+ fprintf(out, "#define Cctype_IDX_LEN\t\t%d\n", CTYPE_IDX_LEN);
+#ifdef CTYPE_PACKED
+ fprintf(out, "#define Cctype_ROW_LEN\t\t%d\n", CTYPE_ROW_LEN >> 1);
+ fprintf(out, "#define Cctype_PACKED\t\t1\n");
+#else
+ fprintf(out, "#define Cctype_ROW_LEN\t\t%d\n", CTYPE_ROW_LEN);
+ fprintf(out, "#undef Cctype_PACKED\n");
+#endif
+
+ fprintf(out, "\n#define Cuplow_IDX_SHIFT\t%d\n", UPLOW_IDX_SHIFT);
+ fprintf(out, "#define Cuplow_IDX_LEN\t\t%d\n", UPLOW_IDX_LEN);
+ fprintf(out, "#define Cuplow_ROW_LEN\t\t%d\n", UPLOW_ROW_LEN);
+
+#ifdef DO_WIDE_CHAR
+ fprintf(out, "\n#define Cc2wc_IDX_LEN\t\t%d\n", C2WC_IDX_LEN);
+ fprintf(out, "#define Cc2wc_IDX_SHIFT\t\t%d\n", C2WC_IDX_SHIFT);
+ fprintf(out, "#define Cc2wc_ROW_LEN\t\t%d\n", C2WC_ROW_LEN);
+#endif
+
+ fprintf(out, "\ntypedef struct {\n");
+ fprintf(out, "\tunsigned char idx8ctype[%d];\n", CTYPE_IDX_LEN);
+ fprintf(out, "\tunsigned char idx8uplow[%d];\n", UPLOW_IDX_LEN);
+#ifdef DO_WIDE_CHAR
+ fprintf(out, "\tunsigned char idx8c2wc[%d];\n", C2WC_IDX_LEN);
+ fprintf(out, "\tunsigned char idx8wc2c[%d];\n", II_LEN);
+#endif
+ fprintf(out, "} codeset_8_bit_t;\n\n");
+
+ fprintf(out, "#ifdef WANT_DATA\n\n");
+ fprintf(out, "static const codeset_8_bit_t codeset_8_bit[%d] = {\n", argc-1);
+
+ max_wchar = 0x7f;
+ numsets = 0;
+ codeset_index[0] = 0;
+ while (--argc) {
+ if (!(fp = fopen(*++argv,"r"))) {
+ printf("error: couldn't open file \"%s\"\n", *argv);
+ return EXIT_FAILURE;
+ }
+ printf("processing %s... ", *argv);
+
+ {
+ char *s0;
+ char *s1;
+ int n;
+
+ s0 = strrchr(*argv, '/');
+ if (!s0) {
+ s0 = *argv;
+ } else {
+ ++s0;
+ }
+ s1 = strchr(s0, '.');
+ if (!s1) {
+ n = strlen(s0);
+ } else {
+ n = s1 - s0;
+ }
+
+/* if ((numsets == 0) && strncmp("ASCII", s0, n)) { */
+/* printf("error - first codeset isn't ASCII!\n"); */
+/* return EXIT_FAILURE; */
+/* } */
+
+ if (numsets >= sizeof(codeset_index)) {
+ printf("error - too many codesets!\n");
+ return EXIT_FAILURE;
+ }
+
+ if (codeset_list_end + n + 1 + numsets + 1 + 1 >= 256) {
+ printf("error - codeset list to big!\n");
+ return EXIT_FAILURE;
+ }
+
+ codeset_index[numsets+1] = codeset_index[numsets] + n+1;
+ strncpy(codeset_list + codeset_list_end, s0, n);
+ codeset_list_end += (n+1);
+ codeset_list[codeset_list_end - 1] = 0;
+
+ fprintf(out, "\t{ /* %.*s */", n, s0);
+ }
+
+ memset(&csd[numsets],sizeof(charset_data),0);
+ memset(xi, sizeof(xi), 0);
+ {
+ unsigned long c, wc;
+ int lines;
+ lines = 0;
+ while (fgets(buf,sizeof(buf),fp)) {
+ if ((2 != sscanf(buf, "{ %lx , %lx", &c, &wc))
+ || (c >= 256) || (wc > MAX_WCHAR)) {
+ printf("error: scanf failure! \"%s\"\n", buf);
+ return EXIT_FAILURE;
+ }
+
+ /* don't put in w2c... dynamicly build tt instead. */
+
+ if (c <= 0x7f) { /* check the 7bit entries but don't store */
+ if (c != wc) {
+ printf("error: c != wc in %s\n", buf);
+ return EXIT_FAILURE;
+ }
+ csd[numsets].c2w[c] = wc;
+ csd[numsets].w2c[wc] = 0; /* ignore */
+ if (wc > max_wchar) {
+ max_wchar = wc;
+ }
+ } else {
+ csd[numsets].c2w[c] = wc;
+ csd[numsets].w2c[wc] = c;
+ if (wc > max_wchar) {
+ max_wchar = wc;
+ }
+ }
+ ++lines;
+ }
+ printf("%d lines ", lines);
+
+ for (i = 0 ; i <= MAX_WCHAR ; i += (1 << TT_SHIFT)) {
+ p = &csd[numsets].w2c[i];
+ for (j = 0 ; j < tt_num ; j++) {
+ if (!memcmp(p, &tt[j << TT_SHIFT], (1 << TT_SHIFT))) {
+ break;
+ }
+ }
+ if (j == tt_num) { /* new entry */
+ memcpy(&tt[j << TT_SHIFT], p, (1 << TT_SHIFT));
+ ++tt_num;
+ }
+ xi[i >> TT_SHIFT] = j;
+ }
+
+ for (i = 0 ; i <= (MAX_WCHAR >> TT_SHIFT) ; i += (1 << TI_SHIFT)) {
+ p = &xi[i];
+ for (j = 0 ; j < ti_num ; j++) {
+ if (!memcmp(p, &ti[j << TI_SHIFT], (1 << TI_SHIFT))) {
+ break;
+ }
+ }
+ if (j == ti_num) { /* new entry */
+ memcpy(&ti[j << TI_SHIFT], p, (1 << TI_SHIFT));
+ ++ti_num;
+ }
+ csd[numsets].ii[i >> TI_SHIFT] = j;
+/* printf("%d ", i >> TI_SHIFT); */
+ }
+
+#if 1
+ fprintf(out, "\n\t\t/* idx8ctype data */\n\t\t{");
+ for (i = 128 ; i < 256 ; i++) {
+ wchar_t c;
+ unsigned int d;
+
+/* if (!(i & 0x7)) { */
+/* fprintf(out, "\n"); */
+/* } */
+
+ c = csd[numsets].c2w[i];
+
+ if (c == 0) { /* non-existant char in codeset */
+ d = __CTYPE_unclassified;
+ } else if (iswdigit(c)) {
+ d = __CTYPE_digit;
+ } else if (iswalpha(c)) {
+ d = __CTYPE_alpha_nonupper_nonlower;
+ if (iswlower(c)) {
+ d = __CTYPE_alpha_lower;
+ if (iswupper(c)) {
+ d = __CTYPE_alpha_upper_lower;
+ }
+ } else if (iswupper(c)) {
+ d = __CTYPE_alpha_upper;
+ }
+ } else if (iswpunct(c)) {
+ d = __CTYPE_punct;
+ } else if (iswgraph(c)) {
+ d = __CTYPE_graph;
+ } else if (iswprint(c)) {
+ d = __CTYPE_print_space_nonblank;
+ if (iswblank(c)) {
+ d = __CTYPE_print_space_blank;
+ }
+ } else if (iswspace(c) && !iswcntrl(c)) {
+ d = __CTYPE_space_nonblank_noncntrl;
+ if (iswblank(c)) {
+ d = __CTYPE_space_blank_noncntrl;
+ }
+ } else if (iswcntrl(c)) {
+ d = __CTYPE_cntrl_nonspace;
+ if (iswspace(c)) {
+ d = __CTYPE_cntrl_space_nonblank;
+ if (iswblank(c)) {
+ d = __CTYPE_cntrl_space_blank;
+ }
+ }
+ } else {
+ d = __CTYPE_unclassified;
+ }
+
+#if 1
+ row[i & (CTYPE_ROW_LEN-1)] = d;
+ if ((i & (CTYPE_ROW_LEN-1)) == (CTYPE_ROW_LEN-1)) {
+ p = ctype_tbl;
+ for (j=0 ; j < n_ctype_rows ; j++) {
+ if (!memcmp(p, row, CTYPE_ROW_LEN)) {
+ break;
+ }
+ p += CTYPE_ROW_LEN;
+ }
+ if (j == n_ctype_rows) { /* new entry */
+ if (++n_ctype_rows > 256) {
+ printf("error -- to many ctype rows!\n");
+ return EXIT_FAILURE;
+ }
+ memcpy(p, row, CTYPE_ROW_LEN);
+ }
+ csd[numsets].ctype_idx[i >> CTYPE_IDX_SHIFT] = j;
+ if (!((i >> CTYPE_IDX_SHIFT) & 0x7)
+ && (i != (127 + CTYPE_ROW_LEN))
+ ) {
+ fprintf(out, "\n\t\t ");
+ }
+ fprintf(out, " %#4x,", j);
+ }
+#else
+ fprintf(out, " %#4x,", d);
+#endif
+ }
+#endif
+ fprintf(out, " }");
+
+#if 1
+ fprintf(out, ",\n\t\t/* idx8uplow data */\n\t\t{");
+ for (i = 128 ; i < 256 ; i++) {
+ wchar_t c, u, l;
+/* if (!(i & 0x7)) { */
+/* fprintf(out, "\n"); */
+/* } */
+ c = csd[numsets].c2w[i];
+ if ((c != 0) || 1) {
+ u = towupper(c);
+ l = towlower(c);
+
+ if (u >= 0x80) u = csd[numsets].w2c[u];
+ if (l >= 0x80) l = csd[numsets].w2c[l];
+
+ if (u == 0) u = i; /* upper is missing, so ignore */
+ if (l == 0) l = i; /* lower is missing, so ignore */
+
+#if 1
+ /* store as unsigned char and let overflow handle it. */
+/* if ((((u-i) < CHAR_MIN) || ((u-i) > CHAR_MAX)) */
+/* || (((i-l) < CHAR_MIN) || ((i-l) > CHAR_MAX)) */
+/* ) { */
+/* printf("error - uplow diff out of range! %d %ld %ld\n", */
+/* i, u, l); */
+/* return EXIT_FAILURE; */
+/* } */
+
+ row[i & (UPLOW_ROW_LEN-1)] = ((l==i) ? (u-i) : (i-l));
+ if ((i & (UPLOW_ROW_LEN-1)) == (UPLOW_ROW_LEN-1)) {
+ p = uplow_tbl;
+ for (j=0 ; j < n_uplow_rows ; j++) {
+ if (!memcmp(p, row, UPLOW_ROW_LEN)) {
+ break;
+ }
+ p += UPLOW_ROW_LEN;
+ }
+ if (j == n_uplow_rows) { /* new entry */
+ if (++n_uplow_rows > 256) {
+ printf("error -- to many uplow rows!\n");
+ return EXIT_FAILURE;
+ }
+ memcpy(p, row, UPLOW_ROW_LEN);
+ }
+ csd[numsets].uplow_idx[i >> UPLOW_IDX_SHIFT] = j;
+ if (!((i >> UPLOW_IDX_SHIFT) & 0x7)
+ && (i != (127 + UPLOW_ROW_LEN))
+ ) {
+ fprintf(out, "\n\t\t ");
+ }
+ fprintf(out, " %#4x,", j);
+ }
+
+#elif 0
+ if (!(i & 0x7) && i) {
+ fprintf(out, "\n");
+ }
+ fprintf(out, " %4ld,", (l==i) ? (u-i) : (i-l));
+/* fprintf(out, " %4ld,", (l==i) ? u : l); */
+#else
+ if ((u != i) || (l != i)) {
+#if 0
+ fprintf(out, " %#08lx, %#08lx, %#08lx, %#08lx, %#08lx, %#08lx, \n",
+ (unsigned long) i,
+ (unsigned long) c,
+ (unsigned long) l,
+ (unsigned long) towlower(c),
+ (unsigned long) u,
+ (unsigned long) towupper(c));
+
+#else
+ fprintf(out, " %#08lx, %8ld, %d, %8ld, %d, %#08lx\n",
+ (unsigned long) i,
+ (long) (l - i),
+ iswupper(c),
+ (long) (i - u),
+ iswlower(c),
+ (unsigned long) c);
+#endif
+ }
+#endif
+ }
+ }
+ fprintf(out, " }");
+#endif
+
+#ifndef DO_WIDE_CHAR
+ fprintf(out,"\n");
+#else /* DO_WIDE_CHAR */
+
+#if 1
+ fprintf(out, ",\n\t\t/* idx8c2wc data */\n\t\t{");
+ for (i = 128 ; i < 256 ; i++) {
+#if 1
+ wrow[i & (C2WC_ROW_LEN-1)] = csd[numsets].c2w[i];
+ if ((i & (C2WC_ROW_LEN-1)) == (C2WC_ROW_LEN-1)) {
+ p = (char *) c2wc_tbl;
+ for (j=0 ; j < n_c2wc_rows ; j++) {
+ if (!memcmp(p, (char *) wrow, 2*C2WC_ROW_LEN)) {
+ break;
+ }
+ p += 2*C2WC_ROW_LEN;
+ }
+ if (j == n_c2wc_rows) { /* new entry */
+ if (++n_c2wc_rows > 256) {
+ printf("error -- to many c2wc rows!\n");
+ return EXIT_FAILURE;
+ }
+ memcpy(p, (char *) wrow, 2*C2WC_ROW_LEN);
+ }
+ csd[numsets].c2wc_idx[i >> C2WC_IDX_SHIFT] = j;
+ if (!((i >> C2WC_IDX_SHIFT) & 0x7)
+ && (i != (127 + C2WC_ROW_LEN))
+ ) {
+ fprintf(out, "\n\t\t ");
+ }
+ fprintf(out, " %#4x,", j);
+ }
+#else
+ if (!(i & 0x7) && i) {
+ fprintf(out, "\n");
+ }
+ fprintf(out, " %#6lx,", csd[numsets].c2w[i]);
+#endif
+ }
+ fprintf(out, " },\n");
+#endif
+
+#if 1
+/* fprintf(out, "\nII_LEN = %d\n", II_LEN); */
+ fprintf(out, "\t\t/* idx8wc2c data */\n\t\t{");
+ for (i = 0 ; i < II_LEN ; i++) {
+ if (!(i & 0x7) && i) {
+ fprintf(out, "\n\t\t ");
+ }
+ fprintf(out, " %#4x,", csd[numsets].ii[i]);
+ }
+ fprintf(out, " }\n");
+#endif
+
+#endif /* DO_WIDE_CHAR */
+ fprintf(out, "\t},\n");
+
+ }
+ ++numsets;
+ printf("done\n");
+ }
+ fprintf(out, "};\n");
+ fprintf(out, "\n#endif /* WANT_DATA */\n");
+
+#ifdef DO_WIDE_CHAR
+ fprintf(out, "\n");
+ fprintf(out, "#define Cwc2c_DOMAIN_MAX\t%#x\n", RANGE);
+ fprintf(out, "#define Cwc2c_TI_SHIFT\t\t%d\n", TI_SHIFT);
+ fprintf(out, "#define Cwc2c_TT_SHIFT\t\t%d\n", TT_SHIFT);
+ fprintf(out, "#define Cwc2c_II_LEN\t\t%d\n", II_LEN);
+ fprintf(out, "#define Cwc2c_TI_LEN\t\t%d\n", ti_num << TI_SHIFT);
+ fprintf(out, "#define Cwc2c_TT_LEN\t\t%d\n", tt_num << TT_SHIFT);
+ fprintf(out, "\n");
+
+ fprintf(out, "\n#define Cwc2c_TBL_LEN\t\t%d\n",
+ (ti_num << TI_SHIFT) + (tt_num << TT_SHIFT));
+
+ fprintf(out, "#ifdef WANT_DATA\n\n");
+ fprintf(out, "static const unsigned char Cwc2c_data[%d] = {\n",
+ (ti_num << TI_SHIFT) + (tt_num << TT_SHIFT));
+ fprintf(out, "\t/* ti_table */\n\t");
+ for (i=0 ; i < ti_num << TI_SHIFT ; i++) {
+ if (!(i & 7) && i) {
+ fprintf(out, "\n\t");
+ }
+ fprintf(out, " %#4x,", ti[i]);
+ }
+ fprintf(out, "\n");
+ fprintf(out, "\t/* tt_table */\n\t");
+ for (i=0 ; i < tt_num << TT_SHIFT ; i++) {
+ if (!(i & 7) && i) {
+ fprintf(out, "\n\t");
+ }
+ fprintf(out, " %#4x,", tt[i]);
+ }
+ fprintf(out, "\n};\n");
+
+ fprintf(out, "\n#endif /* WANT_DATA */\n");
+#endif /* DO_WIDE_CHAR */
+
+ fprintf(out, "\n#define Cuplow_TBL_LEN\t\t%d\n",
+ n_uplow_rows * UPLOW_ROW_LEN);
+ fprintf(out, "\n#ifdef WANT_DATA\n\n");
+
+ fprintf(out, "\nstatic const unsigned char Cuplow_data[%d] = {\n",
+ n_uplow_rows * UPLOW_ROW_LEN);
+ p = uplow_tbl;
+ for (j=0 ; j < n_uplow_rows ; j++) {
+ fprintf(out, "\t");
+ for (i=0 ; i < UPLOW_ROW_LEN ; i++) {
+ fprintf(out, " %#4x,", (unsigned int)((unsigned char) p[i]));
+ }
+ fprintf(out, "\n");
+ p += UPLOW_ROW_LEN;
+ }
+ fprintf(out, "};\n");
+
+ fprintf(out, "\n#endif /* WANT_DATA */\n");
+ fprintf(out, "\n#define Cctype_TBL_LEN\t\t%d\n",
+#ifdef CTYPE_PACKED
+ n_ctype_rows * CTYPE_ROW_LEN / 2
+#else
+ n_ctype_rows * CTYPE_ROW_LEN
+#endif
+ );
+ fprintf(out, "\n#ifdef WANT_DATA\n\n");
+
+
+ fprintf(out, "\nstatic const unsigned char Cctype_data[%d] = {\n",
+#ifdef CTYPE_PACKED
+ n_ctype_rows * CTYPE_ROW_LEN / 2
+#else
+ n_ctype_rows * CTYPE_ROW_LEN
+#endif
+ );
+ p = ctype_tbl;
+ for (j=0 ; j < n_ctype_rows ; j++) {
+ fprintf(out, "\t");
+ for (i=0 ; i < CTYPE_ROW_LEN ; i++) {
+#ifdef CTYPE_PACKED
+ fprintf(out, " %#4x,", (unsigned int)(p[i] + (p[i+1] << 4)));
+ ++i;
+#else
+ fprintf(out, " %#4x,", (unsigned int)p[i]);
+#endif
+ }
+ fprintf(out, "\n");
+ p += CTYPE_ROW_LEN;
+ }
+ fprintf(out, "};\n");
+
+ fprintf(out, "\n#endif /* WANT_DATA */\n");
+
+#ifdef DO_WIDE_CHAR
+
+ fprintf(out, "\n#define Cc2wc_TBL_LEN\t\t%d\n",
+ n_c2wc_rows * C2WC_ROW_LEN);
+ fprintf(out, "\n#ifdef WANT_DATA\n\n");
+
+ fprintf(out, "\nstatic const unsigned short Cc2wc_data[%d] = {\n",
+ n_c2wc_rows * C2WC_ROW_LEN);
+ p = (char *) c2wc_tbl;
+ for (j=0 ; j < n_c2wc_rows ; j++) {
+ fprintf(out, "\t");
+ for (i=0 ; i < C2WC_ROW_LEN ; i++) {
+ fprintf(out, " %#6x,", (unsigned int)(((unsigned short *)p)[i]));
+ }
+ fprintf(out, "\n");
+ p += 2*C2WC_ROW_LEN;
+ }
+ fprintf(out, "};\n");
+ fprintf(out, "\n#endif /* WANT_DATA */\n");
+#endif /* DO_WIDE_CHAR */
+ fprintf(out, "\n\n");
+
+ fprintf(out, "#define NUM_CODESETS\t\t%d\n", numsets);
+ fprintf(out, "#define CODESET_LIST \\\n\t\"");
+ for (i=0 ; i < numsets ; i++) {
+ fprintf(out, "\\x%02x", numsets + 1 + (unsigned char) codeset_index[i]);
+ if (((i & 7) == 7) && (i + 1 < numsets)) {
+ fprintf(out, "\" \\\n\t\"");
+ }
+ }
+ fprintf(out, "\" \\\n\t\"\\0\"");
+ for (i=0 ; i < numsets ; i++) {
+ fprintf(out, " \\\n\t\"%s\\0\"",
+ codeset_list + ((unsigned char)codeset_index[i]));
+ }
+
+ fprintf(out, "\n\n");
+ for (i=0 ; i < numsets ; i++) {
+ char buf[30];
+ char *z;
+ strcpy(buf, codeset_list + ((unsigned char)codeset_index[i]));
+ for (z=buf ; *z ; z++) {
+ if (*z == '-') {
+ *z = '_';
+ }
+ }
+ fprintf(out, "#define __CTYPE_HAS_CODESET_%s\n", buf);
+ }
+#ifdef DO_WIDE_CHAR
+ fprintf(out, "#define __CTYPE_HAS_CODESET_UTF_8\n");
+#endif /* DO_WIDE_CHAR */
+
+#if 0
+ fprintf(out, "\n#endif /* __CTYPE_HAS_8_BIT_LOCALES */\n\n");
+#endif
+
+ fclose(out);
+
+ total_size = 0;
+#ifdef DO_WIDE_CHAR
+ printf("tt_num = %d ti_num = %d\n", tt_num, ti_num);
+ printf("max_wchar = %#lx\n", max_wchar);
+
+ printf("size is %d * %d + %d * %d + %d * %d = %d\n",
+ tt_num, 1 << TT_SHIFT, ti_num, 1 << TI_SHIFT,
+ ((MAX_WCHAR >> (TT_SHIFT + TI_SHIFT)) + 1), numsets,
+ j = tt_num * (1 << TT_SHIFT) + ti_num * (1 << TI_SHIFT)
+ + ((MAX_WCHAR >> (TT_SHIFT + TI_SHIFT)) + 1) * numsets);
+ total_size += j;
+#endif /* DO_WIDE_CHAR */
+
+#ifdef CTYPE_PACKED
+ i = 2;
+#else
+ i = 1;
+#endif
+
+ printf("ctype - CTYPE_IDX_SHIFT = %d -- %d * %d + %d * %d = %d\n",
+ CTYPE_IDX_SHIFT, numsets, CTYPE_IDX_LEN, n_ctype_rows, CTYPE_ROW_LEN / i,
+ j = numsets * CTYPE_IDX_LEN + n_ctype_rows * CTYPE_ROW_LEN / i);
+ total_size += j;
+
+ printf("uplow - UPLOW_IDX_SHIFT = %d -- %d * %d + %d * %d = %d\n",
+ UPLOW_IDX_SHIFT, numsets, UPLOW_IDX_LEN, n_uplow_rows, UPLOW_ROW_LEN,
+ j = numsets * UPLOW_IDX_LEN + n_uplow_rows * UPLOW_ROW_LEN);
+ total_size += j;
+
+#ifdef DO_WIDE_CHAR
+
+ printf("c2wc - C2WC_IDX_SHIFT = %d -- %d * %d + 2 * %d * %d = %d\n",
+ C2WC_IDX_SHIFT, numsets, C2WC_IDX_LEN, n_c2wc_rows, C2WC_ROW_LEN,
+ j = numsets * C2WC_IDX_LEN + 2 * n_c2wc_rows * C2WC_ROW_LEN);
+ total_size += j;
+
+#endif /* DO_WIDE_CHAR */
+
+ printf("total size = %d\n", total_size);
+
+/* for (i=0 ; i < numsets ; i++) { */
+/* printf("codeset_index[i] = %d codeset_list[ci[i]] = \"%s\"\n", */
+/* (unsigned char) codeset_index[i], */
+/* codeset_list + ((unsigned char)codeset_index[i])); */
+/* } */
+
+ return EXIT_SUCCESS;
+}
diff --git a/extra/locale/gen_wctype.c b/extra/locale/gen_wctype.c
new file mode 100644
index 000000000..9508a3bbf
--- /dev/null
+++ b/extra/locale/gen_wctype.c
@@ -0,0 +1,833 @@
+
+#define _GNU_SOURCE
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <locale.h>
+#include <wctype.h>
+#include <limits.h>
+#include <stdint.h>
+#include <wchar.h>
+#include <ctype.h>
+
+
+/* 0x9 : space blank */
+/* 0xa : space */
+/* 0xb : space */
+/* 0xc : space */
+/* 0xd : space */
+/* 0x20 : space blank */
+/* 0x1680 : space blank */
+/* 0x2000 : space blank */
+/* 0x2001 : space blank */
+/* 0x2002 : space blank */
+/* 0x2003 : space blank */
+/* 0x2004 : space blank */
+/* 0x2005 : space blank */
+/* 0x2006 : space blank */
+/* 0x2008 : space blank */
+/* 0x2009 : space blank */
+/* 0x200a : space blank */
+/* 0x200b : space blank */
+/* 0x2028 : space */
+/* 0x2029 : space */
+/* 0x3000 : space blank */
+
+/* typecount[ 0] = 88670 C_alpha_nonupper_nonlower */
+/* typecount[ 1] = 742 C_alpha_lower */
+/* typecount[ 2] = 4 C_alpha_upper_lower */
+/* typecount[ 3] = 731 C_alpha_upper */
+/* typecount[ 4] = 10 C_digit */
+/* typecount[ 5] = 10270 C_punct */
+/* typecount[ 6] = 0 C_graph */
+/* typecount[ 7] = 0 C_print_space_nonblank */
+/* typecount[ 8] = 14 C_print_space_blank */
+/* typecount[ 9] = 0 C_space_nonblank_noncntrl */
+/* typecount[10] = 0 C_space_blank_noncntrl */
+/* typecount[11] = 6 C_cntrl_space_nonblank */
+/* typecount[12] = 1 C_cntrl_space_blank */
+/* typecount[13] = 60 C_cntrl_nonspace */
+/* typecount[14] = 96100 C_unclassified */
+/* typecount[15] = 0 empty_slot */
+
+
+
+/* Set to #if 0 to restrict wchars to 16 bits. */
+#if 1
+#define RANGE 0x2ffffUL
+#elif 0
+#define RANGE 0x1ffffUL
+#else
+#define RANGE 0xffffUL /* Restrict for 16-bit wchar_t... */
+#endif
+
+/* Classification codes. */
+
+static const char *typename[] = {
+ "C_unclassified",
+ "C_alpha_nonupper_nonlower",
+ "C_alpha_lower",
+ "C_alpha_upper_lower",
+ "C_alpha_upper",
+ "C_digit",
+ "C_punct",
+ "C_graph",
+ "C_print_space_nonblank",
+ "C_print_space_blank",
+ "C_space_nonblank_noncntrl",
+ "C_space_blank_noncntrl",
+ "C_cntrl_space_nonblank",
+ "C_cntrl_space_blank",
+ "C_cntrl_nonspace",
+ "empty_slot"
+};
+
+/* Taking advantage of the C99 mutual-exclusion guarantees for the various
+ * (w)ctype classes, including the descriptions of printing and control
+ * (w)chars, we can place each in one of the following mutually-exlusive
+ * subsets. Since there are less than 16, we can store the data for
+ * each (w)chars in a nibble. In contrast, glibc uses an unsigned int
+ * per (w)char, with one bit flag for each is* type. While this allows
+ * a simple '&' operation to determine the type vs. a range test and a
+ * little special handling for the "blank" and "xdigit" types in my
+ * approach, it also uses 8 times the space for the tables on the typical
+ * 32-bit archs we supported.*/
+enum {
+ __CTYPE_unclassified = 0,
+ __CTYPE_alpha_nonupper_nonlower,
+ __CTYPE_alpha_lower,
+ __CTYPE_alpha_upper_lower,
+ __CTYPE_alpha_upper,
+ __CTYPE_digit,
+ __CTYPE_punct,
+ __CTYPE_graph,
+ __CTYPE_print_space_nonblank,
+ __CTYPE_print_space_blank,
+ __CTYPE_space_nonblank_noncntrl,
+ __CTYPE_space_blank_noncntrl,
+ __CTYPE_cntrl_space_nonblank,
+ __CTYPE_cntrl_space_blank,
+ __CTYPE_cntrl_nonspace,
+};
+
+/* Some macros that test for various (w)ctype classes when passed one of the
+ * designator values enumerated above. */
+#define __CTYPE_isalnum(D) ((unsigned int)(D-1) <= (__CTYPE_digit-1))
+#define __CTYPE_isalpha(D) ((unsigned int)(D-1) <= (__CTYPE_alpha_upper-1))
+#define __CTYPE_isblank(D) \
+ ((((unsigned int)(D - __CTYPE_print_space_nonblank)) <= 5) && (D & 1))
+#define __CTYPE_iscntrl(D) (((unsigned int)(D - __CTYPE_cntrl_space_nonblank)) <= 2)
+#define __CTYPE_isdigit(D) (D == __CTYPE_digit)
+#define __CTYPE_isgraph(D) ((unsigned int)(D-1) <= (__CTYPE_graph-1))
+#define __CTYPE_islower(D) (((unsigned int)(D - __CTYPE_alpha_lower)) <= 1)
+#define __CTYPE_isprint(D) ((unsigned int)(D-1) <= (__CTYPE_print_space_blank-1))
+#define __CTYPE_ispunct(D) (D == __CTYPE_punct)
+#define __CTYPE_isspace(D) (((unsigned int)(D - __CTYPE_print_space_nonblank)) <= 5)
+#define __CTYPE_isupper(D) (((unsigned int)(D - __CTYPE_alpha_upper_lower)) <= 1)
+#define __CTYPE_isxdigit(D,X) \
+ (__CTYPE_isdigit(D) || (((unsigned int)(((X)|0x20) - 'a')) <= 5))
+
+#define mywalnum(x) __CTYPE_isalnum(d)
+#define mywalpha(x) __CTYPE_isalpha(d)
+#define mywblank(x) __CTYPE_isblank(d)
+#define mywcntrl(x) __CTYPE_iscntrl(d)
+#define mywdigit(x) __CTYPE_isdigit(d)
+#define mywgraph(x) __CTYPE_isgraph(d)
+#define mywlower(x) __CTYPE_islower(d)
+#define mywprint(x) __CTYPE_isprint(d)
+#define mywpunct(x) __CTYPE_ispunct(d)
+#define mywspace(x) __CTYPE_isspace(d)
+#define mywupper(x) __CTYPE_isupper(d)
+#define mywxdigit(x) __CTYPE_isxdigit(d,x)
+
+typedef struct {
+ short l;
+ short u;
+} uldiff_entry;
+
+typedef struct {
+ uint16_t ii_len;
+ uint16_t ti_len;
+ uint16_t ut_len;
+
+ unsigned char ii_shift;
+ unsigned char ti_shift;
+
+ unsigned char *ii;
+ unsigned char *ti;
+ unsigned char *ut;
+} table_data;
+
+
+void output_table(FILE *fp, const char *name, table_data *tbl)
+{
+ size_t i;
+
+ fprintf(fp, "#define WC%s_II_LEN %7u\n", name, tbl->ii_len);
+ fprintf(fp, "#define WC%s_TI_LEN %7u\n", name, tbl->ti_len);
+ fprintf(fp, "#define WC%s_UT_LEN %7u\n", name, tbl->ut_len);
+
+ fprintf(fp, "#define WC%s_II_SHIFT %7u\n", name, tbl->ii_shift);
+ fprintf(fp, "#define WC%s_TI_SHIFT %7u\n", name, tbl->ti_shift);
+
+ fprintf(fp, "\n#ifdef WANT_WC%s_data\n", name);
+
+ i = tbl->ii_len + tbl->ti_len + tbl->ut_len;
+ fprintf(fp, "\nstatic const unsigned char WC%s_data[%zu] = {", name, i);
+ for (i=0 ; i < tbl->ii_len ; i++) {
+ if (i % 12 == 0) {
+ fprintf(fp, "\n");
+ }
+ fprintf(fp, " %#04x,", tbl->ii[i]);
+ }
+ for (i=0 ; i < tbl->ti_len ; i++) {
+ if (i % 12 == 0) {
+ fprintf(fp, "\n");
+ }
+ fprintf(fp, " %#04x,", tbl->ti[i]);
+ }
+ for (i=0 ; i < tbl->ut_len ; i++) {
+ if (i % 12 == 0) {
+ fprintf(fp, "\n");
+ }
+ fprintf(fp, " %#04x,", tbl->ut[i]);
+ }
+ fprintf(fp, "\n};\n\n");
+
+ fprintf(fp, "#endif /* WANT_WC%s_data */\n\n", name);
+}
+
+static void dump_table_data(table_data *tbl)
+{
+ printf("ii_shift = %d ti_shift = %d\n"
+ "ii_len = %d ti_len = %d ut_len = %d\n"
+ "total = %d\n",
+ tbl->ii_shift, tbl->ti_shift,
+ tbl->ii_len, tbl->ti_len, tbl->ut_len,
+ (int) tbl->ii_len + (int) tbl->ti_len + (int) tbl->ut_len);
+}
+
+/* For sorting the blocks of unsigned chars. */
+static size_t nu_val;
+
+int nu_memcmp(const void *a, const void *b)
+{
+ return memcmp(*(unsigned char**)a, *(unsigned char**)b, nu_val);
+}
+
+static size_t newopt(unsigned char *ut, size_t usize, int shift, table_data *tbl);
+
+#define MAXTO 255 /* Restrict to minimal unsigned char max. */
+
+int main(int argc, char **argv)
+{
+ long int u, l, tt;
+ size_t smallest, t;
+ unsigned int c;
+ unsigned int d;
+ int i, n;
+ int ul_count = 0;
+ uldiff_entry uldiff[MAXTO];
+ table_data cttable;
+ table_data ultable;
+ table_data combtable;
+ table_data widthtable;
+
+ unsigned char wct[(RANGE/2)+1]; /* wctype table (nibble per wchar) */
+ unsigned char ult[RANGE+1]; /* upper/lower table */
+ unsigned char combt[(RANGE/4)+1]; /* combining */
+ unsigned char widtht[(RANGE/4)+1]; /* width */
+ wctrans_t totitle;
+ wctype_t is_comb, is_comb3;
+
+ long int typecount[16];
+ int built = 0;
+
+ setvbuf(stdout, NULL, _IONBF, 0);
+
+ while (--argc) {
+ if (!setlocale(LC_CTYPE, *++argv)) {
+ printf("setlocale(LC_CTYPE,%s) failed!\n", *argv);
+ continue;
+ }
+
+ if (!(totitle = wctrans("totitle"))) {
+ printf("no totitle transformation.\n");
+ }
+ if (!(is_comb = wctype("combining"))) {
+ printf("no combining wctype.\n");
+ }
+ if (!(is_comb3 = wctype("combining_level3"))) {
+ printf("no combining_level3 wctype.\n");
+ }
+
+ if (!built) {
+ built = 1;
+ ul_count = 1;
+ uldiff[0].u = uldiff[0].l = 0;
+
+ memset(wct, 0, sizeof(wct));
+ memset(combt, 0, sizeof(combt));
+ memset(widtht, 0, sizeof(widtht));
+
+ for (i = 0 ; i < 16 ; i++) {
+ typecount[i] = 0;
+ }
+
+ for (c=0 ; c <= RANGE ; c++) {
+ if (iswdigit(c)) {
+ d = __CTYPE_digit;
+ } else if (iswalpha(c)) {
+ d = __CTYPE_alpha_nonupper_nonlower;
+ if (iswlower(c)) {
+ d = __CTYPE_alpha_lower;
+ if (iswupper(c)) {
+ d = __CTYPE_alpha_upper_lower;
+ }
+ } else if (iswupper(c)) {
+ d = __CTYPE_alpha_upper;
+ }
+ } else if (iswpunct(c)) {
+ d = __CTYPE_punct;
+ } else if (iswgraph(c)) {
+ d = __CTYPE_graph;
+ } else if (iswprint(c)) {
+ d = __CTYPE_print_space_nonblank;
+ if (iswblank(c)) {
+ d = __CTYPE_print_space_blank;
+ }
+ } else if (iswspace(c) && !iswcntrl(c)) {
+ d = __CTYPE_space_nonblank_noncntrl;
+ if (iswblank(c)) {
+ d = __CTYPE_space_blank_noncntrl;
+ }
+ } else if (iswcntrl(c)) {
+ d = __CTYPE_cntrl_nonspace;
+ if (iswspace(c)) {
+ d = __CTYPE_cntrl_space_nonblank;
+ if (iswblank(c)) {
+ d = __CTYPE_cntrl_space_blank;
+ }
+ }
+ } else {
+ d = __CTYPE_unclassified;
+ }
+
+ ++typecount[d];
+
+#if 0
+ if (iswspace(c)) {
+ if (iswblank(c)) {
+ printf("%#8x : space blank\n", c);
+ } else {
+ printf("%#8x : space\n", c);
+ }
+ }
+#endif
+
+#if 0
+ if (c < 256) {
+ unsigned int glibc;
+
+ glibc = 0;
+ if (isalnum(c)) ++glibc; glibc <<= 1;
+ if (isalpha(c)) ++glibc; glibc <<= 1;
+ if (isblank(c)) ++glibc; glibc <<= 1;
+ if (iscntrl(c)) ++glibc; glibc <<= 1;
+ if (isdigit(c)) ++glibc; glibc <<= 1;
+ if (isgraph(c)) ++glibc; glibc <<= 1;
+ if (islower(c)) ++glibc; glibc <<= 1;
+ if (isprint(c)) ++glibc; glibc <<= 1;
+ if (ispunct(c)) ++glibc; glibc <<= 1;
+ if (isspace(c)) ++glibc; glibc <<= 1;
+ if (isupper(c)) ++glibc; glibc <<= 1;
+ if (isxdigit(c)) ++glibc;
+ printf("%#8x : ctype %#4x\n", c, glibc);
+ }
+#endif
+
+#if 1
+ /* Paranoid checking... */
+ {
+ unsigned int glibc;
+ unsigned int mine;
+
+ glibc = 0;
+ if (iswalnum(c)) ++glibc; glibc <<= 1;
+ if (iswalpha(c)) ++glibc; glibc <<= 1;
+ if (iswblank(c)) ++glibc; glibc <<= 1;
+ if (iswcntrl(c)) ++glibc; glibc <<= 1;
+ if (iswdigit(c)) ++glibc; glibc <<= 1;
+ if (iswgraph(c)) ++glibc; glibc <<= 1;
+ if (iswlower(c)) ++glibc; glibc <<= 1;
+ if (iswprint(c)) ++glibc; glibc <<= 1;
+ if (iswpunct(c)) ++glibc; glibc <<= 1;
+ if (iswspace(c)) ++glibc; glibc <<= 1;
+ if (iswupper(c)) ++glibc; glibc <<= 1;
+ if (iswxdigit(c)) ++glibc;
+
+ mine = 0;
+ if (mywalnum(c)) ++mine; mine <<= 1;
+ if (mywalpha(c)) ++mine; mine <<= 1;
+ if (mywblank(c)) ++mine; mine <<= 1;
+ if (mywcntrl(c)) ++mine; mine <<= 1;
+ if (mywdigit(c)) ++mine; mine <<= 1;
+ if (mywgraph(c)) ++mine; mine <<= 1;
+ if (mywlower(c)) ++mine; mine <<= 1;
+ if (mywprint(c)) ++mine; mine <<= 1;
+ if (mywpunct(c)) ++mine; mine <<= 1;
+ if (mywspace(c)) ++mine; mine <<= 1;
+ if (mywupper(c)) ++mine; mine <<= 1;
+ if (mywxdigit(c)) ++mine;
+
+ if (glibc != mine) {
+ printf("%#8x : glibc %#4x != %#4x mine %u\n", c, glibc, mine, d);
+ return EXIT_FAILURE;
+ }
+
+#if 0
+ if (iswctype(c,is_comb) || iswctype(c,is_comb3)) {
+/* if (!iswpunct(c)) { */
+ printf("%#8x : %d %d %#4x\n",
+ c, iswctype(c,is_comb),iswctype(c,is_comb3), glibc);
+/* } */
+ }
+#endif
+
+ }
+#endif
+
+ combt[c/4] |= ((((!!iswctype(c,is_comb)) << 1) | !!iswctype(c,is_comb3))
+ << ((c & 3) << 1));
+/* comb3t[c/8] |= ((!!iswctype(c,is_comb3)) << (c & 7)); */
+ widtht[c/4] |= (wcwidth(c) << ((c & 3) << 1));
+
+ if (c & 1) { /* Use the high nibble for odd numbered wchars. */
+ d <<= 4;
+ }
+ wct[c/2] |= d;
+
+ l = towlower(c) - c;
+ u = towupper(c) - c;
+ ult[c] = 0;
+ if (l || u) {
+ if ((l != (short)l) || (u != (short)u)) {
+ printf("range assumption error! %x %ld %ld\n", c, l, u);
+ return EXIT_FAILURE;
+ }
+ for (i=0 ; i < ul_count ; i++) {
+ if ((l == uldiff[i].l) && (u == uldiff[i].u)) {
+ goto found;
+ }
+ }
+ uldiff[ul_count].l = l;
+ uldiff[ul_count].u = u;
+ ++ul_count;
+ if (ul_count > MAXTO) {
+ printf("too many touppers/tolowers!\n");
+ return EXIT_FAILURE;
+ }
+ found:
+ ult[c] = i;
+ }
+ }
+
+ for (i = 0 ; i < 16 ; i++) {
+ printf("typecount[%2d] = %8ld %s\n", i, typecount[i], typename[i]);
+ }
+
+ printf("optimizing is* table..\n");
+ n = -1;
+ smallest = SIZE_MAX;
+ cttable.ii = NULL;
+ for (i=0 ; i < 14 ; i++) {
+ t = newopt(wct, (RANGE/2)+1, i, &cttable);
+ if (smallest >= t) {
+ n = i;
+ smallest = t;
+/* } else { */
+/* break; */
+ }
+ }
+ printf("smallest = %zu\n", smallest);
+ if (!(cttable.ii = malloc(smallest))) {
+ printf("couldn't allocate space!\n");
+ return EXIT_FAILURE;
+ }
+ smallest = SIZE_MAX;
+ newopt(wct, (RANGE/2)+1, n, &cttable);
+ ++cttable.ti_shift; /* correct for nibble mode */
+
+
+
+ printf("optimizing u/l-to table..\n");
+ smallest = SIZE_MAX;
+ ultable.ii = NULL;
+ for (i=0 ; i < 14 ; i++) {
+ t = newopt(ult, RANGE+1, i, &ultable);
+ if (smallest >= t) {
+ n = i;
+ smallest = t;
+/* } else { */
+/* break; */
+ }
+ }
+ printf("%zu (smallest) + %zu (u/l diffs) = %zu\n",
+ smallest, 4 * ul_count, smallest + 4 * ul_count);
+ printf("smallest = %zu\n", smallest);
+ if (!(ultable.ii = malloc(smallest))) {
+ printf("couldn't allocate space!\n");
+ return EXIT_FAILURE;
+ }
+ smallest = SIZE_MAX;
+ newopt(ult, RANGE+1, n, &ultable);
+
+
+ printf("optimizing comb table..\n");
+ smallest = SIZE_MAX;
+ combtable.ii = NULL;
+ for (i=0 ; i < 14 ; i++) {
+ t = newopt(combt, sizeof(combt), i, &combtable);
+ if (smallest >= t) {
+ n = i;
+ smallest = t;
+/* } else { */
+/* break; */
+ }
+ }
+ printf("smallest = %zu\n", smallest);
+ if (!(combtable.ii = malloc(smallest))) {
+ printf("couldn't allocate space!\n");
+ return EXIT_FAILURE;
+ }
+ smallest = SIZE_MAX;
+ newopt(combt, sizeof(combt), n, &combtable);
+ combtable.ti_shift += 4; /* correct for 4 entries per */
+
+
+ printf("optimizing width table..\n");
+ smallest = SIZE_MAX;
+ widthtable.ii = NULL;
+ for (i=0 ; i < 14 ; i++) {
+ t = newopt(widtht, sizeof(widtht), i, &widthtable);
+ if (smallest >= t) {
+ n = i;
+ smallest = t;
+/* } else { */
+/* break; */
+ }
+ }
+ printf("smallest = %zu\n", smallest);
+ if (!(widthtable.ii = malloc(smallest))) {
+ printf("couldn't allocate space!\n");
+ return EXIT_FAILURE;
+ }
+ smallest = SIZE_MAX;
+ newopt(widtht, sizeof(widtht), n, &widthtable);
+ widthtable.ti_shift += 4; /* correct for 4 entries per */
+
+
+#if 0
+ printf("optimizing comb3 table..\n");
+ smallest = SIZE_MAX;
+ comb3table.ii = NULL;
+ for (i=0 ; i < 14 ; i++) {
+ t = newopt(comb3t, sizeof(comb3t), i, &comb3table);
+ if (smallest >= t) {
+ n = i;
+ smallest = t;
+/* } else { */
+/* break; */
+ }
+ }
+ printf("smallest = %zu\n", smallest);
+ if (!(comb3table.ii = malloc(smallest))) {
+ printf("couldn't allocate space!\n");
+ return EXIT_FAILURE;
+ }
+ smallest = SIZE_MAX;
+ newopt(comb3t, sizeof(comb3t), n, &comb3table);
+ comb3table.ti_shift += 8; /* correct for 4 entries per */
+#endif
+
+ dump_table_data(&cttable);
+ dump_table_data(&ultable);
+ dump_table_data(&combtable);
+ dump_table_data(&widthtable);
+ }
+
+ printf("verifying for %s...\n", *argv);
+#if RANGE == 0xffffU
+ for (c=0 ; c <= 0xffffUL ; c++)
+#else
+ for (c=0 ; c <= 0x10ffffUL ; c++)
+#endif
+ {
+ unsigned int glibc;
+ unsigned int mine;
+ unsigned int upper, lower;
+
+#if 0
+#if RANGE < 0x10000UL
+ if (c == 0x10000UL) {
+ c = 0x30000UL; /* skip 1st and 2nd sup planes */
+ }
+#elif RANGE < 0x20000UL
+ if (c == 0x20000UL) {
+ c = 0x30000UL; /* skip 2nd sup planes */
+ }
+#endif
+#endif
+
+ glibc = 0;
+ if (iswalnum(c)) ++glibc; glibc <<= 1;
+ if (iswalpha(c)) ++glibc; glibc <<= 1;
+ if (iswblank(c)) ++glibc; glibc <<= 1;
+ if (iswcntrl(c)) ++glibc; glibc <<= 1;
+ if (iswdigit(c)) ++glibc; glibc <<= 1;
+ if (iswgraph(c)) ++glibc; glibc <<= 1;
+ if (iswlower(c)) ++glibc; glibc <<= 1;
+ if (iswprint(c)) ++glibc; glibc <<= 1;
+ if (iswpunct(c)) ++glibc; glibc <<= 1;
+ if (iswspace(c)) ++glibc; glibc <<= 1;
+ if (iswupper(c)) ++glibc; glibc <<= 1;
+ if (iswxdigit(c)) ++glibc;
+
+ {
+ unsigned int u;
+ int n, sc;
+ int i0, i1;
+
+ u = c;
+ if (u <= RANGE) {
+ sc = u & ((1 << cttable.ti_shift) - 1);
+ u >>= cttable.ti_shift;
+ n = u & ((1 << cttable.ii_shift) - 1);
+ u >>= cttable.ii_shift;
+
+ i0 = cttable.ii[u];
+ i0 <<= cttable.ii_shift;
+ i1 = cttable.ti[i0 + n];
+ i1 <<= (cttable.ti_shift-1);
+ d = cttable.ut[i1 + (sc >> 1)];
+
+ if (sc & 1) {
+ d >>= 4;
+ }
+ d &= 0x0f;
+ } else if ((((unsigned int)(c - 0xe0020UL)) <= 0x5f) || (c == 0xe0001UL)){
+ d = __CTYPE_punct;
+ } else if (((unsigned int)(c - 0xf0000UL)) < 0x20000UL) {
+ if ((c & 0xffffU) <= 0xfffdU) {
+ d = __CTYPE_punct;
+ } else {
+ d = __CTYPE_unclassified;
+ }
+ } else {
+ d = __CTYPE_unclassified;
+ }
+
+ mine = 0;
+ if (mywalnum(c)) ++mine; mine <<= 1;
+ if (mywalpha(c)) ++mine; mine <<= 1;
+ if (mywblank(c)) ++mine; mine <<= 1;
+ if (mywcntrl(c)) ++mine; mine <<= 1;
+ if (mywdigit(c)) ++mine; mine <<= 1;
+ if (mywgraph(c)) ++mine; mine <<= 1;
+ if (mywlower(c)) ++mine; mine <<= 1;
+ if (mywprint(c)) ++mine; mine <<= 1;
+ if (mywpunct(c)) ++mine; mine <<= 1;
+ if (mywspace(c)) ++mine; mine <<= 1;
+ if (mywupper(c)) ++mine; mine <<= 1;
+ if (mywxdigit(c)) ++mine;
+
+ if (glibc != mine) {
+ printf("%#8x : glibc %#4x != %#4x mine %d\n", c, glibc, mine, d);
+ if (c < 0x30000UL) {
+ printf("sc=%#x u=%#x n=%#x i0=%#x i1=%#x\n", sc, u, n, i0, i1);
+ }
+ }
+ upper = lower = u = c;
+ if (u <= RANGE) {
+ sc = u & ((1 << ultable.ti_shift) - 1);
+ u >>= ultable.ti_shift;
+ n = u & ((1 << ultable.ii_shift) - 1);
+ u >>= ultable.ii_shift;
+
+ i0 = ultable.ii[u];
+ i0 <<= ultable.ii_shift;
+ i1 = ultable.ti[i0 + n];
+ i1 <<= (ultable.ti_shift);
+ i1 += sc;
+ i0 = ultable.ut[i1];
+ upper = c + uldiff[i0].u;
+ lower = c + uldiff[i0].l;
+ }
+
+ if (towupper(c) != upper) {
+ printf("%#8x : towupper glibc %#4x != %#4x mine\n",
+ c, towupper(c), upper);
+ }
+
+ if (towlower(c) != lower) {
+ printf("%#8x : towlower glibc %#4x != %#4x mine i0 = %d\n",
+ c, towlower(c), lower, i0);
+ }
+
+ if (totitle && ((tt = towctrans(c, totitle)) != upper)) {
+ printf("%#8x : totitle glibc %#4lx != %#4x mine i0 = %d\n",
+ c, tt, upper, i0);
+ }
+ }
+
+
+ if ((c & 0xfff) == 0xfff) printf(".");
+ }
+ printf("done\n");
+ }
+
+ if (1) {
+ FILE *fp;
+
+ if (!(fp = fopen("wctables.h", "w"))) {
+ printf("couldn't open wctables.h!\n");
+ return EXIT_FAILURE;
+ }
+
+ fprintf(fp, "#define WC_TABLE_DOMAIN_MAX %#8lx\n\n",
+ (unsigned long) RANGE);
+ output_table(fp, "ctype", &cttable);
+ output_table(fp, "uplow", &ultable);
+
+
+#warning fix the upper bound on the upper/lower tables... save 200 bytes or so
+ fprintf(fp, "#define WCuplow_diffs %7u\n", ul_count);
+ fprintf(fp, "\n#ifdef WANT_WCuplow_diff_data\n\n");
+ fprintf(fp, "\nstatic const short WCuplow_diff_data[%zu] = {",
+ 2 * (size_t) ul_count);
+ for (i=0 ; i < ul_count ; i++) {
+ if (i % 4 == 0) {
+ fprintf(fp, "\n");
+ }
+ fprintf(fp, " %6d, %6d,", uldiff[i].u, uldiff[i].l);
+ }
+ fprintf(fp, "\n};\n\n");
+ fprintf(fp, "#endif /* WANT_WCuplow_diff_data */\n\n");
+
+
+ output_table(fp, "comb", &combtable);
+ output_table(fp, "width", &widthtable);
+
+ fclose(fp);
+ }
+
+ return EXIT_SUCCESS;
+}
+
+size_t newopt(unsigned char *ut, size_t usize, int shift, table_data *tbl)
+{
+ static int recurse = 0;
+ unsigned char *ti[RANGE+1]; /* table index */
+ size_t numblocks;
+ size_t blocksize;
+ size_t uniq;
+ size_t i, j;
+ size_t smallest, t;
+ unsigned char *ii_save;
+ int uniqblock[256];
+ unsigned char uit[RANGE+1];
+ int shift2;
+
+ ii_save = NULL;
+ blocksize = 1 << shift;
+ numblocks = usize >> shift;
+
+ /* init table index */
+ for (i=j=0 ; i < numblocks ; i++) {
+ ti[i] = ut + j;
+ j += blocksize;
+ }
+
+ /* sort */
+ nu_val = blocksize;
+ qsort(ti, numblocks, sizeof(unsigned char *), nu_memcmp);
+
+ uniq = 1;
+ uit[(ti[0]-ut)/blocksize] = 0;
+ for (i=1 ; i < numblocks ; i++) {
+ if (memcmp(ti[i-1], ti[i], blocksize) < 0) {
+ if (++uniq > 255) {
+ break;
+ }
+ uniqblock[uniq - 1] = i;
+ }
+#if 1
+ else if (memcmp(ti[i-1], ti[i], blocksize) > 0) {
+ printf("bad sort %i!\n", i);
+ abort();
+ }
+#endif
+ uit[(ti[i]-ut)/blocksize] = uniq - 1;
+ }
+
+ smallest = SIZE_MAX;
+ shift2 = -1;
+ if (uniq <= 255) {
+ smallest = numblocks + uniq * blocksize;
+ if (!recurse) {
+ ++recurse;
+ for (j=1 ; j < 14 ; j++) {
+ if ((numblocks >> j) < 2) break;
+ if (tbl) {
+ ii_save = tbl->ii;
+ tbl->ii = NULL;
+ }
+ if ((t = newopt(uit, numblocks, j, tbl)) < SIZE_MAX) {
+ t += uniq * blocksize;
+ }
+ if (tbl) {
+ tbl->ii = ii_save;
+ }
+ if (smallest >= t) {
+ shift2 = j;
+ smallest = t;
+ if (!tbl->ii) {
+ printf("ishift %zu tshift %zu size %zu\n",
+ shift2, shift, t);
+ }
+/* } else { */
+/* break; */
+ }
+ }
+ --recurse;
+ }
+ } else {
+ return SIZE_MAX;
+ }
+
+ if (tbl->ii) {
+ if (recurse) {
+ tbl->ii_shift = shift;
+ tbl->ii_len = numblocks;
+ memcpy(tbl->ii, uit, numblocks);
+ tbl->ti = tbl->ii + tbl->ii_len;
+ tbl->ti_len = uniq * blocksize;
+ for (i=0 ; i < uniq ; i++) {
+ memcpy(tbl->ti + i * blocksize, ti[uniqblock[i]], blocksize);
+ }
+ } else {
+ ++recurse;
+ printf("setting ishift %zu tshift %zu\n",
+ shift2, shift);
+ newopt(uit, numblocks, shift2, tbl);
+ --recurse;
+ tbl->ti_shift = shift;
+ tbl->ut_len = uniq * blocksize;
+ tbl->ut = tbl->ti + tbl->ti_len;
+ for (i=0 ; i < uniq ; i++) {
+ memcpy(tbl->ut + i * blocksize, ti[uniqblock[i]], blocksize);
+ }
+ }
+ }
+ return smallest;
+}
diff --git a/extra/locale/lmmtolso.c b/extra/locale/lmmtolso.c
new file mode 100644
index 000000000..a1876a7dc
--- /dev/null
+++ b/extra/locale/lmmtolso.c
@@ -0,0 +1,72 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+
+int main(void)
+{
+ FILE *lmm; /* mmap-able file */
+ FILE *lso; /* static object */
+ struct stat fd_stat;
+ int c;
+ size_t i;
+
+ if (!(lmm = fopen("locale.mmap", "r"))) {
+ printf("can't open locale.mmap!\n");
+ return EXIT_FAILURE;
+ }
+
+ if (fstat(fileno(lmm), &fd_stat)) {
+ printf("can't stat locale.mmap!\n");
+ fclose(lmm);
+ return EXIT_FAILURE;
+ }
+
+ if (!(lso = fopen("locale_data.c", "w"))) {
+ printf("can't open locale_data.c!\n");
+ fclose(lmm);
+ return EXIT_FAILURE;
+ }
+
+ fprintf(lso,
+ "#include <stddef.h>\n"
+ "#include <stdint.h>\n"
+ "#include \"lt_defines.h\"\n"
+ "#include \"locale_mmap.h\"\n\n"
+ "typedef union {\n"
+ "\tunsigned char buf[%zu];\n"
+ "\t__locale_mmap_t lmm;\n"
+ "} locale_union_t;\n\n"
+ "static const locale_union_t locale_union = { {",
+ (size_t) fd_stat.st_size
+ );
+
+ i = 0;
+ while ((c = getc(lmm)) != EOF) {
+ if (!(i & 0x7)) {
+ fprintf(lso, "\n\t");
+ }
+ fprintf(lso, "%#04x, ", c);
+ ++i;
+ }
+ fprintf(lso,
+ "\n} };\n\n"
+ "const __locale_mmap_t *__locale_mmap = &locale_union.lmm;\n\n"
+ );
+
+ if (ferror(lmm)) {
+ printf("error reading!\n");
+ return EXIT_FAILURE;
+ }
+
+ if (ferror(lso) || fclose(lso)) {
+ printf("error writing!\n");
+ return EXIT_FAILURE;
+ }
+
+ fclose(lmm);
+
+ return EXIT_SUCCESS;
+}
diff --git a/extra/locale/locale_mmap.h b/extra/locale/locale_mmap.h
new file mode 100644
index 000000000..1ed7a2094
--- /dev/null
+++ b/extra/locale/locale_mmap.h
@@ -0,0 +1,89 @@
+/* #include "lt_defines.h" */
+
+/* TODO - fix */
+#define MAGIC_SIZE 64
+
+/* TODO - fix */
+#ifdef __WCHAR_ENABLED
+#define WCctype_TBL_LEN (WCctype_II_LEN + WCctype_TI_LEN + WCctype_UT_LEN)
+#define WCuplow_TBL_LEN (WCuplow_II_LEN + WCuplow_TI_LEN + WCuplow_UT_LEN)
+#define WCuplow_diff_TBL_LEN (2 * WCuplow_diffs)
+#define WCcomb_TBL_LEN (WCcomb_II_LEN + WCcomb_TI_LEN + WCcomb_UT_LEN)
+#endif
+
+#undef __PASTE2
+#define __PASTE2(A,B) A ## B
+#undef __PASTE3
+#define __PASTE3(A,B,C) A ## B ## C
+
+#define COMMON_MMAP(X) \
+ unsigned char __PASTE3(lc_,X,_data)[__PASTE3(__lc_,X,_data_LEN)];
+
+#define COMMON_MMIDX(X) \
+ unsigned char __PASTE3(lc_,X,_rows)[__PASTE3(__lc_,X,_rows_LEN)]; \
+ uint16_t __PASTE3(lc_,X,_item_offsets)[__PASTE3(__lc_,X,_item_offsets_LEN)]; \
+ uint16_t __PASTE3(lc_,X,_item_idx)[__PASTE3(__lc_,X,_item_idx_LEN)]; \
+
+
+typedef struct {
+ unsigned char magic[MAGIC_SIZE];
+
+#ifdef __CTYPE_HAS_8_BIT_LOCALES
+ const unsigned char tbl8ctype[Cctype_TBL_LEN];
+ const unsigned char tbl8uplow[Cuplow_TBL_LEN];
+#ifdef __WCHAR_ENABLED
+ const uint16_t tbl8c2wc[Cc2wc_TBL_LEN]; /* char > 0x7f to wide char */
+ const unsigned char tbl8wc2c[Cwc2c_TBL_LEN];
+ /* translit */
+#endif /* __WCHAR_ENABLED */
+#endif /* __CTYPE_HAS_8_BIT_LOCALES */
+#ifdef __WCHAR_ENABLED
+ const unsigned char tblwctype[WCctype_TBL_LEN];
+ const unsigned char tblwuplow[WCuplow_TBL_LEN];
+ const int16_t tblwuplow_diff[WCuplow_diff_TBL_LEN];
+ const unsigned char tblwcomb[WCcomb_TBL_LEN];
+ /* width?? */
+#endif /* __WCHAR_ENABLED */
+
+ COMMON_MMAP(numeric);
+ COMMON_MMAP(monetary);
+ COMMON_MMAP(time);
+
+ /* TODO -- collate*/
+
+ COMMON_MMAP(messages);
+
+
+#ifdef __CTYPE_HAS_8_BIT_LOCALES
+ const codeset_8_bit_t codeset_8_bit[NUM_CODESETS];
+#endif /* __CTYPE_HAS_8_BIT_LOCALES */
+
+ COMMON_MMIDX(numeric);
+ COMMON_MMIDX(monetary);
+ COMMON_MMIDX(time);
+
+ /* TODO -- collate*/
+
+ COMMON_MMIDX(messages);
+
+
+ unsigned char lc_common_item_offsets_LEN[CATEGORIES];
+ size_t lc_common_tbl_offsets[CATEGORIES * 4];
+ /* offsets from start of locale_mmap_t */
+ /* rows, item_offsets, item_idx, data */
+
+#ifdef NUM_LOCALES
+ unsigned char locales[NUM_LOCALES * WIDTH_LOCALES];
+ unsigned char locale_names5[5*NUM_LOCALE_NAMES];
+ unsigned char locale_at_modifiers[LOCALE_AT_MODIFIERS_LENGTH];
+#endif /* NUM_LOCALES */
+
+ unsigned char lc_names[lc_names_LEN];
+#ifdef __CTYPE_HAS_8_BIT_LOCALES
+ unsigned char codeset_list[sizeof(CODESET_LIST)]; /* TODO - fix */
+#endif /* __CTYPE_HAS_8_BIT_LOCALES */
+
+
+} __locale_mmap_t;
+
+extern const __locale_mmap_t *__locale_mmap;