From 522636d8fbc6ba6ed905f47acb1f0479a1c8c971 Mon Sep 17 00:00:00 2001 From: Bruno Haible Date: Sun, 5 Apr 2009 13:43:15 +0200 Subject: Documentation of libunistring. --- doc/libunistring.texi | 705 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 705 insertions(+) create mode 100644 doc/libunistring.texi (limited to 'doc') diff --git a/doc/libunistring.texi b/doc/libunistring.texi new file mode 100644 index 0000000..fe4208b --- /dev/null +++ b/doc/libunistring.texi @@ -0,0 +1,705 @@ +\input texinfo @c -*-texinfo-*- +@comment %**start of header +@setfilename libunistring.info +@settitle GNU libunistring +@finalout +@c Indices: +@c cp = concept @cindex +@c fn = function @findex +@c tp = type @tindex +@c Unused predefined indices: +@c ky = keystroke @kindex +@c pg = program @pindex +@c vr = variable @vindex +@syncodeindex fn cp +@syncodeindex tp cp +@ifclear texi2html +@firstparagraphindent insert +@end ifclear +@comment %**end of header + +@include version.texi + +@ifinfo +@dircategory Software development +@direntry +* GNU libunistring: (libunistring). Unicode string library. +@end direntry +@end ifinfo + +@ifinfo +This manual is for GNU libunistring. + +@ignore +@c This was: @copying but it triggers a makeinfo 4.13 bug +Copyright (C) 2001-2009 Free Software Foundation, Inc. + +This manual is free documentation. It is dually licensed under the +GNU FDL and the GNU GPL. This means that you can redistribute this +manual under either of these two licenses, at your choice. + +This manual is covered by the GNU FDL. Permission is granted to copy, +distribute and/or modify this document under the terms of the +GNU Free Documentation License (FDL), either version 1.2 of the +License, or (at your option) any later version published by the +Free Software Foundation (FSF); with no Invariant Sections, with no +Front-Cover Text, and with no Back-Cover Texts. +A copy of the license is included in @ref{GNU FDL}. + +This manual is covered by the GNU GPL. You can redistribute it and/or +modify it under the terms of the GNU General Public License (GPL), either +version 3 of the License, or (at your option) any later version published +by the Free Software Foundation (FSF). +A copy of the license is included in @ref{GNU GPL}. +@end ignore +@end ifinfo + +@titlepage +@title GNU libunistring, version @value{VERSION} +@subtitle updated @value{UPDATED} +@subtitle Edition @value{EDITION}, @value{UPDATED} +@author Bruno Haible + +@ifnothtml +@page +@vskip 0pt plus 1filll +@c @insertcopying +Copyright (C) 2001-2009 Free Software Foundation, Inc. + +This manual is free documentation. It is dually licensed under the +GNU FDL and the GNU GPL. This means that you can redistribute this +manual under either of these two licenses, at your choice. + +This manual is covered by the GNU FDL. Permission is granted to copy, +distribute and/or modify this document under the terms of the +GNU Free Documentation License (FDL), either version 1.2 of the +License, or (at your option) any later version published by the +Free Software Foundation (FSF); with no Invariant Sections, with no +Front-Cover Text, and with no Back-Cover Texts. +A copy of the license is included in @ref{GNU FDL}. + +This manual is covered by the GNU GPL. You can redistribute it and/or +modify it under the terms of the GNU General Public License (GPL), either +version 3 of the License, or (at your option) any later version published +by the Free Software Foundation (FSF). +A copy of the license is included in @ref{GNU GPL}. +@end ifnothtml +@end titlepage + +@ifnottex +@c Table of Contents +@contents +@end ifnottex + +@ifnottex +@node Top +@top GNU libunistring +@end ifnottex + +@menu +* Introduction:: Who may need Unicode strings? +* Conventions:: Conventions used in this manual +* unitypes.h:: Elementary types +* unistr.h:: Elementary Unicode string functions +* uniconv.h:: Conversions between Unicode and encodings +* unistdio.h:: Output with Unicode strings +* uniname.h:: Names of Unicode characters +* unictype.h:: Unicode character classification and properties +* uniwidth.h:: Display width +* uniwbrk.h:: Word breaks in strings +* unilbrk.h:: Line breaking +* uninorm.h:: Normalization forms +* unicase.h:: Case mappings +* uniregex.h:: Regular expressions +* More functionality:: More advanced functionality +* Licenses:: Licenses + +* Index:: General Index + +@detailmenu + --- The Detailed Node Listing --- + +Introduction + +* Unicode:: What is Unicode? +* Unicode and i18n:: Unicode and internationalization +* Locale encodings:: What is a locale encoding? +* In-memory representation:: How to represent strings in memory? +* char * strings:: What to keep in mind with @code{char *} strings +* The wchar_t mess:: Why @code{wchar_t *} strings are useless +* Unicode strings:: How are Unicode strings represented? + +Licenses + +* GNU GPL:: GNU General Public License +* GNU LGPL:: GNU Lesser General Public License +* GNU FDL:: GNU Free Documentation License + +@end detailmenu +@end menu + +@node Introduction +@chapter Introduction + +This library provides functions for manipulating Unicode strings and +for manipulating C strings according to the Unicode standard. + +It consists of the following parts: + +@table @code +@item +elementary string functions +@item +conversion from/to legacy encodings +@item +formatted output to strings +@item +character names +@item +character classification and properties +@item +string width when using nonproportional fonts +@item +word breaks +@item +line breaking algorithm +@item +normalization (composition and decomposition) +@item +case folding +@item +regular expressions (not yet implemented) +@end table + +libunistring is for you if your application involves non-trivial text +processing, such as upper/lower case conversions, line breaking, operations +on words, or more advanced analysis of text. Text provided by the user can, +in general, contain characters of all kinds of scripts. The text processing +functions provided by this library handle all scripts and all languages. + +libunistring is for you if your application already uses the ISO C / POSIX +@code{}, @code{} functions and the text it operates on is +provided by the user and can be in any language. + +libunistring is also for you if your application uses Unicode strings as +internal in-memory representation. + +@menu +* Unicode:: What is Unicode? +* Unicode and i18n:: Unicode and internationalization +* Locale encodings:: What is a locale encoding? +* In-memory representation:: How to represent strings in memory? +* char * strings:: What to keep in mind with @code{char *} strings +* The wchar_t mess:: Why @code{wchar_t *} strings are useless +* Unicode strings:: How are Unicode strings represented? +@end menu + +@node Unicode +@section Unicode + +Unicode is a standardized repertoire of characters that contains characters +from all scripts of the world, from Latin letters to Chinese ideographs +and Babylonian cuneiform glyphs. It also specifies how these characters +are to be rendered on a screen or on paper, and how common text processing +(word selection, line breaking, uppercasing of page titles etc.) is supposed +to behave on Unicode text. + +Unicode also specifies three ways of storing sequences of Unicode +characters in a computer whose basic unit of data is an 8-bit byte: +@table @asis +@item UTF-8 +Every character is represented as 1 to 4 bytes. +@item UTF-16 +Every character is represented as 1 to 2 units of 16 bits. +@item UTF-32, a.k.a@. UCS-4 +Every character is represented as 1 unit of 32 bits. +@end table + +For encoding Unicode text in a file, UTF-8 is usually used. For encoding +Unicode strings in memory for a program, either of the three encoding forms +can be reasonably used. + +Unicode is widely used on the web. Prior to the use of Unicode, web pages +were in many different encodings (ISO-8859-1 for English, French, Spanish, +ISO-8859-2 for Polish, ISO-8859-7 for Greek, KOI8-R for Russian, GB2312 or +BIG5 for Chinese, ISO-2022-JP-2 or EUC-JP or Shift_JIS for Japanese, and many +many others). It was next to impossible to create a document that contained +Chinese and Polish text in the same document. Due to the many encodings for +Japanese, even the processing of pure Japanese text was error prone. + +References: +@itemize @bullet +@item +The Unicode standard: @url{http://www.unicode.org/} +@item +Definition of UTF-8: @url{http://www.rfc-editor.org/rfc/rfc3629.txt} +@item +Definition of UTF-16: @url{http://www.rfc-editor.org/rfc/rfc2781.txt} +@item +Markus Kuhn's UTF-8 and Unicode FAQ: +@url{http://www.cl.cam.ac.uk/~mgk25/unicode.html} +@end itemize + +@node Unicode and i18n +@section Unicode and Internationalization + +Internationalization is the process of changing the source code of a program +so that it can meet the expectations of users in any culture, if culture +specific data (translations, images etc.) are provided. + +Use of Unicode is not strictly required for internationalization, but it +makes internationalization much easier, because operations that need to +look at specific characters (like hyphenation, spell checking, or the +automatic conversion of double-quotes to opening and closing double-quote +characters) don't need to consider multiple possible encodings of the text. + +Use of Unicode also enables multilingualization: the ability of having text +in multiple languages present in the same document or even in the same line +of text. + +But use of Unicode is not everything. Internationalization usually consists +of three features: +@itemize @bullet +@item +Use of Unicode where needed for text processing. This is what this library +is for. +@item +Use of message catalogs for messages shown to the user, This is what +GNU gettext is about. +@item +Use of locale specific conventions for date and time formats, for numeric +formatting, or for sorting of text. This can be done adequately with the +POSIX APIs and the implementation of locales in the GNU C library. +@end itemize + +@node Locale encodings +@section Locale encodings + +A locale is a set of cultural conventions. According to POSIX, for a program, +at any moment, there is one locale being designated as the ``current locale''. +(Actually, POSIX supports also one locale per thread, but this feature is not +yet universally implemented and not widely used.) The locale is partitioned +into several aspects, called the ``categories'' of the locale. The main +various aspects are: +@itemize +@item +The character encoding and the character properties. This is the +@code{LC_CTYPE} category. +@item +The sorting rules for text. This is the @code{LC_COLLATE} category. +@item +The language specific translations of messages. This is the +@code{LC_MESSAGES} category. +@item +The formatting rules for numbers, such as the decimal separator. This is +the @code{LC_NUMERIC} category. +@item +The formatting rules for amounts of money. This is the @code{LC_MONETARY} +category. +@item +The formatting of date and time. This is the @code{LC_TIME} category. +@end itemize + +In particular, the @code{LC_CTYPE} category of the current locale determines +the character encoding. This is the encoding of @samp{char *} strings. +We also call it the ``locale encoding''. GNU libunistring has a function, +@code{locale_charset()}, that returns a standardized (platform independent) +name for this encoding. + +All locale encodings used on glibc systems are essentially ASCII compatible: +Most graphic ASCII characters have the same representation, as a single byte, +in that encoding as in ASCII. + +Among the possible locale encodings are UTF-8 and GB18030. Both allow +to represent any Unicode character as a sequence of bytes. UTF-8 is used in +most of the world, whereas GB18030 is used in the People's Republic of China, +because it is backward compatible with the GB2312 encoding that was used in +this country earlier. + +The legacy locale encodings, ISO-8859-15 (which supplanted ISO-8859-1 in +most of Europe), ISO-8859-2, KOI8-R, EUC-JP, etc., are still in use in +many places, though. + +UTF-16 and UTF-32 are not used as locale encodings, because they are not +ASCII compatible. + +@node In-memory representation +@section Choice of in-memory representation of strings + +There are three ways of representing strings in memory of a running +program. +@itemize +@item +As @samp{char *} strings. Such strings are represented in locale encoding. +This approach is employed when not much text processing is done by the +program. When some Unicode aware processing is to be done, a string is +converted to Unicode on the fly and back to locale encoding afterwards. +@item +As UTF-8 or UTF-16 or UTF-32 strings. This implies that conversion from +locale encoding to Unicode is performed on input, and in the opposite +direction on output. This approach is employed when the program does +a significant amount of text processing, or when the program has multiple +threads operating on the same data but in different locales. +@item +As @samp{wchar_t *}, a.k.a@. ``wide strings''. This approach is misguided, +see @ref{The wchar_t mess}. +@end itemize + +@node char * strings +@section @samp{char *} strings + +The classical C strings, with its C library support standardized by +ISO C and POSIX, can be used in internationalized programs with some +precautions. The problem with this API is that many of the C library +functions for strings don't work correctly on strings in locale +encodings, leading to bugs that only people in some cultures of the +world will experience. + +The first problem with the C library API is the support of multibyte +locales. According to the locale encoding, in general, every character +is represented by one or more bytes (up to 4 bytes in practice --- but +use @code{MB_LEN_MAX} instead of the number 4 in the code). +When every character is represented by only 1 byte, we speak of an +``unibyte locale'', otherwise of a ``multibyte locale''. It is important +to realize that the majority of Unix installations nowadays use UTF-8 +or GB18030 as locale encoding; therefore, the majority of users are +using multibyte locales. + +The important fact to remember is: +@cartouche +@emph{A @samp{char} is a byte, not a character.} +@end cartouche + +As a consequence: +@itemize +@item +The @code{} API is useless in this context; it does not work in +multibyte locales. +@item +The @code{strlen()} function does not return the number of characters +in a string. Nor does it return the number of screen columns occupied +by a string after it is output. It merely returns the number of +@emph{bytes} occupied by a string. +@item +Truncating a string, for example, with @code{strncpy()}, can have the +effect of truncating it in the middle of a multibyte character. Such +a string will, when output, have a garbled character at its end, often +represented by a hollow box. +@item +@code{strchr()} and @code{strrchr()} do not work with multibyte strings +if the locale encoding is GB18030 and the character to be searched is +a digit. +@item +@code{strstr()} does not work with multibyte strings if the locale encoding +is different from UTF-8. +@item +@code{strcspn()}, @code{strpbrk()}, @code{strspn()} cannot work correctly +in multibyte locales: they assume the second argument is a list of +single-byte characters. Even in this simple case, they do not work with +multibyte strings if the locale encoding is GB18030 and one of the +characters to be searched is a digit. +@item +@code{strsep()} and @code{strtok_r()} do not work with multibyte strings +unless all of the delimiter characters are ASCII characters < 0x30. +@item +The @code{strcasecmp()}, @code{strncasecmp()}, and @code{strcasestr()} +functions do not work with multibyte strings. +@end itemize + +The workarounds can be found in GNU gnulib @url{http://wwww.gnu.org/software/gnulib/}. +@itemize +@item +gnulib has modules @samp{mbchar}, @samp{mbiter}, @samp{mbuiter} that +represent multibyte characters and allow to iterate across a multibyte +string with the same ease as through a unibyte string. +@item +gnulib has functions @code{mbslen()} and @code{mbswidth()} that can be +used instead of @code{strlen()} when the number of characters or the +number of screen columns of a string is requested. +@item +gnulib has functions @code{mbschr()} and @code{mbsrrchr()} that are +like @code{strchr()} and @code{strrchr()}, but work in multibyte locales. +@item +gnulib has a function @code{mbsstr()}, like @code{strstr()}, but works +in multibyte locales. +@item +gnulib has functions @code{mbscspn()}, @code{mbspbrk()}, @code{mbsspn()} +that are like @code{strcspn()}, @code{strpbrk()}, @code{strspn()} , but +work in multibyte locales. +@item +gnulib has functions @code{mbssep()} and @code{mbstok_r()} that are +like @code{strsep()} and @code{strtok_r()} but work in multibyte locales. +@item +gnulib has functions @code{mbscasecmp()}, @code{mbsncasecmp()}, +@code{mbspcasecmp()}, and @code{mbscasestr()} that are like +@code{strcasecmp()}, @code{strncasecmp()}, and @code{strcasestr()}, but +work in multibyte locales. Still, the function @code{ulc_casecmp} is +preferable to these functions; see below. +@end itemize + +The second problem with the C library API is that it has some assumptions built-in that are not valid in some languages: +@itemize +@item +It assumes that there are only two forms of every character: uppercase +and lowercase. This is not true for Croatian, where the character +@sc{LETTER DZ WITH CARON} comes in three forms: +@sc{LATIN CAPITAL LETTER DZ WITH CARON} (DZ), +@sc{LATIN CAPITAL LETTER D WITH SMALL LETTER Z WITH CARON} (Dz), +@sc{LATIN SMALL LETTER DZ WITH CARON} (dz). +@item +It assumes that uppercasing of 1 character leads to 1 character. This +is not true for German, where the @sc{LATIN SMALL LETTER SHARP S}, when +uppercased, becomes @samp{SS}. +@item +It assumes that there is 1:1 mapping between uppercase and lowercase forms. +This is not true for the Greek sigma: @sc{GREEK CAPITAL LETTER SIGMA} is +the uppercase of both @sc{GREEK SMALL LETTER SIGMA} and +@sc{GREEK SMALL LETTER FINAL SIGMA}. +@item +It assumes that the upper/lowercase mappings are position independent. +This is not true for the Greek sigma and the Lithuanian i. +@end itemize + +The correct way to deal with this problem is +@enumerate +@item +to provide functions for titlecasing, as well as for upper- and +lowercasing, +@item +to view case transformations as functions that operates on strings, +rather than on characters. +@end enumerate + +This is implemented in this library, through the functions declared in @code{}, see @ref{unicase.h}. + +@node The wchar_t mess +@section The @code{wchar_t} mess + +The ISO C and POSIX standard creators made an attempt to fix the first +problem mentioned in the previous section. They introduced +@itemize +@item +a type @samp{wchar_t}, designed to encapsulate an entire character, +@item +a ``wide string'' type @samp{wchar_t *}, and +@item +functions declared in @code{} that were meant to supplant the +ones in @code{}. +@end itemize + +Unfortunately, this API and its implementation has numerous problems: + +@itemize +@item +On AIX and Windows platforms, @code{wchar_t} is a 16-bit type. This +means that it can never accomodate an entire Unicode character. Either +the @code{wchar_t *} strings are limited to characters in UCS-2 (the +``Basic Multilingual Plane'' of Unicode), or --- if @code{wchar_t *} +strings are encoded in UTF-16 --- a @code{wchar_t} represents only half +of a character in the worst case, making the @code{} functions +pointless. + +@item +On Solaris and FreeBSD, the @code{wchar_t} encoding is locale dependent +and undocumented. This means, if you want to know any property of a +@code{wchar_t} character, other than the properties defined by +@code{} --- such as whether it's a dash, currency symbol, +paragraph separator, or similar ---, you have to convert it to +@code{char *} encoding first, by use of the function @code{wctomb()}. + +@item +When you read a stream of wide characters, through the functions +@code{fgetwc()} and @code{fgetws()}, and when the input stream/file is +not in the expected encoding, you have no way to determine the invalid +byte sequence and do some corrective action. If you use these +functions, your program becomes ``garbage in - more garbage out'' or +``garbage in - abort''. +@end itemize + +As a consequence, it is better to use multibyte strings, as explained in +the previous section. Such multibyte strings can bypass limitations +of the @code{wchar_t} type, if you use functions defined in gnulib and +libunistring for text processing. They can also faithfully transport +malformed characters that were present in the input, without requiring +the program to produce garbage or abort. + +@node Unicode strings +@section Unicode strings + +libunistring supports Unicode strings in three representations: +@itemize +@item +UTF-8 strings, through the type @samp{uint8_t *}. The units are bytes +(@code{uint8_t}). +@item +UTF-16 strings, through the type @samp{uint16_t *}, The units are 16-bit +memory words (@code{uint16_t}). +@item +UTF-32 strings, through the type @samp{uint32_t *}. The units are 32-bit +memory words (@code{uint32_t}). +@end itemize + +As with C strings, there are two variants: +@itemize +@item +Unicode strings with a terminating NUL character are represented as +a pointer to the first unit of the string. There is a unit containing +a 0 value at the end. It is considered part of the string for all +memory allocation purposes, but is not considered part of the string +for all other logical purposes. +@item +Unicode strings where embedded NUL characters are allowed. These +are represented by a pointer to the first unit and the number of units +(not bytes!) of the string. In this setting, there is no trailing +zero-valued unit used as ``end marker'' +@end itemize + +@node Conventions +@chapter Conventions + +This chapter explains conventions valid throughout the libunistring library. + +Variables of type @code{char *} denote C strings in locale encoding. +See @ref{Locale encodings}. + +Variables of type @code{uint8_t *} denote UTF-8 strings. Their units +are bytes. + +Variables of type @code{uint16_t *} denote UTF-16 strings, without byte +order mark. Their units are 2-byte words. + +Variables of type @code{uint32_t *} denote UTF-32 strings, without byte +order mark. Their units are 4-byte words. + +Argument pairs @code{(@var{s}, @var{n})} denote a string +@code{@var{s}[0..@var{n}-1]} with exactly @var{n} units. + +All functions with prefix @samp{ulc_} operate on C strings in locale +encoding. + +All functions with prefix @samp{u8_} operate on UTF-8 strings. + +All functions with prefix @samp{u16_} operate on UTF-16 strings. + +All functions with prefix @samp{u32_} operate on UTF-32 strings. + +For every function with prefix @samp{u8_}, operating on UTF-8 strings, +there is also a corresponding function with prefix @samp{u16_}, +operating on UTF-16 strings, and a corresponding function with prefix +@samp{u32_}, operating on UTF-32 strings. Their description is +analogous; in this documentation we describe only the function that +operates on UTF-8 strings, for brevity. + +A declaration with a variable @var{n} denotes the three concrete +declarations with @var{n} = 8, @var{n} = 16, @var{n} = 32. + +All parameters starting with @samp{str} and the parameters of +functions starting with @code{u8_str}/@code{u16_str}/@code{u32_str} +denote a NUL terminated string. + +Error values are always returned through the @code{errno} variable, +usually with a return value that indicates the presence of an error +(NULL for functions that return an pointer, or -1 for functions that +return an @code{int}). + +Functions returning a string result take a +@code{(@var{resultbuf}, @var{lengthp})} +argument pair. If @var{resultbuf} is not NULL and the result fits +into @code{*@var{lengthp}} units, it is put in @var{resultbuf}, and +@var{resultbuf} is returned. Otherwise, a freshly allocated string +is returned. In both cases, @code{*@var{lengthp}} is set to the +length (number of units) of the returned string. In case of error, +NULL is returned and @code{errno} is set. + +@include unitypes.texi +@include unistr.texi +@include uniconv.texi +@include unistdio.texi +@include uniname.texi +@include unictype.texi +@include uniwidth.texi +@include uniwbrk.texi +@include unilbrk.texi +@include uninorm.texi +@include unicase.texi +@include uniregex.texi + +@node More functionality +@chapter More advanced functionality + +For bidirectional reordering of strings, we recommend the GNU FriBidi library: +@url{http://www.fribidi.org/}. + +For the rendering of Unicode strings outside of the context of a given toolkit +(KDE/Qt or GNOME/Gtk), we recommend the Pango library: +@url{http://www.pango.org/}. + +@node Licenses +@appendix Licenses +@cindex Licenses + +The files of this package are covered by the licenses indicated in each +particular file or directory. Here is a summary: + +@itemize @bullet +@item +The @code{libunistring} library is covered by the +GNU Lesser General Public License (LGPL). +A copy of the license is included in @ref{GNU LGPL}. + +@item +This manual is free documentation. It is dually licensed under the +GNU FDL and the GNU GPL. This means that you can redistribute this +manual under either of these two licenses, at your choice. +@* +This manual is covered by the GNU FDL. Permission is granted to copy, +distribute and/or modify this document under the terms of the +GNU Free Documentation License (FDL), either version 1.2 of the +License, or (at your option) any later version published by the +Free Software Foundation (FSF); with no Invariant Sections, with no +Front-Cover Text, and with no Back-Cover Texts. +A copy of the license is included in @ref{GNU FDL}. +@* +This manual is covered by the GNU GPL. You can redistribute it and/or +modify it under the terms of the GNU General Public License (GPL), either +version 3 of the License, or (at your option) any later version published +by the Free Software Foundation (FSF). +A copy of the license is included in @ref{GNU GPL}. +@end itemize + +@menu +* GNU GPL:: GNU General Public License +* GNU LGPL:: GNU Lesser General Public License +* GNU FDL:: GNU Free Documentation License +@end menu + +@page +@node GNU GPL +@appendixsec GNU GENERAL PUBLIC LICENSE +@cindex GPL, GNU General Public License +@cindex License, GNU GPL +@include gpl.texi +@page +@node GNU LGPL +@appendixsec GNU LESSER GENERAL PUBLIC LICENSE +@cindex LGPL, GNU Lesser General Public License +@cindex License, GNU LGPL +@include lgpl.texi +@page +@node GNU FDL +@appendixsec GNU Free Documentation License +@cindex FDL, GNU Free Documentation License +@cindex License, GNU FDL +@include fdl.texi + +@node Index +@unnumbered Index + +@printindex cp + +@bye + +@c Local Variables: +@c indent-tabs-mode: nil +@c whitespace-check-buffer-indent: nil +@c End: -- cgit v1.2.1