diff options
Diffstat (limited to 'src/assistant')
503 files changed, 52461 insertions, 77 deletions
diff --git a/src/assistant/3rdparty/clucene/APACHE.license b/src/assistant/3rdparty/clucene/APACHE.license new file mode 100644 index 000000000..261eeb9e9 --- /dev/null +++ b/src/assistant/3rdparty/clucene/APACHE.license @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/src/assistant/3rdparty/clucene/AUTHORS b/src/assistant/3rdparty/clucene/AUTHORS new file mode 100644 index 000000000..4a7904b6d --- /dev/null +++ b/src/assistant/3rdparty/clucene/AUTHORS @@ -0,0 +1,22 @@ +As with most development projects, contributions come from many people and in +many forms. The CLucene project would like to thank it's many contributors. +Omissions are merely accidental, please e-mail ustramooner@users.sourceforge.net +if you have been left out or a contribution is not mentioned. + +CLucene was originally ported to C++ by Ben van Klinken (ustramooner@users.sourceforge.net) +from Doug Cutting's popular java search engine, Lucene (see http://lucene.apache.org). + +Here is a list of contributors. Please send me an email at ustramooner@users.sourceforge.net +if I have left you out. + +Doug Cutting cutting@users.sourceforge.net +John Wheeler j_wheeler@users.sourceforge.net +Robert G. Ristroph rgristroph@users.sourceforge.net +David Rushby woodsplitter@users.sourceforge.net +Jimmy Pritts jpritts@sdf.lonestar.org +Peter Edwards peter@dragonstaff.co.uk +Jorge Sabater Redondo jsabater@elderecho.com +Daniel Glassey danglassey@ntlworld.com +Peter Gladkikh batyi@mail.ru +Pedja amigo@max3d.com +Peter Hodges hodges.peter@gmail.com diff --git a/src/assistant/3rdparty/clucene/COPYING b/src/assistant/3rdparty/clucene/COPYING new file mode 100644 index 000000000..0e32bb4f3 --- /dev/null +++ b/src/assistant/3rdparty/clucene/COPYING @@ -0,0 +1,30 @@ +License + +The CLucene Core Library uses a dual license strategy for the source code. +These licenses are the GNU Lesser General Public License (LGPL) and the Apache +License (Version 2.0). Users can choose the license they wish to distribute +their software under. This means that you do not need to abide by *both* +licenses, but rather than you can choose the license which most suits your +needs. + +To rephrase this and to make it perfectly clear: +CLucene is distributed under the GNU Lesser General Public License (LGPL) + *or* +the Apache License, Version 2.0 + +However, we are an open source project, and we encourage users to use the LGPL +license and participate fully in the free software community. Dual licensing +of the CLucene source code provides open and free access to the technology both +for the GPL community and for other developers or companies that cannot use the +GPL. + +You can freely modify, extend, and improve the CLucene source code. The only +question is whether or not you must provide the source code and contribute +modifications to the community. The GNU and Apache licenses allow different +ranges of flexibility in this regard, but in the end, regardless of the license +used, we highly recommend that you submit any bugs, incompatibilities or +added features. + +Note that this same license does *not* apply to the CLucene Contributions +package. You should read the COPYING file in that directory or package for +more information.
\ No newline at end of file diff --git a/src/assistant/3rdparty/clucene/ChangeLog b/src/assistant/3rdparty/clucene/ChangeLog new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/src/assistant/3rdparty/clucene/ChangeLog diff --git a/src/assistant/3rdparty/clucene/LGPL.license b/src/assistant/3rdparty/clucene/LGPL.license new file mode 100644 index 000000000..422c76072 --- /dev/null +++ b/src/assistant/3rdparty/clucene/LGPL.license @@ -0,0 +1,475 @@ + GNU LESSER GENERAL PUBLIC LICENSE + Version 2.1, February 1999 + + Copyright (C) 1991, 1999 Free Software Foundation, Inc. + 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + +[This is the first released version of the Lesser GPL. It also counts + as the successor of the GNU Library Public License, version 2, hence + the version number 2.1.] + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +Licenses are intended to guarantee your freedom to share and change +free software--to make sure the software is free for all its users. + + This license, the Lesser General Public License, applies to some +specially designated software packages--typically libraries--of the +Free Software Foundation and other authors who decide to use it. You +can use it too, but we suggest you first think carefully about whether +this license or the ordinary General Public License is the better +strategy to use in any particular case, based on the explanations below. + + When we speak of free software, we are referring to freedom of use, +not price. Our General Public Licenses are designed to make sure that +you have the freedom to distribute copies of free software (and charge +for this service if you wish); that you receive source code or can get +it if you want it; that you can change the software and use pieces of +it in new free programs; and that you are informed that you can do +these things. + + To protect your rights, we need to make restrictions that forbid +distributors to deny you these rights or to ask you to surrender these +rights. These restrictions translate to certain responsibilities for +you if you distribute copies of the library or if you modify it. + + For example, if you distribute copies of the library, whether gratis +or for a fee, you must give the recipients all the rights that we gave +you. You must make sure that they, too, receive or can get the source +code. If you link other code with the library, you must provide +complete object files to the recipients, so that they can relink them +with the library after making changes to the library and recompiling +it. And you must show them these terms so they know their rights. + + We protect your rights with a two-step method: (1) we copyright the +library, and (2) we offer you this license, which gives you legal +permission to copy, distribute and/or modify the library. + + To protect each distributor, we want to make it very clear that +there is no warranty for the free library. Also, if the library is +modified by someone else and passed on, the recipients should know +that what they have is not the original version, so that the original +author's reputation will not be affected by problems that might be +introduced by others. + +------------------------------------------------------------------------------- + + Finally, software patents pose a constant threat to the existence of +any free program. We wish to make sure that a company cannot +effectively restrict the users of a free program by obtaining a +restrictive license from a patent holder. Therefore, we insist that +any patent license obtained for a version of the library must be +consistent with the full freedom of use specified in this license. + + Most GNU software, including some libraries, is covered by the +ordinary GNU General Public License. This license, the GNU Lesser +General Public License, applies to certain designated libraries, and +is quite different from the ordinary General Public License. We use +this license for certain libraries in order to permit linking those +libraries into non-free programs. + + When a program is linked with a library, whether statically or using +a shared library, the combination of the two is legally speaking a +combined work, a derivative of the original library. The ordinary +General Public License therefore permits such linking only if the +entire combination fits its criteria of freedom. The Lesser General +Public License permits more lax criteria for linking other code with +the library. + + We call this license the "Lesser" General Public License because it +does Less to protect the user's freedom than the ordinary General +Public License. It also provides other free software developers Less +of an advantage over competing non-free programs. These disadvantages +are the reason we use the ordinary General Public License for many +libraries. However, the Lesser license provides advantages in certain +special circumstances. + + For example, on rare occasions, there may be a special need to +encourage the widest possible use of a certain library, so that it becomes +a de-facto standard. To achieve this, non-free programs must be +allowed to use the library. A more frequent case is that a free +library does the same job as widely used non-free libraries. In this +case, there is little to gain by limiting the free library to free +software only, so we use the Lesser General Public License. + + In other cases, permission to use a particular library in non-free +programs enables a greater number of people to use a large body of +free software. For example, permission to use the GNU C Library in +non-free programs enables many more people to use the whole GNU +operating system, as well as its variant, the GNU/Linux operating +system. + + Although the Lesser General Public License is Less protective of the +users' freedom, it does ensure that the user of a program that is +linked with the Library has the freedom and the wherewithal to run +that program using a modified version of the Library. + + The precise terms and conditions for copying, distribution and +modification follow. Pay close attention to the difference between a +"work based on the library" and a "work that uses the library". The +former contains code derived from the library, whereas the latter must +be combined with the library in order to run. + +------------------------------------------------------------------------------- + + GNU LESSER GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License Agreement applies to any software library or other +program which contains a notice placed by the copyright holder or +other authorized party saying it may be distributed under the terms of +this Lesser General Public License (also called "this License"). +Each licensee is addressed as "you". + + A "library" means a collection of software functions and/or data +prepared so as to be conveniently linked with application programs +(which use some of those functions and data) to form executables. + + The "Library", below, refers to any such software library or work +which has been distributed under these terms. A "work based on the +Library" means either the Library or any derivative work under +copyright law: that is to say, a work containing the Library or a +portion of it, either verbatim or with modifications and/or translated +straightforwardly into another language. (Hereinafter, translation is +included without limitation in the term "modification".) + + "Source code" for a work means the preferred form of the work for +making modifications to it. For a library, complete source code means +all the source code for all modules it contains, plus any associated +interface definition files, plus the scripts used to control compilation +and installation of the library. + + Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running a program using the Library is not restricted, and output from +such a program is covered only if its contents constitute a work based +on the Library (independent of the use of the Library in a tool for +writing it). Whether that is true depends on what the Library does +and what the program that uses the Library does. + + 1. You may copy and distribute verbatim copies of the Library's +complete source code as you receive it, in any medium, provided that +you conspicuously and appropriately publish on each copy an +appropriate copyright notice and disclaimer of warranty; keep intact +all the notices that refer to this License and to the absence of any +warranty; and distribute a copy of this License along with the +Library. + + You may charge a fee for the physical act of transferring a copy, +and you may at your option offer warranty protection in exchange for a +fee. + +------------------------------------------------------------------------------- + + 2. You may modify your copy or copies of the Library or any portion +of it, thus forming a work based on the Library, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) The modified work must itself be a software library. + + b) You must cause the files modified to carry prominent notices + stating that you changed the files and the date of any change. + + c) You must cause the whole of the work to be licensed at no + charge to all third parties under the terms of this License. + + d) If a facility in the modified Library refers to a function or a + table of data to be supplied by an application program that uses + the facility, other than as an argument passed when the facility + is invoked, then you must make a good faith effort to ensure that, + in the event an application does not supply such function or + table, the facility still operates, and performs whatever part of + its purpose remains meaningful. + + (For example, a function in a library to compute square roots has + a purpose that is entirely well-defined independent of the + application. Therefore, Subsection 2d requires that any + application-supplied function or table used by this function must + be optional: if the application does not supply it, the square + root function must still compute square roots.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Library, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Library, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote +it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Library. + +In addition, mere aggregation of another work not based on the Library +with the Library (or with a work based on the Library) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may opt to apply the terms of the ordinary GNU General Public +License instead of this License to a given copy of the Library. To do +this, you must alter all the notices that refer to this License, so +that they refer to the ordinary GNU General Public License, version 2, +instead of to this License. (If a newer version than version 2 of the +ordinary GNU General Public License has appeared, then you can specify +that version instead if you wish.) Do not make any other change in +these notices. + +------------------------------------------------------------------------------- + + Once this change is made in a given copy, it is irreversible for +that copy, so the ordinary GNU General Public License applies to all +subsequent copies and derivative works made from that copy. + + This option is useful when you wish to copy part of the code of +the Library into a program that is not a library. + + 4. You may copy and distribute the Library (or a portion or +derivative of it, under Section 2) in object code or executable form +under the terms of Sections 1 and 2 above provided that you accompany +it with the complete corresponding machine-readable source code, which +must be distributed under the terms of Sections 1 and 2 above on a +medium customarily used for software interchange. + + If distribution of object code is made by offering access to copy +from a designated place, then offering equivalent access to copy the +source code from the same place satisfies the requirement to +distribute the source code, even though third parties are not +compelled to copy the source along with the object code. + + 5. A program that contains no derivative of any portion of the +Library, but is designed to work with the Library by being compiled or +linked with it, is called a "work that uses the Library". Such a +work, in isolation, is not a derivative work of the Library, and +therefore falls outside the scope of this License. + + However, linking a "work that uses the Library" with the Library +creates an executable that is a derivative of the Library (because it +contains portions of the Library), rather than a "work that uses the +library". The executable is therefore covered by this License. +Section 6 states terms for distribution of such executables. + + When a "work that uses the Library" uses material from a header file +that is part of the Library, the object code for the work may be a +derivative work of the Library even though the source code is not. +Whether this is true is especially significant if the work can be +linked without the Library, or if the work is itself a library. The +threshold for this to be true is not precisely defined by law. + + If such an object file uses only numerical parameters, data +structure layouts and accessors, and small macros and small inline +functions (ten lines or less in length), then the use of the object +file is unrestricted, regardless of whether it is legally a derivative +work. (Executables containing this object code plus portions of the +Library will still fall under Section 6.) + + Otherwise, if the work is a derivative of the Library, you may +distribute the object code for the work under the terms of Section 6. +Any executables containing that work also fall under Section 6, +whether or not they are linked directly with the Library itself. + +------------------------------------------------------------------------------- + + 6. As an exception to the Sections above, you may also combine or +link a "work that uses the Library" with the Library to produce a +work containing portions of the Library, and distribute that work +under terms of your choice, provided that the terms permit +modification of the work for the customer's own use and reverse +engineering for debugging such modifications. + + You must give prominent notice with each copy of the work that the +Library is used in it and that the Library and its use are covered by +this License. You must supply a copy of this License. If the work +during execution displays copyright notices, you must include the +copyright notice for the Library among them, as well as a reference +directing the user to the copy of this License. Also, you must do one +of these things: + + a) Accompany the work with the complete corresponding + machine-readable source code for the Library including whatever + changes were used in the work (which must be distributed under + Sections 1 and 2 above); and, if the work is an executable linked + with the Library, with the complete machine-readable "work that + uses the Library", as object code and/or source code, so that the + user can modify the Library and then relink to produce a modified + executable containing the modified Library. (It is understood + that the user who changes the contents of definitions files in the + Library will not necessarily be able to recompile the application + to use the modified definitions.) + + b) Use a suitable shared library mechanism for linking with the + Library. A suitable mechanism is one that (1) uses at run time a + copy of the library already present on the user's computer system, + rather than copying library functions into the executable, and (2) + will operate properly with a modified version of the library, if + the user installs one, as long as the modified version is + interface-compatible with the version that the work was made with. + + c) Accompany the work with a written offer, valid for at + least three years, to give the same user the materials + specified in Subsection 6a, above, for a charge no more + than the cost of performing this distribution. + + d) If distribution of the work is made by offering access to copy + from a designated place, offer equivalent access to copy the above + specified materials from the same place. + + e) Verify that the user has already received a copy of these + materials or that you have already sent this user a copy. + + For an executable, the required form of the "work that uses the +Library" must include any data and utility programs needed for +reproducing the executable from it. However, as a special exception, +the materials to be distributed need not include anything that is +normally distributed (in either source or binary form) with the major +components (compiler, kernel, and so on) of the operating system on +which the executable runs, unless that component itself accompanies +the executable. + + It may happen that this requirement contradicts the license +restrictions of other proprietary libraries that do not normally +accompany the operating system. Such a contradiction means you cannot +use both them and the Library together in an executable that you +distribute. + +------------------------------------------------------------------------------- + + 7. You may place library facilities that are a work based on the +Library side-by-side in a single library together with other library +facilities not covered by this License, and distribute such a combined +library, provided that the separate distribution of the work based on +the Library and of the other library facilities is otherwise +permitted, and provided that you do these two things: + + a) Accompany the combined library with a copy of the same work + based on the Library, uncombined with any other library + facilities. This must be distributed under the terms of the + Sections above. + + b) Give prominent notice with the combined library of the fact + that part of it is a work based on the Library, and explaining + where to find the accompanying uncombined form of the same work. + + 8. You may not copy, modify, sublicense, link with, or distribute +the Library except as expressly provided under this License. Any +attempt otherwise to copy, modify, sublicense, link with, or +distribute the Library is void, and will automatically terminate your +rights under this License. However, parties who have received copies, +or rights, from you under this License will not have their licenses +terminated so long as such parties remain in full compliance. + + 9. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Library or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Library (or any work based on the +Library), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Library or works based on it. + + 10. Each time you redistribute the Library (or any work based on the +Library), the recipient automatically receives a license from the +original licensor to copy, distribute, link with or modify the Library +subject to these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties with +this License. + +------------------------------------------------------------------------------- + + 11. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Library at all. For example, if a patent +license would not permit royalty-free redistribution of the Library by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Library. + +If any portion of this section is held invalid or unenforceable under any +particular circumstance, the balance of the section is intended to apply, +and the section as a whole is intended to apply in other circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 12. If the distribution and/or use of the Library is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Library under this License may add +an explicit geographical distribution limitation excluding those countries, +so that distribution is permitted only in or among countries not thus +excluded. In such case, this License incorporates the limitation as if +written in the body of this License. + + 13. The Free Software Foundation may publish revised and/or new +versions of the Lesser General Public License from time to time. +Such new versions will be similar in spirit to the present version, +but may differ in detail to address new problems or concerns. + +Each version is given a distinguishing version number. If the Library +specifies a version number of this License which applies to it and +"any later version", you have the option of following the terms and +conditions either of that version or of any later version published by +the Free Software Foundation. If the Library does not specify a +license version number, you may choose any version ever published by +the Free Software Foundation. + +------------------------------------------------------------------------------- + + 14. If you wish to incorporate parts of the Library into other free +programs whose distribution conditions are incompatible with these, +write to the author to ask for permission. For software which is +copyrighted by the Free Software Foundation, write to the Free +Software Foundation; we sometimes make exceptions for this. Our +decision will be guided by the two goals of preserving the free status +of all derivatives of our free software and of promoting the sharing +and reuse of software generally. + + NO WARRANTY + + 15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO +WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW. +EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR +OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY +KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE +LIBRARY IS WITH YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME +THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN +WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY +AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU +FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR +CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE +LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING +RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A +FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF +SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH +DAMAGES. + + END OF TERMS AND CONDITIONS + diff --git a/src/assistant/3rdparty/clucene/README b/src/assistant/3rdparty/clucene/README new file mode 100644 index 000000000..ee4f49369 --- /dev/null +++ b/src/assistant/3rdparty/clucene/README @@ -0,0 +1,92 @@ +CLucene README +============== + +------------------------------------------------------ +CLucene is a C++ port of Lucene. +It is a high-performance, full-featured text search +engine written in C++. CLucene is faster than lucene +as it is written in C++. +------------------------------------------------------ + +CLucene has contributions from many, see AUTHORS + +CLucene is distributed under the GNU Lesser General Public License (LGPL) + *or* +the Apache License, Version 2.0 +See the LGPL.license and APACHE.license for the respective license information. +Read COPYING for more about the license. + +Installation +------------ +* For Linux, MacOSX, cygwin and MinGW build information, read INSTALL. +* Boost.Jam files are provided in the root directory and subdirectories. +* Microsoft Visual Studio (6&7) are provided in the win32 folder. + +Mailing List +------------ +Questions and discussion should be directed to the CLucene mailing list + at clucene-developers@lists.sourceforge.net +Find subscription instructions at + http://lists.sourceforge.net/lists/listinfo/clucene-developers +Suggestions and bug reports can be made on our bug tracking database + (http://sourceforge.net/tracker/?group_id=80013&atid=558446) + +The latest version +------------------ +Details of the latest version can be found on the CLucene sourceforge project +web site: http://www.sourceforge.net/projects/clucene + +Documentation +------------- +Documentation is provided at http://clucene.sourceforge.net/doc/doxygen/html/ +You can also build your own documentation by running doxygen from the root directory +of clucene. +CLucene is a very close port of Java Lucene, so you can also try looking at the +Java Docs on http://lucene.apache.org/java/ + + +Performance +----------- +Very little benchmarking has been done on clucene. Andi Vajda posted some +limited statistics on the clucene list a while ago with the following results. + +There are 250 HTML files under $JAVA_HOME/docs/api/java/util for about +6108kb of HTML text. +org.apache.lucene.demo.IndexFiles with java and gcj: +on mac os x 10.3.1 (panther) powerbook g4 1ghz 1gb: + . running with java 1.4.1_01-99 : 20379 ms + . running with gcj 3.3.2 -O2 : 17842 ms + . running clucene 0.8.9's demo : 9930 ms + +I recently did some more tests and came up with these rough tests: +663mb (797 files) of Guttenberg texts +on a Pentium 4 running Windows XP with 1 GB of RAM. Indexing max 100,000 fields +• Jlucene: 646453ms. peak mem usage ~72mb, avg ~14mb ram +• Clucene: 232141. peak mem usage ~60, avg ~4mb ram + +Searching indexing using 10,000 single word queries +• Jlucene: ~60078ms and used ~13mb ram +• Clucene: ~48359ms and used ~4.2mb ram + +Platform notes +-------------- + +'Too many open files' +Some platforms don't provide enough file handles to run CLucene properly. +To solve this, increase the open file limit: + +On Solaris: +ulimit -n 1024 +set rlim_fd_cur=1024 + +Acknowledgments +---------------- + +The Apache Lucene project is the basis for this software, so the biggest +acknoledgment goes to that project. + +We wish to acknowledge the following copyrighted works that +make up portions of the CLucene software: + +CLucene relies heavily on the use of autoconf and libtool to provide +a build environment. diff --git a/src/assistant/3rdparty/clucene/src/CLucene.h b/src/assistant/3rdparty/clucene/src/CLucene.h new file mode 100644 index 000000000..1eac800fd --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene.h @@ -0,0 +1,38 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +//Includes some standard headers for searching and indexing. +#ifndef _lucene_CLucene_ +#define _lucene_CLucene_ + +#include "CLucene/StdHeader.h" +#include "CLucene/debug/condition.h" +#include "CLucene/debug/mem.h" +#include "CLucene/index/IndexReader.h" +#include "CLucene/index/IndexWriter.h" +#include "CLucene/index/MultiReader.h" +#include "CLucene/index/Term.h" +#include "CLucene/search/IndexSearcher.h" +#include "CLucene/search/MultiSearcher.h" +#include "CLucene/search/DateFilter.h" +#include "CLucene/search/WildcardQuery.h" +#include "CLucene/search/FuzzyQuery.h" +#include "CLucene/search/PhraseQuery.h" +#include "CLucene/search/PrefixQuery.h" +#include "CLucene/search/RangeQuery.h" +#include "CLucene/search/BooleanQuery.h" +#include "CLucene/document/Document.h" +#include "CLucene/document/Field.h" +#include "CLucene/document/DateField.h" +#include "CLucene/store/Directory.h" +#include "CLucene/store/FSDirectory.h" +#include "CLucene/queryParser/QueryParser.h" +#include "CLucene/queryParser/MultiFieldQueryParser.h" +#include "CLucene/analysis/standard/StandardAnalyzer.h" +#include "CLucene/analysis/Analyzers.h" +#include "CLucene/util/Reader.h" + +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/CLBackwards.h b/src/assistant/3rdparty/clucene/src/CLucene/CLBackwards.h new file mode 100644 index 000000000..ffaf42824 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/CLBackwards.h @@ -0,0 +1,87 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _clucene_backwards_h +#define _clucene_backwards_h + +//In light of the recent major changes to clucene, +//this file should help to maintain some backwards compatibility +//include it after including StdHeader.h +// +//Note: I haven't tested this much, so please send me your changes + +//dirent is one of the most major changes that won't easily port. +//you can use the TCHAR copying macros, STRCPY_TtoA and STRCPY_AtoT +//to copy between different character types. + +//If you use stringPrintF, you will need to add the target string +//size parameter, because it is using _sntprintf... Change this if you +//want, but _sntprintf is much safer + +#define char_t TCHAR +#define uchar_t TCHAR +#define l_byte_t byte_t + +//#define stringSpn _tcsspn //not used in clucene anymore +#define stringCSpn _tcscspn +#define stringLength _tcslen +//#define stringToInteger _ttoi //not used in clucene anymore, use integer w/ base +#define stringFind _tcsstr +#define stringFindChar _tcschr +#define stringCompare _tcscmp +#define stringNCopy _tcsncpy +#define stringCopy _tcscpy +#define stringCat _tcscat +//#define stringToken _tcstok //not used in clucene anymore +#define stringPrintF _sntprintf //you will have errors, because now we used printf w/ bufferlen count +#define printFormatted _tprintf + +//conversion functions +#define integerToString _i64tot +#define stringToIntegerBase _tcstoi64 +#define stringToFloat _tcstod + +//file find structures +#define Cmd_Stat fileStat +#define Struct_Stat fileStat +#define stringICompare _tcsicmp +#define stringNCompare _tcsncmp +#define stringDifference _tcscmp + +//character conversion functions +#define isSpace _istspace +#define isDigit _istwdigit +#define isAlNum _istwalnum +#define toLower _totlower +#define stringUpper _tcsupr +//#define stringLower _tcslwr //not used in clucene anymore + +#define _THROWX(y) _THROWT(y) +#define _THROWC(y) _THROWA(y) + +//file naming stuff - remember we have changed all names to file naming lower case +#define fileRename _rename +#define fileFullName(abs,rel) _realpath(rel,abs) +#define makeDirectory _tmkdir +#define unlinkFile _unlink + +//no longer supported definitions +#ifdef _UNICODE + #define TO_CHAR_T STRDUP_AtoT + #define _cout wcout + #define _cin wcin + #define _cerr wcerr +#else + #define TO_CHAR_T STRDUP_WtoT + #define _cout cout + #define _cin cin + #define _cerr cerr +#endif + +//some headers that used to be automatically included: +#include "CLucene/util/dirent.h" //if we have dirent, then the native one will be used + +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/CLConfig.h b/src/assistant/3rdparty/clucene/src/CLucene/CLConfig.h new file mode 100644 index 000000000..c63c083ff --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/CLConfig.h @@ -0,0 +1,304 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _lucene_Config_ +#define _lucene_Config_ + + +//////////////////////////////////////////////////////////////////// +// this settings should be set up in the compiler, +// but are put here for reference as to what could be defined +//////////////////////////////////////////////////////////////////// +// +//define this if you want debugging code to be enabled +//#define _DEBUG +// +//define this if you want condition debugging to be enabled +#if defined(_DEBUG) && !defined(_CL__CND_DEBUG) + #define _CL__CND_DEBUG +#endif +// +//define this to print out lots of information about merges, etc +//requires __CL__CND_DEBUG to be defined +//#define _CL_DEBUG_INFO stdout +// +//to disable namespaces define this +//#define DISABLE_NAMESPACE +// +//This is mostly for windows. If you have put the google sparse +//map code in your include path somewhere, then define this +//to use it. +//However, for msvc, there are no significant gains since there +//is already a compatible hashmap available. +//#define _CL_HAVE_GOOGLE_DENSE_HASH_MAP +// +//////////////////////////////////////////////////////////////////// + +//////////////////////////////////////////////////////////////////// +// These options can be set depending on the particular needs of +// Your application +//////////////////////////////////////////////////////////////////// +// +//define this to force the build into ascii mode +//#define _ASCII +// +//define this to force the build into ucs2 mode +//#define _UCS2 +// +//if a wide character is being converted to a ascii character and it +//cannot fit, this character is used instead. Required. +#define LUCENE_OOR_CHAR(c) ((char)(((unsigned short)c)&0xFF)) +// +//define if you would like to force clucene to use the internal +//character functions. +//Tests may display unpredictable behaviour if this is not defined. +#define LUCENE_USE_INTERNAL_CHAR_FUNCTIONS +// +//define this to enable mmap support in the fsdirectory IndexInput +//todo: only available for windows so far...need to add MMapInput.cpp to project +//EXPERIMENTAL +//#define LUCENE_FS_MMAP +// +//LOCK_DIR implementation: +//define this to set an exact directory for the lock dir (not recommended) +//all other methods of getting the temporary directory will be ignored +//#define LUCENE_LOCK_DIR "/tmp" +// +//define this to try and load the lock dir from this specified environment variable +#define LUCENE_LOCK_DIR_ENV_1 "TEMP" +//define this if you want to have look up this environment variable if the first one fails +#define LUCENE_LOCK_DIR_ENV_2 "TMP" +//define this if you want to have a fallback directory, if not defined then +//the lockdirectory will be the index directory +#define LUCENE_LOCK_DIR_ENV_FALLBACK "/tmp" +// +//////////////////////////////////////////////////////////////////// + + + +//////////////////////////////////////////////////////////////////// +// The following are search query options +// THe NO_* options can make CLucene faster and/or smaller +// special queries sometime require longer search times or may +// not be required +//////////////////////////////////////////////////////////////////// +// +//Define this to remove fuzzy query and sloppy scoring +//#define NO_FUZZY_QUERY +// +//Define to remove wildcard t*m or te?m to match term +//#define NO_WILDCARD_QUERY +// +//Define to remove prefix term query - ter* to match term or terms +//#define NO_PREFIX_QUERY +// +//Define to remove range (exlusive and inclusive) +//#define NO_RANGE_QUERY +// +//This must always be defined. They can be adjusted if required. But +//general Wildcard string would be '*' and Wildcard Char would be '?' +//Both are Required. +#define LUCENE_WILDCARDTERMENUM_WILDCARD_STRING '*' +#define LUCENE_WILDCARDTERMENUM_WILDCARD_CHAR '?' +// +//////////////////////////////////////////////////////////////////// + +//////////////////////////////////////////////////////////////////// +// memory handling configurations +//////////////////////////////////////////////////////////////////// +// +//If this is defined, lucene's configurations are changed +//to use less memory, but may run slower. +//todo: i dont think this actualy changes speed much, just memory +#define LUCENE_OPTIMIZE_FOR_MEMORY +// +//define this if you want the pointer tracking to be enabled +//this is a useful tool for memory leak tracking +//The LuceneBase can slow down the code a *lot* +#if defined(_DEBUG) + #if !defined(LUCENE_DISABLE_MEMTRACKING) && !defined(LUCENE_ENABLE_MEMLEAKTRACKING) + #define LUCENE_ENABLE_MEMLEAKTRACKING + #endif +#endif +// +//enable use of rich file/line tracking. use CL_FILELINE to pass +//to functions like stringDuplicate (or use CL_STRDUP* functions instead) and +//CLStringIntern::x. +#if defined(LUCENE_ENABLE_MEMLEAKTRACKING) + #define LUCENE_ENABLE_FILELINEINFO +#endif +// +//enable creation of clucene.log file. Logs every +//call to new operator. Must have LUCENE_ENABLE_MEMLEAKTRACKING enabled. +//writes log in this format. +//action,file name,file line,allocation size +//logging can be disabled by setting _lucene_disable_debuglogging to true +#if defined(LUCENE_ENABLE_MEMLEAKTRACKING) && defined(_DEBUG) +//#define LUCENE_ENABLE_CONSTRUCTOR_LOG +#endif +// +// +//enable this if you want to enable reference counting. This is +//not necessary or useful in most cases except when implementing wrappers +//which have reference counting. If the wrapper wraps a StringReader, +//for example, it should expect that the wrapped StringReader should not +//be deleted. However, when the stringreader is added into a Field, +//the Field usually takes over the stringReader and deletes it on completion. +//If reference counting is enabled, the wrapper can add a reference to any class +//and when _CLDECDELETE is called, the reference is decremented and only deleted +//if the refcount is zero. +#define LUCENE_ENABLE_REFCOUNT + + +//////////////////////////////////////////////////////////////////// +// These options allow you to remove certain implementations +// out of clucene so that they can be implemented in the client +// application +//////////////////////////////////////////////////////////////////// +// +//define this to your own setting if you would like to implement your own +//threading locking code. it should have the same sort of functions as +//mutex_default. If not defined, clucene will try and use posix,win32 critical +//sections, or a timer based mutex hack. +//#define _LUCENE_THREADMUTEX CL_NS(util)::mutex_default +// +//define this if you want to implement the _Cnd_OutDebug routine yourself +//you can then easily customise in your own application how to handle debug messages +//#define _CND_DEBUG_DONTIMPLEMENT_OUTDEBUG +// +//define this if you want to implement your own namespace macros +//#define _LUCENE_DONTIMPLEMENT_NS_MACROS +// +//define this if you do not want clucene to include any standard libraries. +//this could be useful if you want to use alternate libraries +//#define LUCENE_DISABLE_INCLUDES +// +//////////////////////////////////////////////////////////////////// + + +//////////////////////////////////////////////////////////////////// +// These options will be changed depending on your compiler/platform +// but can also be changed here if required +//////////////////////////////////////////////////////////////////// +// +//define this if multi-threading support is not required +//if not defined, multi-thread locking will +//occur (and its related processing overhead) +//note: it is recommended to disable multithreading if you do not need it +//there is a lot of overhead that can be avoided. +//#define _CL_DISABLE_MULTITHREADING +// +//if you want to define your own default file encoding. specify it +//here - normally defined in the platform specific headers +//#define PLATFORM_DEFAULT_READER_ENCODING CL_NS(util)::FileReader::ENCODING_ASCII +// +//disable hash implementations (if available) +//#define LUCENE_DISABLE_HASHING +//////////////////////////////////////////////////////////////////// + + + +//////////////////////////////////////////////////////////////////// +// These options should not be changed. But you can experiment with +// them to optimize performance +//////////////////////////////////////////////////////////////////// +// +//some defaults, wouldn't usually need to be changed +//Buffer size for input/output streams. Required. +#define LUCENE_STREAM_BUFFER_SIZE 1024 +// +// DSR:2004.08.19: +// Formerly, StringBuffer used 1024 as the default size of its internal buffer. +// However, StringBuffer is used primarily for token- and term-oriented +// processing, e.g. in StandardTokenizer. I've calculated that the average +// token (as produced by StandardTokenizer) in all .txt files distributed in +// the Project Gutenberg CD Image (August 2003 release) has only 6 characters. +// Although most languages are likely to have a longer average word length than +// English due to the popularity of "non-atomized" conjugation and declension +// mechanisms, 1024 is still vastly excessive. +// I made two changes intended to deliver better overall performance: +// a) Switched to a default StringBuffer character capacity of 32. Though 32 +// is longer than the average token, the high cost of realloc makes a +// slightly liberal default size optimal. I chose the default size of 32 +// after fairly extensive experimentation on the Gutenberg e-texts. The +// results are summarized in the following table: +// ------------------------------------------------------------------------ +// LUCENE_DEFAULT_TOKEN_BUFFER_SIZE value | % faster than default size 1024 +// ------------------------------------------------------------------------ +// 8 : 4% +// 16 : 7% +// 32 : 6% +// 64 : 3% +// A default size of 32 is actually slightly slower than 16, but I was +// experimenting on English text; I expect that 32 will maintain decent +// performance in languages such as German, and in technical documents +// with long tokens. +// +// b) To offset the switch to a smaller default buffer size, I implemented a +// more aggressive growth strategy. A StringBuffer now [at least] doubles +// the size of its internal buffer every time it needs to grow, rather +// than [at least] increasing by LUCENE_DEFAULT_TOKEN_BUFFER_SIZE no +// matter how many times it has already grown. +//Required. +#define LUCENE_DEFAULT_TOKEN_BUFFER_SIZE 32 +//todo: should implement a similar strategy in analysis/token +// +//Expert: The fraction of {@link TermDocs} entries stored in skip tables, +//used to accellerate {@link TermDocs#skipTo(int)}. Larger values result in +//smaller indices, greater acceleration, but fewer accelerable cases, while +//smaller values result in bigger indices, less acceleration and more +//accelerable cases. More detailed experiments would be useful here. */ +#define LUCENE_DEFAULT_TERMDOCS_SKIP_INTERVAL 16 +// +//Size of TermScore cache. Required. +#define LUCENE_SCORE_CACHE_SIZE 32 +// +//analysis options +//maximum length that the CharTokenizer uses. Required. +//By adjusting this value, you can greatly improve the performance of searching +//and especially indexing. Default is 255, but smaller numbers will decrease +//the amount of memory used as well as increasing the speed. +#define LUCENE_MAX_WORD_LEN 255 +//Maximum length of a token word. +//Should be the same or more than LUCENE_MAX_WORD_LEN +//if not defined, then no token limit, but may be slower +//if defined will be faster (up to 15% in some cases), but will use more memory +#ifndef LUCENE_OPTIMIZE_FOR_MEMORY + #define LUCENE_TOKEN_WORD_LENGTH LUCENE_MAX_WORD_LEN +#endif +// +//maximum field length. some optimisation can be done if a maximum field +//length is given... The smaller the better +#define LUCENE_MAX_FIELD_LEN 100 +// +//The initial value set to BooleanQuery::maxClauseCount. Default is 1024 +#define LUCENE_BOOLEANQUERY_MAXCLAUSECOUNT 1024 +// +//bvk: 12.3.2005 +//============================================================================== +//Previously the way the tokenizer has worked has been changed to optionally +//use a a fixed word length. I have implemented this in the Term class as well. +//It seems that by predefining the text length instead of using new TCHAR[x] +//in the constructor greatly improves the performance by 20-30% for certain +//operations. +//Maximum length of a term text. +//Should be the same or more than LUCENE_MAX_WORD_LEN +//if not defined, then no term text limit, but may be slower +//if defined will be faster (up to 30% in some cases), but will use more memory +#ifndef LUCENE_OPTIMIZE_FOR_MEMORY + #define LUCENE_TERM_TEXT_LENGTH LUCENE_MAX_WORD_LEN +#endif +// +//Size of the CharTokenizer buffersize. Required. +#define LUCENE_IO_BUFFER_SIZE 1024 +// +//the minimum amount the segment term enum should grow by. Must be at least 1 +#define LUCENE_SEGMENTTERMENUM_GROWSIZE 8 +// +//////////////////////////////////////////////////////////////////// + +#endif + diff --git a/src/assistant/3rdparty/clucene/src/CLucene/CLMonolithic.cpp b/src/assistant/3rdparty/clucene/src/CLucene/CLMonolithic.cpp new file mode 100644 index 000000000..e3c279876 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/CLMonolithic.cpp @@ -0,0 +1,115 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +* +* Changes are Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies). +------------------------------------------------------------------------------*/ +/* +* this is a monolithic file that can be used to compile clucene using one source file. +* it simplifies some build processes by avoiding static & dynamic compalation pitfalls. +* +* note: when creating a project add either this file, or all the other .cpp files, not both! +*/ +#include "CLucene/StdHeader.cpp" +#include "CLucene/analysis/Analyzers.cpp" +#include "CLucene/analysis/AnalysisHeader.cpp" +#include "CLucene/analysis/standard/StandardAnalyzer.cpp" +#include "CLucene/analysis/standard/StandardFilter.cpp" +#include "CLucene/analysis/standard/StandardTokenizer.cpp" +#include "CLucene/config/gunichartables.cpp" +#include "CLucene/config/repl_tcscasecmp.cpp" +#include "CLucene/config/repl_tcslwr.cpp" +#include "CLucene/config/repl_tcstod.cpp" +#include "CLucene/config/repl_lltot.cpp" +#include "CLucene/config/repl_tcstoll.cpp" +#include "CLucene/config/repl_tprintf.cpp" +#include "CLucene/config/threads.cpp" +#include "CLucene/config/utf8.cpp" +#include "CLucene/debug/condition.cpp" +#include "CLucene/debug/error.cpp" +#include "CLucene/debug/memtracking.cpp" +#include "CLucene/document/DateField.cpp" +#include "CLucene/document/Document.cpp" +#include "CLucene/document/Field.cpp" +#include "CLucene/index/CompoundFile.cpp" +#include "CLucene/index/DocumentWriter.cpp" +#include "CLucene/index/FieldInfos.cpp" +#include "CLucene/index/FieldsReader.cpp" +#include "CLucene/index/FieldsWriter.cpp" +#include "CLucene/index/IndexWriter.cpp" +#include "CLucene/index/IndexReader.cpp" +#include "CLucene/index/MultiReader.cpp" +#include "CLucene/index/SegmentInfos.cpp" +#include "CLucene/index/SegmentMergeInfo.cpp" +#include "CLucene/index/SegmentMergeQueue.cpp" +#include "CLucene/index/SegmentMerger.cpp" +#include "CLucene/index/SegmentReader.cpp" +#include "CLucene/index/SegmentTermDocs.cpp" +#include "CLucene/index/SegmentTermEnum.cpp" +#include "CLucene/index/SegmentTermPositions.cpp" +#include "CLucene/index/SegmentTermVector.cpp" +#include "CLucene/index/Term.cpp" +#include "CLucene/index/TermInfo.cpp" +#include "CLucene/index/TermInfosReader.cpp" +#include "CLucene/index/TermInfosWriter.cpp" +#include "CLucene/index/TermVectorReader.cpp" +#include "CLucene/index/TermVectorWriter.cpp" +#include "CLucene/queryParser/Lexer.cpp" +#include "CLucene/queryParser/MultiFieldQueryParser.cpp" +#include "CLucene/queryParser/QueryParser.cpp" +#include "CLucene/queryParser/QueryParserBase.cpp" +#include "CLucene/queryParser/QueryToken.cpp" +#include "CLucene/queryParser/TokenList.cpp" +#include "CLucene/search/BooleanQuery.cpp" +#include "CLucene/search/BooleanScorer.cpp" +#include "CLucene/search/CachingWrapperFilter.cpp" +#include "CLucene/search/ChainedFilter.cpp" +#include "CLucene/search/DateFilter.cpp" +#include "CLucene/search/ConjunctionScorer.cpp" +#include "CLucene/search/ExactPhraseScorer.cpp" +#include "CLucene/search/Explanation.cpp" +#include "CLucene/search/FieldCache.cpp" +#include "CLucene/search/FieldCacheImpl.cpp" +#include "CLucene/search/FieldDocSortedHitQueue.cpp" +#include "CLucene/search/FieldSortedHitQueue.cpp" +#include "CLucene/search/FilteredTermEnum.cpp" +#include "CLucene/search/FuzzyQuery.cpp" +#include "CLucene/search/Hits.cpp" +#include "CLucene/search/HitQueue.cpp" +#include "CLucene/search/IndexSearcher.cpp" +#include "CLucene/search/MultiSearcher.cpp" +#include "CLucene/search/MultiTermQuery.cpp" +#include "CLucene/search/PhrasePositions.cpp" +#include "CLucene/search/PhraseQuery.cpp" +#include "CLucene/search/PhraseScorer.cpp" +#include "CLucene/search/PrefixQuery.cpp" +#include "CLucene/search/QueryFilter.cpp" +#include "CLucene/search/RangeQuery.cpp" +#include "CLucene/search/RangeFilter.cpp" +#include "CLucene/search/SearchHeader.cpp" +#include "CLucene/search/Similarity.cpp" +#include "CLucene/search/SloppyPhraseScorer.cpp" +#include "CLucene/search/Sort.cpp" +#include "CLucene/search/TermQuery.cpp" +#include "CLucene/search/TermScorer.cpp" +#include "CLucene/search/WildcardQuery.cpp" +#include "CLucene/search/WildcardTermEnum.cpp" +#include "CLucene/store/FSDirectory.cpp" +#include "CLucene/store/IndexInput.cpp" +#include "CLucene/store/Lock.cpp" +#include "CLucene/store/MMapInput.cpp" +#include "CLucene/store/IndexOutput.cpp" +#include "CLucene/store/RAMDirectory.cpp" +#include "CLucene/store/TransactionalRAMDirectory.cpp" +#include "CLucene/util/BitSet.cpp" +#include "CLucene/util/Equators.cpp" +#include "CLucene/util/FastCharStream.cpp" +#include "CLucene/util/fileinputstream.cpp" +#include "CLucene/util/Misc.cpp" +#include "CLucene/util/Reader.cpp" +#include "CLucene/util/StringBuffer.cpp" +#include "CLucene/util/StringIntern.cpp" +#include "CLucene/util/dirent.cpp" +#include "CLucene/util/ThreadLocal.cpp" diff --git a/src/assistant/3rdparty/clucene/src/CLucene/LuceneThreads.h b/src/assistant/3rdparty/clucene/src/CLucene/LuceneThreads.h new file mode 100644 index 000000000..cad07869f --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/LuceneThreads.h @@ -0,0 +1,72 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _LuceneThreads_h +#define _LuceneThreads_h +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#if defined(_CL_DISABLE_MULTITHREADING) + #define SCOPED_LOCK_MUTEX(theMutex) + #define DEFINE_MUTEX(x) + #define STATIC_DEFINE_MUTEX(x) + #define _LUCENE_SLEEP(x) + #define _LUCENE_CURRTHREADID 1 + #define _LUCENE_THREADID_TYPE char + + CL_NS_DEF(util) + class CLuceneThreadIdCompare + { + public: + enum + { // parameters for hash table + bucket_size = 4, // 0 < bucket_size + min_buckets = 8 + }; // min_buckets = 2 ^^ N, 0 < N + + bool operator()( char t1, char t2 ) const{ + return t1 < t2; + } + }; + CL_NS_END +#else + + #if defined(_LUCENE_DONTIMPLEMENT_THREADMUTEX) + //do nothing + #elif defined(_CL_HAVE_PTHREAD) + #include "CLucene/config/threadPthread.h" + #elif defined(_CL_HAVE_WIN32_THREADS) || defined(_CLCOMPILER_MSVC) || defined(__MINGW32__) //note that mingw32 could have pthreads, so put this after. + #if !defined(_CL_HAVE_WIN32_THREADS) + #define _CL_HAVE_WIN32_THREADS + #endif + #include "CLucene/config/threadCSection.h" + #else + #error A valid thread library was not found + #endif //mutex types + + CL_NS_DEF(util) + /** @internal */ + class mutexGuard + { + private: + _LUCENE_THREADMUTEX* mrMutex; + mutexGuard(const mutexGuard& clone); + public: + mutexGuard( _LUCENE_THREADMUTEX& rMutex ); + ~mutexGuard(); + }; + CL_NS_END + + #define SCOPED_LOCK_MUTEX(theMutex) CL_NS(util)::mutexGuard theMutexGuard(theMutex); + #define DEFINE_MUTEX(theMutex) _LUCENE_THREADMUTEX theMutex; + #define STATIC_DEFINE_MUTEX(theMutex) static _LUCENE_THREADMUTEX theMutex; + +#endif //_CL_DISABLE_MULTITHREADING + + + +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/StdHeader.cpp b/src/assistant/3rdparty/clucene/src/CLucene/StdHeader.cpp new file mode 100644 index 000000000..d64c51f77 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/StdHeader.cpp @@ -0,0 +1,134 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +* +* Changes are Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies). +------------------------------------------------------------------------------*/ +#include "CLucene/StdHeader.h" +#include "CLucene/util/Misc.h" + +#include "CLucene/search/Sort.h" +#include "CLucene/search/Similarity.h" +#include "CLucene/search/FieldCache.h" +#include "CLucene/search/FieldSortedHitQueue.h" + +#if defined(_CLCOMPILER_MSVC) && defined(_DEBUG) +# define CRTDBG_MAP_ALLOC +# include <stdlib.h> +#ifndef UNDER_CE +# include <crtdbg.h> +#endif +#endif + +CL_NS_USE(util) + +TCHAR* _LUCENE_BLANK_STRING = _T(""); +char* _LUCENE_BLANK_ASTRING = ""; + +#ifndef Q_CC_MIPS +#if defined(_LUCENE_THREADMUTEX_USINGDEFAULT) +# if defined(_LUCENE_PRAGMA_WARNINGS) +# pragma message ("==================Using clunky thread mutex!!!==================") +# else +# if !defined(Q_OS_SOLARIS) +# warning "==================Using clunky thread mutex!!!==================" +# endif +# endif +#endif + +#if defined(_ASCII) +# if defined(_LUCENE_PRAGMA_WARNINGS) +# pragma message ("==================Using ascii mode!!!==================") +# else +# if !defined(Q_OS_SOLARIS) +# warning "==================Using ascii mode!!!==================" +# endif +# endif +#endif + +//This causes confusion, because CLucene doesn't really need hashed maps/sets. My experience with the +//hash maps on linux are that there are no significant improvements in using them (infact it adversely +//affected performance... therefore we'll just silently ignore +/*#if defined(LUCENE_DISABLE_HASHING) +# if defined(_LUCENE_PRAGMA_WARNINGS) +# pragma message ("==================Hashing not available or is disabled! CLucene may run slower than optimal ==================") +# else +# if !defined(Q_OS_SOLARIS) +# warning "==================Hashing not available or is disabled! CLucene may run slower than optimal ==================" +# endif +# endif +#endif*/ +#endif + +//clears all static memory. do not attempt to do anything else +//in clucene after calling this function +void _lucene_shutdown(){ + CL_NS(search)::FieldSortedHitQueue::Comparators.clear(); + _CLDELETE(CL_NS(search)::Sort::RELEVANCE); + _CLDELETE(CL_NS(search)::Sort::INDEXORDER); + _CLDELETE(CL_NS(search)::ScoreDocComparator::INDEXORDER); + _CLDELETE(CL_NS(search)::ScoreDocComparator::RELEVANCE); + _CLDELETE(CL_NS(search)::SortField::FIELD_SCORE); + _CLDELETE(CL_NS(search)::SortField::FIELD_DOC); + _CLDELETE(CL_NS(search)::FieldCache::DEFAULT); + + _CLLDELETE(CL_NS(search)::Similarity::getDefault()); + + CL_NS(util)::CLStringIntern::shutdown(); +} + +void CLDebugBreak(){ + //can be used for debug breaking... +#if defined(_CLCOMPILER_MSVC) && defined(_DEBUG) + _CrtDbgBreak(); +#else + int i=0; //a line to put breakpoint on +#endif +} + +//these are functions that lucene uses which +//are not replacement functions +char* lucenestrdup(const char* v CL_FILELINEPARAM){ + size_t len = strlen(v); + char* ret = new char[len+1]; + strncpy(ret,v,len+1); +#if defined(LUCENE_ENABLE_MEMLEAKTRACKING) +# if defined(LUCENE_ENABLE_FILELINEINFO) + CL_NS(debug)::LuceneBase::__cl_voidpadd((void*)ret,file,line,len); +# else + CL_NS(debug)::LuceneBase::__cl_voidpadd((void*)ret,__FILE__,__LINE__,len); +# endif +#endif + return ret; +} + +#ifdef _UCS2 +wchar_t* lucenewcsdup(const wchar_t* v CL_FILELINEPARAM){ + size_t len = _tcslen(v); + wchar_t* ret = new wchar_t[len+1]; + _tcsncpy(ret,v,len+1); +#if defined(LUCENE_ENABLE_MEMLEAKTRACKING) +# if defined(LUCENE_ENABLE_FILELINEINFO) + CL_NS(debug)::LuceneBase::__cl_voidpadd((void*)ret,file,line,len); +# else + CL_NS(debug)::LuceneBase::__cl_voidpadd((void*)ret,__FILE__,__LINE__,len); +# endif +#endif + return ret; +} +#endif //ucs2 + + +//ok, these are the exceptions, but these never +//exist on non-msvc platform, so lets put it here +#ifndef _CL_HAVE_FILELENGTH +int64_t lucene_filelength(int filehandle) +{ + struct fileStat info; + if (fileHandleStat(filehandle, &info) == -1) + _CLTHROWA( CL_ERR_IO,"fileStat error" ); + return info.st_size; +} +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/StdHeader.h b/src/assistant/3rdparty/clucene/src/CLucene/StdHeader.h new file mode 100644 index 000000000..fbb3fd949 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/StdHeader.h @@ -0,0 +1,501 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +* +* Changes are Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies). +------------------------------------------------------------------------------*/ +#ifndef lucene_stdheader_h +#define lucene_stdheader_h + +#if defined(OVERRIDE_DEFAULT_CLCONFIG) + #include "AltCLConfig.h" +#else + #include "CLucene/CLConfig.h" +#endif + +//first inclusion of compiler.h (it will be called again later) +#include "CLucene/config/compiler.h" + +extern void _lucene_shutdown(); +extern int _lucene_counter_break; //can set a watch on this +#if defined(LUCENE_ENABLE_MEMLEAKTRACKING) + extern bool _lucene_disable_debuglogging; //if LUCENE_ENABLE_CONSTRUCTOR_LOG is on, dont do log if this is true +#endif + +//////////////////////////////////////////////////////// +// default includes +//////////////////////////////////////////////////////// +#ifndef LUCENE_DISABLE_INCLUDES + +#include <stdio.h> + +#if defined(_CL_STDC_HEADERS) + #include <stdlib.h> + #include <stddef.h> +#else + #if defined(_CL_HAVE_STDLIB_H) + #include <stdlib.h> + #endif +#endif + +#if defined(_CL_HAVE_STRING_H) + #if !defined(_CL_STDC_HEADERS) && defined(_CL_HAVE_MEMORY_H) + #include <memory.h> + #endif + #include <string.h> +#elif defined(_CL_HAVE_STRINGS_H) + //note: as a side note, strtok is not thread-safe.. so be careful where you use it! + #error "strtok replacement for BSD has not been implemented" + #include <strings.h> + #if !defined(_CL_HAVE_STRCHR) + #define strchr index + #define strrchr rindex + #endif +#endif + +#if defined(_CL_HAVE_UNISTD_H) + #include <unistd.h> +#elif defined(_CL_HAVE_IO_H) && defined(_CL_HAVE_DIRECT_H) +#ifndef UNDER_CE + #include <io.h> + #include <direct.h> +#endif +#else + #error "Neither unistd.h or (io.h & direct.h) were available" +#endif + +#ifndef _CL_DISABLE_NATIVE_EXCEPTIONS + #ifdef _CL_HAVE_STDEXCEPT + #include <stdexcept> + #else + #error "CLucene can't compile with exception handling on because <stdexcept> header is not available" + #endif +#endif + +#if defined(_CL_STAT_MACROS_BROKEN) + #error "Haven't implemented STAT_MACROS_BROKEN fix yet" +#elif defined(_CL_HAVE_SYS_STAT_H) +#ifdef UNDER_CE + #include <types.h> +#else + #include <sys/stat.h> +#endif +#else + #error "Haven't implemented platforms with no sys/stat.h" +#endif + +#if defined(_CL_HAVE_STDARG_H) + #include <stdarg.h> +#else + #error "CLucene can compile, but some extras may not work" +#endif + +#if defined(_CL_HAVE_MATH_H) + #include <math.h> +#else + #error "CLucene can't compile without <math.h>" +#endif + +#if defined(_CL_HAVE_MAP) + #include <map> +#else + #error "CLucene can't compile without the map header" +#endif + +#if defined(_CL_HAVE_LIST) + #include <list> +#else + #error "CLucene can't compile without the list header" +#endif + +#if defined(_CL_HAVE_SET) + #include <set> +#else + #error "CLucene can't compile without the set header" +#endif + +#if defined(_CL_HAVE_VECTOR) + #include <vector> +#else + #error "CLucene can't compile without the vector header" +#endif + +#if !defined(LUCENE_DISABLE_HASHING) && defined(_CL_HAVE_HASH_MAP) && defined(_CL_HAVE_HASH_SET) + //hashing is all or nothing! + #include <hash_map> + #include <hash_set> +#elif !defined(LUCENE_DISABLE_HASHING) && defined(_CL_HAVE_EXT_HASH_MAP) && defined(_CL_HAVE_EXT_HASH_SET) + #include <ext/hash_map> + #include <ext/hash_set> +#elif !defined(LUCENE_DISABLE_HASHING) + #define LUCENE_DISABLE_HASHING +#endif +#if !defined(LUCENE_DISABLE_HASHING) && !defined(CL_NS_HASHING) + #define CL_NS_HASHING(func) std::func +#endif + +#if defined(_CL_HAVE_ALGORITHM) +# include <algorithm> +#else +# error "Can't compile clucene without <algorithm>" +#endif + +#if defined(_CL_HAVE_FUNCTIONAL) +# include <functional> +#else +# error "Can't compile clucene without <functional>" +#endif + +#if !defined(_CL_HAVE_PRINTF) + #error "CLucene can't compile without printf, replacements have not been implemented" +#endif + +#if !defined(_CL_HAVE_SNPRINTF) && !defined(_CL_HAVE__SNPRINTF) + #error "CLucene can't compile without snprintf, replacements have not been implemented" +#elif !defined(_CL_HAVE__SNPRINTF)&& defined(_CL_HAVE_SVNPRINTF) + #define _snprintf snprintf +#endif + +#if defined(_UCS2) + #if defined(_CL_HAVE_WCHAR_H) + #include <wchar.h> + #else + //actually the repl_wchar.h replacements header will + //always be included. It replaces some functions + //that are missing in some wchar.h headers. + #endif +#endif + +#if defined(_UCS2) && defined(_CL_HAVE_WCTYPE_H) + #include <wctype.h> +#elif defined(_ASCII) && defined(_CL_HAVE_CTYPE_H) + #include <ctype.h> + #undef LUCENE_USE_INTERNAL_CHAR_FUNCTIONS +#elif defined(_UCS2) + //must be in _UCS2 to use internal char functions + #undef LUCENE_USE_INTERNAL_CHAR_FUNCTIONS + #define LUCENE_USE_INTERNAL_CHAR_FUNCTIONS +#else + #error "Cannot compile in _ASCII without ctype.h" +#endif + +//always include replacement, some missing tchar defines +#include "CLucene/config/repl_tchar.h" + +#if defined(_CL_HAVE_ERRNO_H) +#ifndef UNDER_CE + #include <errno.h> +#endif +#else + #error "Haven't implemented platforms with no errno.h" +#endif + +#if defined(_CL_HAVE_FCNTL_H) +#ifndef UNDER_CE + #include <fcntl.h> +#endif +#else + #error "Haven't implemented platforms with no fcntl.h" +#endif + +#if defined(_CL_HAVE_WINDOWS_H) + #include <windows.h> +#endif + +#endif //LUCENE_DISABLE_INCLUDES +// +//////////////////////////////////////////////////////// + +//second inclusion of compiler.h +//this gives CompilerXXX.h a chance to include other headers +#include "CLucene/config/compiler.h" +// +//////////////////////////////////////////////////////// + + +//////////////////////////////////////////////////////// +// Character functions. +// Here we decide whose character functions to use +//////////////////////////////////////////////////////// +#if defined(LUCENE_USE_INTERNAL_CHAR_FUNCTIONS) + #define stringCaseFold cl_tcscasefold + #define stringCaseFoldCmp cl_tcscasefoldcmp + + #undef _istspace + #undef _istdigit + #undef _istalnum + #undef _istalpha + #undef _totlower + #undef _totupper + #define _istalnum cl_isalnum + #define _istalpha cl_isletter + #define _istspace cl_isspace + #define _istdigit cl_isdigit + #define _totlower cl_tolower + #define _totupper cl_toupper + + //here are some functions to help deal with utf8/ucs2 conversions + //lets let the user decide what mb functions to use... we provide pure utf8 ones no matter what. + /*#undef _mbtowc + #undef _mbstowcs + #undef _wctomb + #undef _wcstombs + #define _mbtowc lucene_mbstowc + #define _mbsstowcs lucene_mbstowcs + #define _wctomb lucene_wcto_mb + #define _wcstombs lucene_wcstombs*/ +#else + //we are using native functions + //here are some functions to help deal with utf8/ucs2 conversions + /*#define _mbtowc mbtowc + #define _wctomb wctomb + #define _mbstowcs mbstowcs + #define _wcstombs wcstombs*/ + + //we are using native character functions + #if defined(_ASCII) + #undef _istspace + #undef _istdigit + #undef _istalnum + #undef _istalpha + #undef _totlower + #undef _totupper + #define _istspace(x) isspace((unsigned char)x) + #define _istdigit(x) isdigit((unsigned char)x) + #define _istalnum(x) isalnum((unsigned char)x) + #define _istalpha(x) isalpha((unsigned char)x) + #define _totlower(x) tolower((unsigned char)x) + #define _totupper(x) toupper((unsigned char)x) + #endif +#endif + +//the methods contained in gunichartables.h +typedef unsigned long clunichar; +bool cl_isletter(clunichar c); +bool cl_isalnum(clunichar c); +bool cl_isdigit(clunichar c); +bool cl_isspace (clunichar c); +TCHAR cl_tolower (TCHAR c); +TCHAR cl_toupper (TCHAR c); + +int cl_tcscasefoldcmp(const TCHAR * dst, const TCHAR * src); +TCHAR* cl_tcscasefold( TCHAR * str, int len=-1 ); + +//we provide utf8 conversion functions +size_t lucene_utf8towc (wchar_t *ret, const char *s, size_t n); +size_t lucene_utf8towcs(wchar_t *, const char *, size_t maxslen); +size_t lucene_wctoutf8 (char * ret, const wchar_t str); +size_t lucene_wcstoutf8 (char *, const wchar_t *, size_t maxslen); +size_t lucene_utf8charlen(const char *p); + +///a blank string... +extern TCHAR* _LUCENE_BLANK_STRING; +#define LUCENE_BLANK_STRING _LUCENE_BLANK_STRING +extern char* _LUCENE_BLANK_ASTRING; +#define LUCENE_BLANK_ASTRING _LUCENE_BLANK_ASTRING + +/* Converts a string into a form that is independent of case. The + * result will not correspond to any particular case, but can be + * compared for equality or ordered with the results of calling + * stringCaseFold() on other strings. + * + * If we did not define this elsewhere, then just convert to lower case + */ +#ifndef stringCaseFold + #define stringCaseFold _tcslwr +#endif +/* Compares 2 strings using case folding (if available) + * If we did not define this elsewhere, then just compare + * using normal method + */ +#ifndef stringCaseFoldCmp + #define stringCaseFoldCmp _tcsicmp +#endif + +//now that all the character routines are completed, include the +//wchar.h replacements. +#include "CLucene/config/repl_wchar.h" //always include replacements + +//a replacement for _tcsdup. This uses new TCHAR[] instead of malloc, so that we can use delete[] to free +#if defined(LUCENE_ENABLE_FILELINEINFO) + #define CL_FILELINE ,__FILE__,__LINE__ + #define CL_FILELINEREF ,file,line ///<for passing the reference along to another function + #define CL_FILELINEREF2 ,file,line ///<for passing the reference along to another function + #define CL_FILELINEPARAM ,char* file,int line +#else + #define CL_FILELINE + #define CL_FILELINEREF + #define CL_FILELINEREF2 ,NULL,-1 + #define CL_FILELINEPARAM +#endif + +char* lucenestrdup(const char* v CL_FILELINEPARAM); +#if defined(_UCS2) + wchar_t* lucenewcsdup(const wchar_t* v CL_FILELINEPARAM); + #define stringDuplicate(x) lucenewcsdup(x CL_FILELINE) //don't change this... uses [] instead of malloc +#else + #define stringDuplicate(x) lucenestrdup(x CL_FILELINE) //don't change this... uses [] instead of malloc +#endif + +#define STRCPY_AtoA(target,src,len) strncpy(target,src,len) +#define STRDUP_AtoA(x) lucenestrdup(x CL_FILELINE) + +#if defined(_UCS2) + #define STRDUP_WtoW(x) lucenewcsdup(x CL_FILELINE) + #define STRDUP_TtoT STRDUP_WtoW + #define STRDUP_WtoT STRDUP_WtoW + #define STRDUP_TtoW STRDUP_WtoW + + #define STRDUP_AtoW(x) CL_NS(util)::Misc::_charToWide(x CL_FILELINE) + #define STRDUP_AtoT STRDUP_AtoW + + #define STRDUP_WtoA(x) CL_NS(util)::Misc::_wideToChar(x CL_FILELINE) + #define STRDUP_TtoA STRDUP_WtoA + + #define STRCPY_WtoW(target,src,len) _tcsncpy(target,src,len) + #define STRCPY_TtoW STRCPY_WtoW + #define STRCPY_WtoT STRCPY_WtoW + #define STRCPY_TtoT STRCPY_WtoW + + #define STRCPY_AtoW(target,src,len) CL_NS(util)::Misc::_cpycharToWide(src,target,len) + #define STRCPY_AtoT STRCPY_AtoW + + #define STRCPY_WtoA(target,src,len) CL_NS(util)::Misc::_cpywideToChar(src,target,len) + #define STRCPY_TtoA STRCPY_WtoA +#else + #define STRDUP_AtoT STRDUP_AtoA + #define STRDUP_TtoA STRDUP_AtoA + #define STRDUP_TtoT STRDUP_AtoA + + #define STRDUP_WtoT(x) xxxxxxxxxxxxxxx //not possible + #define STRCPY_WtoT(target,src,len) xxxxxxxxxxxxxxx //not possible + + #define STRCPY_AtoT STRCPY_AtoA + #define STRCPY_TtoA STRCPY_AtoA + #define STRCPY_TtoT STRCPY_AtoA +#endif + +// +//////////////////////////////////////////////////////// + +//////////////////////////////////////////////////////// +//namespace helper +//////////////////////////////////////////////////////// +#if defined(_LUCENE_DONTIMPLEMENT_NS_MACROS) + +#elif !defined(DISABLE_NAMESPACE) +// +// W A R N I N G +// ------------- +// +// adjustments here, need to be done in +// QTDIR/src/tools/assistant/lib/fulltextsearch/qclucene_global.h as well +// +# ifdef QT_NAMESPACE +# define CL_NS_DEF(sub) namespace QT_NAMESPACE { namespace lucene{ namespace sub{ +# define CL_NS_DEF2(sub,sub2) namespace QT_NAMESPACE { namespace lucene{ namespace sub{ namespace sub2 { + +# define CL_NS_END }}} +# define CL_NS_END2 }}}} + +# define CL_NS_USE(sub) using namespace QT_NAMESPACE::lucene::sub; +# define CL_NS_USE2(sub,sub2) using namespace QT_NAMESPACE::lucene::sub::sub2; + +# define CL_NS(sub) QT_NAMESPACE::lucene::sub +# define CL_NS2(sub,sub2) QT_NAMESPACE::lucene::sub::sub2 +# else +# define CL_NS_DEF(sub) namespace lucene{ namespace sub{ +# define CL_NS_DEF2(sub,sub2) namespace lucene{ namespace sub{ namespace sub2 { + +# define CL_NS_END }} +# define CL_NS_END2 }}} + +# define CL_NS_USE(sub) using namespace lucene::sub; +# define CL_NS_USE2(sub,sub2) using namespace lucene::sub::sub2; + +# define CL_NS(sub) lucene::sub +# define CL_NS2(sub,sub2) lucene::sub::sub2 +# endif +#else +# define CL_NS_DEF(sub) +# define CL_NS_DEF2(sub, sub2) +# define CL_NS_END +# define CL_NS_END2 +# define CL_NS_USE(sub) +# define CL_NS_USE2(sub,sub2) +# define CL_NS(sub) +# define CL_NS2(sub,sub2) +#endif + +#if defined(LUCENE_NO_STDC_NAMESPACE) + //todo: haven't actually tested this on a non-stdc compliant compiler + #define CL_NS_STD(func) ::func +#else + #define CL_NS_STD(func) std::func +#endif +// +//////////////////////////////////////////////////////// + +// +void CLDebugBreak(); //define a debugbreak function + + +//////////////////////////////////////////////////////////////// +// These are defines and functions used throughout clucene +//////////////////////////////////////////////////////////////// +#undef _T //remove any previously defined _T - required for ppc os +#if defined(_UCS2) + #define _T(x) L ## x +#else + #define _T(x) x +#endif + +//third inclusion of compiler.h +//this gives CompilerXXX.h a chance to fix any of the +//default settings +#include "CLucene/config/compiler.h" + +#if defined _MSC_VER && (_MSC_VER < 1300) +# define LUCENE_NO_STDC_NAMESPACE +#endif + +//use std namespace +#ifndef LUCENE_NO_STDC_NAMESPACE +using namespace std; +#endif + + +//////////////////////////////////////////////////////// +//misc shortcut defines +//////////////////////////////////////////////////////// + +//include the headers that we need practically everywhere +#include "CLucene/debug/error.h" //all delete/creation/mem debugging code +#include "CLucene/debug/condition.h" //conditional debugging (like assert) +#include "CLucene/debug/mem.h" //all delete/creation/mem debugging code +#include "LuceneThreads.h" //lucene threading support + +#include "CLucene/util/Misc.h" +#include "CLucene/util/Equators.h" +#include "CLucene/util/VoidList.h" +#include "CLucene/util/VoidMap.h" + + +CL_NS_DEF(util) +typedef CL_NS(util)::CLVector<TCHAR*> StringArray; +typedef CL_NS(util)::CLVector<TCHAR*, CL_NS(util)::Deletor::tcArray > StringArrayWithDeletor; +typedef CL_NS(util)::CLVector<const TCHAR*> StringArrayConst; +typedef CL_NS(util)::CLVector<const TCHAR*, CL_NS(util)::Deletor::tcArray > StringArrayConstWithDeletor; + +typedef CL_NS(util)::CLVector<char*> AStringArray; +typedef CL_NS(util)::CLVector<char*, CL_NS(util)::Deletor::acArray > AStringArrayWithDeletor; +typedef CL_NS(util)::CLVector<const char*> AStringArrayConst; +typedef CL_NS(util)::CLVector<const char*, CL_NS(util)::Deletor::acArray > AStringArrayConstWithDeletor; +CL_NS_END + +// +//////////////////////////////////////////////////////// + +#endif // STDHEADER_H diff --git a/src/assistant/3rdparty/clucene/src/CLucene/analysis/AnalysisHeader.cpp b/src/assistant/3rdparty/clucene/src/CLucene/analysis/AnalysisHeader.cpp new file mode 100644 index 000000000..03f61a038 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/analysis/AnalysisHeader.cpp @@ -0,0 +1,200 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#include "CLucene/StdHeader.h" +#include "AnalysisHeader.h" +#include "CLucene/util/StringBuffer.h" + +CL_NS_USE(util) +CL_NS_DEF(analysis) + +const TCHAR* Token::defaultType=_T("word"); + +Token::Token(): + _startOffset (0), + _endOffset (0), + _type ( defaultType ), + positionIncrement (1) +{ + _termTextLen = 0; +#ifndef LUCENE_TOKEN_WORD_LENGTH + _termText = NULL; + bufferTextLen = 0; +#else + _termText[0] = 0; //make sure null terminated + bufferTextLen = LUCENE_TOKEN_WORD_LENGTH+1; +#endif +} + +Token::~Token(){ +#ifndef LUCENE_TOKEN_WORD_LENGTH + free(_termText); +#endif +} + +Token::Token(const TCHAR* text, const int32_t start, const int32_t end, const TCHAR* typ): + _startOffset (start), + _endOffset (end), + _type ( typ ), + positionIncrement (1) +{ + _termTextLen = 0; +#ifndef LUCENE_TOKEN_WORD_LENGTH + _termText = NULL; + bufferTextLen = 0; +#else + _termText[0] = 0; //make sure null terminated + bufferTextLen = LUCENE_TOKEN_WORD_LENGTH+1; +#endif + setText(text); +} + +void Token::set(const TCHAR* text, const int32_t start, const int32_t end, const TCHAR* typ){ + _startOffset = start; + _endOffset = end; + _type = typ; + positionIncrement = 1; + setText(text); +} + +void Token::setText(const TCHAR* text){ + _termTextLen = _tcslen(text); + +#ifndef LUCENE_TOKEN_WORD_LENGTH + growBuffer(_termTextLen+1); + _tcsncpy(_termText,text,_termTextLen+1); +#else + if ( _termTextLen > LUCENE_TOKEN_WORD_LENGTH ){ + //in the case where this occurs, we will leave the endOffset as it is + //since the actual word still occupies that space. + _termTextLen=LUCENE_TOKEN_WORD_LENGTH; + } + _tcsncpy(_termText,text,_termTextLen+1); +#endif + _termText[_termTextLen] = 0; //make sure null terminated +} + +void Token::growBuffer(size_t size){ + if(bufferTextLen>=size) + return; +#ifndef LUCENE_TOKEN_WORD_LENGTH + if ( _termText == NULL ) + _termText = (TCHAR*)malloc( size * sizeof(TCHAR) ); + else + _termText = (TCHAR*)realloc( _termText, size * sizeof(TCHAR) ); + bufferTextLen = size; +#else + _CLTHROWA(CL_ERR_TokenMgr,"Couldn't grow Token buffer"); +#endif +} + +void Token::setPositionIncrement(int32_t posIncr) { + if (posIncr < 0) { + _CLTHROWA(CL_ERR_IllegalArgument,"positionIncrement must be >= 0"); + } + positionIncrement = posIncr; +} + +int32_t Token::getPositionIncrement() const { return positionIncrement; } + +// Returns the Token's term text. +const TCHAR* Token::termText() const{ + return (const TCHAR*) _termText; +} +size_t Token::termTextLength() { + if ( _termTextLen == -1 ) //it was invalidated by growBuffer + _termTextLen = _tcslen(_termText); + return _termTextLen; +} +void Token::resetTermTextLen(){ + _termTextLen=-1; +} +bool Token::OrderCompare::operator()( Token* t1, Token* t2 ) const{ + if(t1->startOffset()>t2->startOffset()) + return false; + if(t1->startOffset()<t2->startOffset()) + return true; + return true; +} +TCHAR* Token::toString() const{ + StringBuffer sb; + sb.append(_T("(")); + sb.append( _termText ); + sb.append(_T(",")); + sb.appendInt( _startOffset ); + sb.append(_T(",")); + sb.appendInt( _endOffset ); + + if (!_tcscmp( _type, _T("word")) == 0 ){ + sb.append(_T(",type=")); + sb.append(_type); + } + if (positionIncrement != 1){ + sb.append(_T(",posIncr=")); + sb.appendInt(positionIncrement); + } + sb.append(_T(")")); + + return sb.toString(); +} + + +Token* TokenStream::next(){ + Token* t = _CLNEW Token; //deprecated + if ( !next(t) ) + _CLDELETE(t); + return t; +} + + +TokenFilter::TokenFilter(TokenStream* in, bool deleteTS): + input(in), + deleteTokenStream(deleteTS) +{ +} +TokenFilter::~TokenFilter(){ + close(); +} + +// Close the input TokenStream. +void TokenFilter::close() { + if ( input != NULL ){ + input->close(); + if ( deleteTokenStream ) + _CLDELETE( input ); + } + input = NULL; +} + + + +Tokenizer::Tokenizer() { + input = NULL; +} + +Tokenizer::Tokenizer(CL_NS(util)::Reader* _input): + input(_input) +{ +} + +void Tokenizer::close(){ + if (input != NULL) { + // ? delete input; + input = NULL; + } +} + +Tokenizer::~Tokenizer(){ + close(); +} + + +int32_t Analyzer::getPositionIncrementGap(const TCHAR* fieldName) +{ + return 0; +} + +CL_NS_END diff --git a/src/assistant/3rdparty/clucene/src/CLucene/analysis/AnalysisHeader.h b/src/assistant/3rdparty/clucene/src/CLucene/analysis/AnalysisHeader.h new file mode 100644 index 000000000..0cfd9c684 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/analysis/AnalysisHeader.h @@ -0,0 +1,234 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _lucene_analysis_AnalysisHeader_ +#define _lucene_analysis_AnalysisHeader_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include "CLucene/util/Reader.h" + +CL_NS_DEF(analysis) + + +/** A Token is an occurence of a term from the text of a field. It consists of +* a term's text, the start and end offset of the term in the text of the field, +* and a type string. +* +* The start and end offsets permit applications to re-associate a token with +* its source text, e.g., to display highlighted query terms in a document +* browser, or to show matching text fragments in a KWIC (KeyWord In Context) +* display, etc. +* +* The type is an interned string, assigned by a lexical analyzer +* (a.k.a. tokenizer), naming the lexical or syntactic class that the token +* belongs to. For example an end of sentence marker token might be implemented +* with type "eos". The default token type is "word". +*/ +class Token:LUCENE_BASE{ +private: + int32_t _startOffset; // start in source text + int32_t _endOffset; // end in source text + const TCHAR* _type; // lexical type + int32_t positionIncrement; + size_t bufferTextLen; + +public: + #ifndef LUCENE_TOKEN_WORD_LENGTH + TCHAR* _termText; // the text of the term + #else + TCHAR _termText[LUCENE_TOKEN_WORD_LENGTH+1]; // the text of the term + #endif + int32_t _termTextLen; + static const TCHAR* defaultType; + + Token(); + ~Token(); + // Constructs a Token with the given text, start and end offsets, & type. + Token(const TCHAR* text, const int32_t start, const int32_t end, const TCHAR* typ=defaultType); + void set(const TCHAR* text, const int32_t start, const int32_t end, const TCHAR* typ=defaultType); + + size_t bufferLength(){ return bufferTextLen; } + void growBuffer(size_t size); + + /* Set the position increment. This determines the position of this + * token relative to the previous Token in a TokenStream, used in + * phrase searching. + * + * The default value is 1. + * + * Some common uses for this are: + * + * - Set it to zero to put multiple terms in the same position. This is + * useful if, e.g., a word has multiple stems. Searches for phrases + * including either stem will match. In this case, all but the first stem's + * increment should be set to zero: the increment of the first instance + * should be one. Repeating a token with an increment of zero can also be + * used to boost the scores of matches on that token. + * + * - Set it to values greater than one to inhibit exact phrase matches. + * If, for example, one does not want phrases to match across removed stop + * words, then one could build a stop word filter that removes stop words and + * also sets the increment to the number of stop words removed before each + * non-stop word. Then exact phrase queries will only match when the terms + * occur with no intervening stop words. + */ + void setPositionIncrement(int32_t posIncr); + int32_t getPositionIncrement() const; + const TCHAR* termText() const; + size_t termTextLength(); + void resetTermTextLen(); + void setText(const TCHAR* txt); + + /** + * Returns this Token's starting offset, the position of the first character + * corresponding to this token in the source text. + * + * Note that the difference between endOffset() and startOffset() may not be + * equal to termText.length(), as the term text may have been altered by a + * stemmer or some other filter. + */ + int32_t startOffset() const { return _startOffset; } + void setStartOffset(int32_t val){ _startOffset =val; } + + /** + * Returns this Token's ending offset, one greater than the position of the + * last character corresponding to this token in the source text. + */ + int32_t endOffset() const { return _endOffset; } + void setEndOffset(int32_t val){ _endOffset =val; } + + // Returns this Token's lexical type. Defaults to "word". + const TCHAR* type() const { return _type; } ///<returns reference + void setType(const TCHAR* val) { _type = val; } ///<returns reference + + TCHAR* toString() const; + + ///Compares the Token for their order + class OrderCompare:LUCENE_BASE, public CL_NS(util)::Compare::_base //<Token*> + { + public: + bool operator()( Token* t1, Token* t2 ) const; + }; +}; + +/** +* A TokenStream enumerates the sequence of tokens, either from +* fields of a document or from query text. +* <p> +* This is an abstract class. Concrete subclasses are: +* <ul> +* <li>{@link Tokenizer}, a TokenStream +* whose input is a Reader; and +* <li>{@link TokenFilter}, a TokenStream +* whose input is another TokenStream. +* </ul> +*/ +class TokenStream:LUCENE_BASE { +public: + /** Sets token to the next token in the stream, returns false at the EOS. */ + virtual bool next(Token* token) = 0; + + /** Releases resources associated with this stream. */ + virtual void close() = 0; + + virtual ~TokenStream(){ + } + + /* This is for backwards compatibility only. You should pass the token you want to fill + * to next(), this will save a lot of object construction and destructions. + * @deprecated. use next(token). Kept only to avoid breaking existing code. + */ + _CL_DEPRECATED(next(Token)) Token* next(); +}; + + +/** An Analyzer builds TokenStreams, which analyze text. It thus represents a + * policy for extracting index terms from text. + * <p> + * Typical implementations first build a Tokenizer, which breaks the stream of + * characters from the Reader into raw Tokens. One or more TokenFilters may + * then be applied to the output of the Tokenizer. + * <p> + * WARNING: You must override one of the methods defined by this class in your + * subclass or the Analyzer will enter an infinite loop. + */ +class Analyzer:LUCENE_BASE{ +public: + /** Creates a TokenStream which tokenizes all the text in the provided + Reader. Default implementation forwards to tokenStream(Reader) for + compatibility with older version. Override to allow Analyzer to choose + strategy based on document and/or field. Must be able to handle null + field name for backward compatibility. */ + virtual TokenStream* tokenStream(const TCHAR* fieldName, CL_NS(util)::Reader* reader)=0; + + virtual ~Analyzer(){ + } + + /** + * Invoked before indexing a Field instance if + * terms have already been added to that field. This allows custom + * analyzers to place an automatic position increment gap between + * Field instances using the same field name. The default value + * position increment gap is 0. With a 0 position increment gap and + * the typical default token position increment of 1, all terms in a field, + * including across Field instances, are in successive positions, allowing + * exact PhraseQuery matches, for instance, across Field instance boundaries. + * + * @param fieldName Field name being indexed. + * @return position increment gap, added to the next token emitted from {@link #tokenStream(TCHAR*, Reader*)} + */ + virtual int32_t getPositionIncrementGap(const TCHAR* fieldName); +}; + + +/** A Tokenizer is a TokenStream whose input is a Reader. +<p> +This is an abstract class. +*/ +class Tokenizer:public TokenStream { +protected: + /** The text source for this Tokenizer. */ + CL_NS(util)::Reader* input; + +public: + /** Construct a tokenizer with null input. */ + Tokenizer(); + /** Construct a token stream processing the given input. */ + Tokenizer(CL_NS(util)::Reader* _input); + + // ** By default, closes the input Reader. */ + virtual void close(); + virtual ~Tokenizer(); +}; + +/** A TokenFilter is a TokenStream whose input is another token stream. +<p> +This is an abstract class. +*/ +class TokenFilter:public TokenStream { +protected: + /** The source of tokens for this filter. */ + TokenStream* input; + /** If true then input will be deleted in the destructor */ + bool deleteTokenStream; + + /** Construct a token stream filtering the given input. + * + * @param in The TokenStream to filter from + * @param deleteTS If true, input will be deleted in the destructor + */ + TokenFilter(TokenStream* in, bool deleteTS=false); + virtual ~TokenFilter(); +public: + /** Close the input TokenStream. */ + void close(); +}; + +CL_NS_END +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/analysis/Analyzers.cpp b/src/assistant/3rdparty/clucene/src/CLucene/analysis/Analyzers.cpp new file mode 100644 index 000000000..142bbfb63 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/analysis/Analyzers.cpp @@ -0,0 +1,389 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +* +* Changes are Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies). +------------------------------------------------------------------------------*/ +#include "CLucene/StdHeader.h" +#include "Analyzers.h" +#include "CLucene/util/StringBuffer.h" + +CL_NS_USE(util) +CL_NS_DEF(analysis) + +CharTokenizer::CharTokenizer(Reader* in) : + Tokenizer(in), + offset(0), + bufferIndex(0), + dataLen(0), + ioBuffer(NULL) +{ + buffer[0]=0; +} + +TCHAR CharTokenizer::normalize(const TCHAR c) const +{ + return c; +} +bool CharTokenizer::next(Token* token){ + int32_t length = 0; + int32_t start = offset; + while (true) { + TCHAR c; + offset++; + if (bufferIndex >= dataLen) { + dataLen = input->read(ioBuffer, LUCENE_IO_BUFFER_SIZE); + if (dataLen == -1) + dataLen = 0; + bufferIndex = 0; + } + if (dataLen <= 0 ) { + if (length > 0) + break; + else + return false; + }else + c = ioBuffer[bufferIndex++]; + if (isTokenChar(c)) { // if it's a token TCHAR + + if (length == 0) // start of token + start = offset-1; + + buffer[length++] = normalize(c); // buffer it, normalized + + if (length == LUCENE_MAX_WORD_LEN) // buffer overflow! + break; + + } else if (length > 0) // at non-Letter w/ chars + break; // return 'em + + } + buffer[length]=0; + token->set( buffer, start, start+length); + return true; +} + +bool LetterTokenizer::isTokenChar(const TCHAR c) const { + return _istalpha(c)!=0; +} + + +TCHAR LowerCaseTokenizer::normalize(const TCHAR chr) const { + return _totlower(chr); +} + +bool WhitespaceTokenizer::isTokenChar(const TCHAR c) const{ + return _istspace(c)==0; //(return true if NOT a space) +} + +TokenStream* WhitespaceAnalyzer::tokenStream(const TCHAR* fieldName, Reader* reader) { + return _CLNEW WhitespaceTokenizer(reader); +} + +TokenStream* SimpleAnalyzer::tokenStream(const TCHAR* fieldName, Reader* reader) { + return _CLNEW LowerCaseTokenizer(reader); +} + +bool LowerCaseFilter::next(Token* t){ + if (!input->next(t)) + return false; + stringCaseFold( t->_termText ); + return true; +} + +StopFilter::StopFilter(TokenStream* in, bool deleteTokenStream, const TCHAR** stopWords): + TokenFilter(in, deleteTokenStream), + table(_CLNEW CLSetList<const TCHAR*>(false)) +{ + fillStopTable( table,stopWords ); +} + +void StopFilter::fillStopTable(CLSetList<const TCHAR*>* stopTable, + const TCHAR** stopWords) { + for (int32_t i = 0; stopWords[i]!=NULL; i++) + stopTable->insert(stopWords[i]); +} + +bool StopFilter::next(Token* token) { + // return the first non-stop word found + while (input->next(token)){ + if (table->find(token->_termText)==table->end()){ + return true; + } + } + + // reached EOS -- return nothing + return false; +} + +StopAnalyzer::StopAnalyzer():stopTable(false) +{ + StopFilter::fillStopTable(&stopTable,ENGLISH_STOP_WORDS); +} +StopAnalyzer::~StopAnalyzer() +{ +} +StopAnalyzer::StopAnalyzer( const TCHAR** stopWords) { + StopFilter::fillStopTable(&stopTable,stopWords); +} +TokenStream* StopAnalyzer::tokenStream(const TCHAR* fieldName, Reader* reader) { + return _CLNEW StopFilter(_CLNEW LowerCaseTokenizer(reader),true, &stopTable); +} + +const TCHAR* StopAnalyzer::ENGLISH_STOP_WORDS[] = +{ + _T("a"), _T("an"), _T("and"), _T("are"), _T("as"), _T("at"), _T("be"), _T("but"), _T("by"), + _T("for"), _T("if"), _T("in"), _T("into"), _T("is"), _T("it"), + _T("no"), _T("not"), _T("of"), _T("on"), _T("or"), _T("s"), _T("such"), + _T("t"), _T("that"), _T("the"), _T("their"), _T("then"), _T("there"), _T("these"), + _T("they"), _T("this"), _T("to"), _T("was"), _T("will"), _T("with"), NULL +}; + +PerFieldAnalyzerWrapper::PerFieldAnalyzerWrapper(Analyzer* defaultAnalyzer): + analyzerMap(true,true) +{ + this->defaultAnalyzer = defaultAnalyzer; +} +PerFieldAnalyzerWrapper::~PerFieldAnalyzerWrapper(){ + analyzerMap.clear(); + _CLDELETE(defaultAnalyzer); +} + +void PerFieldAnalyzerWrapper::addAnalyzer(const TCHAR* fieldName, Analyzer* analyzer) { + analyzerMap.put(STRDUP_TtoT(fieldName), analyzer); +} + +TokenStream* PerFieldAnalyzerWrapper::tokenStream(const TCHAR* fieldName, Reader* reader) { + Analyzer* analyzer = (fieldName==NULL?defaultAnalyzer:analyzerMap.get(fieldName)); + if (analyzer == NULL) { + analyzer = defaultAnalyzer; + } + + return analyzer->tokenStream(fieldName, reader); +} + + + +bool ISOLatin1AccentFilter::next(Token* token){ + if ( input->next(token) ){ + int32_t l = token->termTextLength(); + const TCHAR* chars = token->termText(); + bool doProcess = false; + for (int32_t i = 0; i < l; ++i) { +#ifdef _UCS2 + if ( chars[i] >= 0xC0 && chars[i] <= 0x178 ) { +#else + if ( (chars[i] >= 0xC0 && chars[i] <= 0xFF) || chars[i] < 0 ) { +#endif + doProcess = true; + break; + } + } + if ( !doProcess ) { + return true; + } + + StringBuffer output(l*2); + for (int32_t j = 0; j < l; j++) { + #ifdef _UCS2 + TCHAR c = chars[j]; + #else + unsigned char c = chars[j]; + #endif + switch (c) { + case 0xC0 : + case 0xC1 : + case 0xC2 : + case 0xC3 : + case 0xC4 : + case 0xC5 : + output.appendChar('A'); + break; + case 0xC6 : + output.append(_T("AE")); + break; + case 0xC7 : + output.appendChar('C'); + break; + case 0xC8 : + case 0xC9 : + case 0xCA : + case 0xCB : + output.appendChar('E'); + break; + case 0xCC : + case 0xCD : + case 0xCE : + case 0xCF : + output.appendChar('I'); + break; + case 0xD0 : + output.appendChar('D'); + break; + case 0xD1 : + output.appendChar('N'); + break; + case 0xD2 : + case 0xD3 : + case 0xD4 : + case 0xD5 : + case 0xD6 : + case 0xD8 : + output.appendChar('O'); + break; + case 0xDE : + output.append(_T("TH")); + break; + case 0xD9 : + case 0xDA : + case 0xDB : + case 0xDC : + output.appendChar('U'); + break; + case 0xDD : + output.appendChar('Y'); + break; + case 0xE0 : + case 0xE1 : + case 0xE2 : + case 0xE3 : + case 0xE4 : + case 0xE5 : + output.appendChar('a'); + break; + case 0xE6 : + output.append(_T("ae")); + break; + case 0xE7 : + output.appendChar('c'); + break; + case 0xE8 : + case 0xE9 : + case 0xEA : + case 0xEB : + output.appendChar('e'); + break; + case 0xEC : + case 0xED : + case 0xEE : + case 0xEF : + output.appendChar('i'); + break; + case 0xF0 : + output.appendChar('d'); + break; + case 0xF1 : + output.appendChar('n'); + break; + case 0xF2 : + case 0xF3 : + case 0xF4 : + case 0xF5 : + case 0xF6 : + case 0xF8 : + output.appendChar('o'); + break; + case 0xDF : + output.append(_T("ss")); + break; + case 0xFE : + output.append(_T("th")); + break; + case 0xF9 : + case 0xFA : + case 0xFB : + case 0xFC : + output.appendChar('u'); + break; + case 0xFD : + case 0xFF : + output.appendChar('y'); + break; + + #ifdef _UCS2 + case 0x152 : + output.append(_T("OE")); + break; + case 0x153 : + output.append(_T("oe")); + break; + case 0x178 : + output.appendChar('Y'); + break; + #endif + default : + output.appendChar(c); + break; + } + } + token->setText(output.getBuffer()); + return true; + } + return false; +} + + +TokenStream* KeywordAnalyzer::tokenStream(const TCHAR* fieldName, CL_NS(util)::Reader* reader){ + return _CLNEW KeywordTokenizer(reader); +} + +KeywordTokenizer::KeywordTokenizer(CL_NS(util)::Reader* input, int bufferSize): + Tokenizer(input) +{ + this->done = false; + if ( bufferSize < 0 ) + this->bufferSize = DEFAULT_BUFFER_SIZE; +} +KeywordTokenizer::~KeywordTokenizer(){ +} + +bool KeywordTokenizer::next(Token* token){ + if (!done) { + done = true; + int32_t rd; + const TCHAR* buffer=0; + while (true) { + rd = input->read(buffer, bufferSize); + if (rd == -1) + break; + token->growBuffer(token->_termTextLen +rd+1); + + int32_t cp = rd; + if ( token->_termTextLen + cp > token->bufferLength() ) + cp = token->bufferLength() - token->_termTextLen; + _tcsncpy(token->_termText+token->_termTextLen,buffer,cp); + token->_termTextLen+=rd; + } + token->_termText[token->_termTextLen]=0; + token->set(token->_termText,0,token->_termTextLen); + return true; + } + return false; +} + + +LengthFilter::LengthFilter(TokenStream* in, int _min, int _max): + TokenFilter(in) +{ + this->_min = _min; + this->_max = _max; +} + +bool LengthFilter::next(Token* token) +{ + // return the first non-stop word found + while ( input->next(token) ) + { + size_t len = token->termTextLength(); + if (len >= _min && len <= _max) + return true; + // note: else we ignore it but should we index each part of it? + } + // reached EOS -- return null + return false; +} + + +CL_NS_END diff --git a/src/assistant/3rdparty/clucene/src/CLucene/analysis/Analyzers.h b/src/assistant/3rdparty/clucene/src/CLucene/analysis/Analyzers.h new file mode 100644 index 000000000..a12bd653f --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/analysis/Analyzers.h @@ -0,0 +1,309 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _lucene_analysis_Analyzers_ +#define _lucene_analysis_Analyzers_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include "CLucene/util/Reader.h" +#include "AnalysisHeader.h" +#include "CLucene/util/Misc.h" + +CL_NS_DEF(analysis) + +/** An abstract base class for simple, character-oriented tokenizers.*/ +class CharTokenizer:public Tokenizer { +private: + int32_t offset, bufferIndex, dataLen; + TCHAR buffer[LUCENE_MAX_WORD_LEN+1]; + const TCHAR* ioBuffer; +protected: + + /** Returns true iff a character should be included in a token. This + * tokenizer generates as tokens adjacent sequences of characters which + * satisfy this predicate. Characters for which this is false are used to + * define token boundaries and are not included in tokens. */ + virtual bool isTokenChar(const TCHAR c) const = 0; + + /** Called on each token character to normalize it before it is added to the + * token. The default implementation does nothing. Subclasses may use this + * to, e.g., lowercase tokens. */ + virtual TCHAR normalize(const TCHAR c) const; + +public: + CharTokenizer(CL_NS(util)::Reader* in); + virtual ~CharTokenizer(){ + } + bool next(Token* token); +}; + + +/** A LetterTokenizer is a tokenizer that divides text at non-letters. That's +to say, it defines tokens as maximal strings of adjacent letters, as defined +by java.lang.Character.isLetter() predicate. + +Note: this does a decent job for most European languages, but does a terrible +job for some Asian languages, where words are not separated by spaces. */ +class LetterTokenizer:public CharTokenizer { +public: + // Construct a new LetterTokenizer. + LetterTokenizer(CL_NS(util)::Reader* in): + CharTokenizer(in) {} + + ~LetterTokenizer(){} +protected: + /** Collects only characters which satisfy _istalpha.*/ + bool isTokenChar(const TCHAR c) const; +}; + + + +/** +* LowerCaseTokenizer performs the function of LetterTokenizer +* and LowerCaseFilter together. It divides text at non-letters and converts +* them to lower case. While it is functionally equivalent to the combination +* of LetterTokenizer and LowerCaseFilter, there is a performance advantage +* to doing the two tasks at once, hence this (redundant) implementation. +* <P> +* Note: this does a decent job for most European languages, but does a terrible +* job for some Asian languages, where words are not separated by spaces. +*/ +class LowerCaseTokenizer:public LetterTokenizer { +public: + /** Construct a new LowerCaseTokenizer. */ + LowerCaseTokenizer(CL_NS(util)::Reader* in): + LetterTokenizer(in) {} + + ~LowerCaseTokenizer(){} +protected: + /** Collects only characters which satisfy _totlower. */ + TCHAR normalize(const TCHAR chr) const; +}; + + +/** A WhitespaceTokenizer is a tokenizer that divides text at whitespace. + * Adjacent sequences of non-Whitespace characters form tokens. */ +class WhitespaceTokenizer: public CharTokenizer { +public: + /** Construct a new WhitespaceTokenizer. */ + WhitespaceTokenizer(CL_NS(util)::Reader* in):CharTokenizer(in) {} + ~WhitespaceTokenizer(){} +protected: + /** Collects only characters which do not satisfy _istspace. + */ + bool isTokenChar(const TCHAR c) const; +}; + + +/** An Analyzer that uses WhitespaceTokenizer. */ +class WhitespaceAnalyzer: public Analyzer { + public: + TokenStream* tokenStream(const TCHAR* fieldName, CL_NS(util)::Reader* reader); + ~WhitespaceAnalyzer(){} +}; + +/** An Analyzer that filters LetterTokenizer with LowerCaseFilter. */ +class SimpleAnalyzer: public Analyzer { +public: + TokenStream* tokenStream(const TCHAR* fieldName, CL_NS(util)::Reader* reader); + ~SimpleAnalyzer(){} +}; + + + +/** +* Normalizes token text to lower case. +*/ +class LowerCaseFilter: public TokenFilter { +public: + LowerCaseFilter(TokenStream* in, bool deleteTokenStream):TokenFilter(in,deleteTokenStream) {} + ~LowerCaseFilter(){} + bool next(Token* token); +}; + + +/** + * Removes stop words from a token stream. + */ +class StopFilter: public TokenFilter { +private: + //bvk: i found this to work faster with a non-hash table. the number of items + //in the stop table is not like to make it worth having hashing. + CL_NS(util)::CLSetList<const TCHAR*>* table; +public: + // Constructs a filter which removes words from the input + // TokenStream that are named in the array of words. + StopFilter(TokenStream* in, bool deleteTokenStream, const TCHAR** stopWords); + + ~StopFilter(){} + + /** Constructs a filter which removes words from the input + * TokenStream that are named in the CLSetList. + */ + StopFilter(TokenStream* in, bool deleteTokenStream, CL_NS(util)::CLSetList<const TCHAR*>* stopTable): + TokenFilter(in, deleteTokenStream), + table(stopTable) + {} + + + /** + * Builds a Hashtable from an array of stop words, appropriate for passing + * into the StopFilter constructor. This permits this table construction to + * be cached once when an Analyzer is constructed. + * Note: the stopWords list must be a static list because the strings are not copied + */ + static void fillStopTable(CL_NS(util)::CLSetList<const TCHAR*>* stopTable, + const TCHAR** stopWords); + + /** + * Returns the next input Token whose termText() is not a stop word. + */ + bool next(Token* token); +}; + + + + +/** Filters LetterTokenizer with LowerCaseFilter and StopFilter. */ +class StopAnalyzer: public Analyzer { + CL_NS(util)::CLSetList<const TCHAR*> stopTable; + +public: + /** Builds an analyzer which removes words in ENGLISH_STOP_WORDS. */ + StopAnalyzer(); + ~StopAnalyzer(); + + /** Builds an analyzer which removes words in the provided array. */ + StopAnalyzer( const TCHAR** stopWords ); + /** Filters LowerCaseTokenizer with StopFilter. */ + TokenStream* tokenStream(const TCHAR* fieldName, CL_NS(util)::Reader* reader); + + /** An array containing some common English words that are not usually useful + for searching. */ + static const TCHAR* ENGLISH_STOP_WORDS[]; +}; + + + +/** + * This analyzer is used to facilitate scenarios where different + * fields require different analysis techniques. Use {@link #addAnalyzer} + * to add a non-default analyzer on a field name basis. + * + * <p>Example usage: + * + * <pre> + * PerFieldAnalyzerWrapper aWrapper = + * new PerFieldAnalyzerWrapper(new StandardAnalyzer()); + * aWrapper.addAnalyzer("firstname", new KeywordAnalyzer()); + * aWrapper.addAnalyzer("lastname", new KeywordAnalyzer()); + * </pre> + * + * <p>In this example, StandardAnalyzer will be used for all fields except "firstname" + * and "lastname", for which KeywordAnalyzer will be used. + * + * <p>A PerFieldAnalyzerWrapper can be used like any other analyzer, for both indexing + * and query parsing. + */ +class PerFieldAnalyzerWrapper : public Analyzer { +private: + Analyzer* defaultAnalyzer; + CL_NS(util)::CLHashMap<const TCHAR*, Analyzer*, CL_NS(util)::Compare::TChar, + CL_NS(util)::Equals::TChar, CL_NS(util)::Deletor::tcArray,CL_NS(util)::Deletor::Void<Analyzer> > analyzerMap; +public: + /** + * Constructs with default analyzer. + * + * @param defaultAnalyzer Any fields not specifically + * defined to use a different analyzer will use the one provided here. + */ + PerFieldAnalyzerWrapper(Analyzer* defaultAnalyzer); + ~PerFieldAnalyzerWrapper(); + + /** + * Defines an analyzer to use for the specified field. + * + * @param fieldName field name requiring a non-default analyzer + * @param analyzer non-default analyzer to use for field + */ + void addAnalyzer(const TCHAR* fieldName, Analyzer* analyzer); + TokenStream* tokenStream(const TCHAR* fieldName, CL_NS(util)::Reader* reader); +}; + + +/** + * A filter that replaces accented characters in the ISO Latin 1 character set + * (ISO-8859-1) by their unaccented equivalent. The case will not be altered. + * <p> + * For instance, 'à' will be replaced by 'a'. + * <p> + */ +class ISOLatin1AccentFilter: public TokenFilter { +public: + ISOLatin1AccentFilter(TokenStream* input, bool deleteTs): + TokenFilter(input,deleteTs) + { + } + + /** + * To replace accented characters in a String by unaccented equivalents. + */ + bool next(Token* token); +}; + + +/** + * Emits the entire input as a single token. + */ +class KeywordTokenizer: public Tokenizer { +private: + LUCENE_STATIC_CONSTANT(int, DEFAULT_BUFFER_SIZE = 256); + bool done; + int bufferSize; +public: + KeywordTokenizer(CL_NS(util)::Reader* input, int bufferSize=-1); + virtual ~KeywordTokenizer(); + bool next(Token* token); +}; + +/** + * "Tokenizes" the entire stream as a single token. This is useful + * for data like zip codes, ids, and some product names. + */ +class KeywordAnalyzer: public Analyzer { +public: + TokenStream* tokenStream(const TCHAR* fieldName, CL_NS(util)::Reader* reader); + virtual ~KeywordAnalyzer(){} +}; + + +/** + * Removes words that are too long and too short from the stream. + * + */ +class LengthFilter: public TokenFilter { +private: + int _min; + int _max; +public: + /** + * Build a filter that removes words that are too long or too + * short from the text. + */ + LengthFilter(TokenStream* in, int _min, int _max); + + /** + * Returns the next input Token whose termText() is the right len + */ + bool next(Token* token); +}; + + +CL_NS_END +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/analysis/standard/StandardAnalyzer.cpp b/src/assistant/3rdparty/clucene/src/CLucene/analysis/standard/StandardAnalyzer.cpp new file mode 100644 index 000000000..e0994c41a --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/analysis/standard/StandardAnalyzer.cpp @@ -0,0 +1,46 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#include "CLucene/StdHeader.h" +#include "StandardAnalyzer.h" + +#include "CLucene/util/VoidMap.h" +#include "CLucene/util/Reader.h" +#include "CLucene/analysis/AnalysisHeader.h" +#include "CLucene/analysis/Analyzers.h" +#include "StandardFilter.h" +#include "StandardTokenizer.h" + +CL_NS_USE(util) +CL_NS_USE(analysis) + +CL_NS_DEF2(analysis,standard) + + StandardAnalyzer::StandardAnalyzer(): + stopSet(false) + { + StopFilter::fillStopTable( &stopSet,CL_NS(analysis)::StopAnalyzer::ENGLISH_STOP_WORDS); + } + + StandardAnalyzer::StandardAnalyzer( const TCHAR** stopWords): + stopSet(false) + { + StopFilter::fillStopTable( &stopSet,stopWords ); + } + + StandardAnalyzer::~StandardAnalyzer(){ + } + + + TokenStream* StandardAnalyzer::tokenStream(const TCHAR* fieldName, Reader* reader) + { + TokenStream* ret = _CLNEW StandardTokenizer(reader); + ret = _CLNEW StandardFilter(ret,true); + ret = _CLNEW LowerCaseFilter(ret,true); + ret = _CLNEW StopFilter(ret,true, &stopSet); + return ret; + } +CL_NS_END2 diff --git a/src/assistant/3rdparty/clucene/src/CLucene/analysis/standard/StandardAnalyzer.h b/src/assistant/3rdparty/clucene/src/CLucene/analysis/standard/StandardAnalyzer.h new file mode 100644 index 000000000..9cce041df --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/analysis/standard/StandardAnalyzer.h @@ -0,0 +1,47 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _lucene_analysis_standard_StandardAnalyzer +#define _lucene_analysis_standard_StandardAnalyzer + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include "CLucene/util/VoidMap.h" +#include "CLucene/util/Reader.h" +#include "CLucene/analysis/AnalysisHeader.h" +#include "CLucene/analysis/Analyzers.h" +#include "StandardFilter.h" +#include "StandardTokenizer.h" + + +CL_NS_DEF2(analysis,standard) + + /** Represents a standard analyzer. */ + class StandardAnalyzer : public Analyzer + { + private: + CL_NS(util)::CLSetList<const TCHAR*> stopSet; + public: + /** Builds an analyzer.*/ + StandardAnalyzer(); + + /** Builds an analyzer with the given stop words. */ + StandardAnalyzer( const TCHAR** stopWords); + + ~StandardAnalyzer(); + + + /** + * Constructs a StandardTokenizer filtered by a + * StandardFilter, a LowerCaseFilter and a StopFilter. + */ + TokenStream* tokenStream(const TCHAR* fieldName, CL_NS(util)::Reader* reader) + ; + }; +CL_NS_END2 +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/analysis/standard/StandardFilter.cpp b/src/assistant/3rdparty/clucene/src/CLucene/analysis/standard/StandardFilter.cpp new file mode 100644 index 000000000..9869d2592 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/analysis/standard/StandardFilter.cpp @@ -0,0 +1,58 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#include "CLucene/StdHeader.h" +#include "StandardFilter.h" + +#include "../AnalysisHeader.h" +#include "../Analyzers.h" +#include "StandardTokenizerConstants.h" +#include "CLucene/util/StringBuffer.h" + +CL_NS_USE(analysis) +CL_NS_USE(util) +CL_NS_DEF2(analysis,standard) + + StandardFilter::StandardFilter(TokenStream* in, bool deleteTokenStream): + TokenFilter(in, deleteTokenStream) + { + } + + StandardFilter::~StandardFilter(){ + } + + bool StandardFilter::next(Token* t) { + if (!input->next(t)) + return false; + + TCHAR* text = t->_termText; + const int32_t textLength = t->termTextLength(); + const TCHAR* type = t->type(); + + if ( type == tokenImage[APOSTROPHE] && //we can compare the type directy since the type should always come from the tokenImage + ( textLength >= 2 && _tcsicmp(text+textLength-2, _T("'s"))==0 ) ) + { + // remove 's + text[textLength-2]=0; + t->resetTermTextLen(); + + return true; + + } else if ( type == tokenImage[ACRONYM] ) { // remove dots + int32_t j = 0; + for ( int32_t i=0;i<textLength;i++ ){ + if ( text[i] != '.' ) + text[j++]=text[i]; + } + text[j]=0; + return true; + + } else { + return true; + } + } + +CL_NS_END2 diff --git a/src/assistant/3rdparty/clucene/src/CLucene/analysis/standard/StandardFilter.h b/src/assistant/3rdparty/clucene/src/CLucene/analysis/standard/StandardFilter.h new file mode 100644 index 000000000..59657fdb3 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/analysis/standard/StandardFilter.h @@ -0,0 +1,37 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _lucene_analysis_standard_StandardFilter +#define _lucene_analysis_standard_StandardFilter + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include "../AnalysisHeader.h" +#include "../Analyzers.h" +#include "StandardTokenizerConstants.h" +#include "CLucene/util/StringBuffer.h" + +CL_NS_DEF2(analysis,standard) + + /** Normalizes tokens extracted with {@link StandardTokenizer}. */ + class StandardFilter: public TokenFilter{ + public: + // Construct filtering <i>in</i>. + StandardFilter(TokenStream* in, bool deleteTokenStream); + + ~StandardFilter(); + + + /** Returns the next token in the stream, or NULL at EOS. + * <p>Removes <tt>'s</tt> from the end of words. + * <p>Removes dots from acronyms. + */ + bool next(Token* token); + }; +CL_NS_END2 +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/analysis/standard/StandardTokenizer.cpp b/src/assistant/3rdparty/clucene/src/CLucene/analysis/standard/StandardTokenizer.cpp new file mode 100644 index 000000000..60f9a449c --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/analysis/standard/StandardTokenizer.cpp @@ -0,0 +1,446 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#include "CLucene/StdHeader.h" +#include "StandardTokenizer.h" + +CL_NS_USE(analysis) +CL_NS_USE(util) +CL_NS_DEF2(analysis,standard) + + const static TCHAR* tokenImageArray[] = { + _T("<EOF>"), + _T("<UNKNOWN>"), + _T("<ALPHANUM>"), + _T("<APOSTROPHE>"), + _T("<ACRONYM>"), + _T("<COMPANY>"), + _T("<EMAIL>"), + _T("<HOST>"), + _T("<NUM>"), + _T("<CJK>") + }; + const TCHAR** tokenImage = tokenImageArray; + + /* A bunch of shortcut macros, many of which make assumptions about variable + ** names. These macros enhance readability, not just convenience! */ + #define EOS (ch==-1 || rd->Eos()) + #define SPACE (_istspace((TCHAR)ch) != 0) + #define ALPHA (_istalpha((TCHAR)ch) != 0) + #define ALNUM (_istalnum(ch) != 0) + #define DIGIT (_istdigit(ch) != 0) + #define UNDERSCORE (ch == '_') + + #define _CJK ( (ch>=0x3040 && ch<=0x318f) || \ + (ch>=0x3300 && ch<=0x337f) || \ + (ch>=0x3400 && ch<=0x3d2d) || \ + (ch>=0x4e00 && ch<=0x9fff) || \ + (ch>=0xf900 && ch<=0xfaff) || \ + (ch>=0xac00 && ch<=0xd7af) ) //korean + + + #define DASH (ch == '-') + #define NEGATIVE_SIGN_ DASH + //#define POSITIVE_SIGN_ (ch == '+') + //#define SIGN (NEGATIVE_SIGN_ || POSITIVE_SIGN_) + + #define DOT (ch == '.') + #define DECIMAL DOT + + + //freebsd seems to have a problem with defines over multiple lines, so this has to be one long line + #define _CONSUME_AS_LONG_AS(conditionFails) while (true) { ch = readChar(); if (ch==-1 || (!(conditionFails) || str.len >= LUCENE_MAX_WORD_LEN)) { break; } str.appendChar(ch);} + + #define CONSUME_ALPHAS _CONSUME_AS_LONG_AS(ALPHA) + + #define CONSUME_DIGITS _CONSUME_AS_LONG_AS(DIGIT) + + /* otherMatches is a condition (possibly compound) under which a character + ** that's not an ALNUM or UNDERSCORE can be considered not to break the + ** span. Callers should pass false if only ALNUM/UNDERSCORE are acceptable. */ + #define CONSUME_WORD _CONSUME_AS_LONG_AS(ALNUM || UNDERSCORE) + + /* + ** Consume CJK characters + */ + #define CONSUME_CJK _CONSUME_AS_LONG_AS(_CJK) + + + /* It is considered that "nothing of value" has been read if: + ** a) The "read head" hasn't moved since specialCharPos was established. + ** or + ** b) The "read head" has moved by one character, but that character was + ** either whitespace or not among the characters found in the body of + ** a token (deliberately doesn't include the likes of '@'/'&'). */ + #define CONSUMED_NOTHING_OF_VALUE (rdPos == specialCharPos || (rdPos == specialCharPos+1 && ( SPACE || !(ALNUM || DOT || DASH || UNDERSCORE) ))) + + #define RIGHTMOST(sb) (sb.getBuffer()[sb.len-1]) + #define RIGHTMOST_IS(sb, c) (RIGHTMOST(sb) == c) + /* To discard the last character in a StringBuffer, we decrement the buffer's + ** length indicator and move the terminator back by one character. */ + #define SHAVE_RIGHTMOST(sb) (sb.getBuffer()[--sb.len] = '\0') + + //#define REMOVE_TRAILING_CHARS(sb, charMatchesCondition) { TCHAR* sbBuf = sb.getBuffer(); for (int32_t i = sb.len-1; i >= 0; i--) { TCHAR c = sbBuf[i]; if (charMatchesCondition) { sbBuf[--sb.len] = '\0'; } else {break;}}} + + /* Does StringBuffer sb contain any of the characters in string ofThese? */ + #define CONTAINS_ANY(sb, ofThese) (_tcscspn(sb.getBuffer(), _T(ofThese)) != static_cast<size_t>(sb.len)) + + + StandardTokenizer::StandardTokenizer(Reader* reader): + rd(_CLNEW FastCharStream(reader)), + /* rdPos is zero-based. It starts at -1, and will advance to the first + ** position when readChar() is first called. */ + rdPos(-1), + tokenStart(-1) + { + } + + StandardTokenizer::~StandardTokenizer() { + _CLDELETE(rd); + } + + int StandardTokenizer::readChar() { + /* Increment by 1 because we're speaking in terms of characters, not + ** necessarily bytes: */ + rdPos++; + return rd->GetNext(); + } + + void StandardTokenizer::unReadChar() { + rd->UnGet(); + rdPos--; + } + + inline bool StandardTokenizer::setToken(Token* t, StringBuffer* sb, TokenTypes tokenCode) { + t->setStartOffset(tokenStart); + t->setEndOffset(tokenStart+sb->length()); + t->setType(tokenImage[tokenCode]); + sb->getBuffer(); //null terminates the buffer + t->resetTermTextLen(); + return true; + } + + bool StandardTokenizer::next(Token* t) { + int ch=0; + while (!EOS) { + ch = readChar(); + + if ( ch == 0 || ch == -1 ){ + continue; + } else if (SPACE) { + continue; + } else if (ALPHA || UNDERSCORE) { + tokenStart = rdPos; + return ReadAlphaNum(ch,t); + } else if (DIGIT || NEGATIVE_SIGN_ || DECIMAL) { + tokenStart = rdPos; + /* ReadNumber returns NULL if it fails to extract a valid number; in + ** that case, we just continue. */ + if (ReadNumber(NULL, ch,t)) + return true; + } else if ( _CJK ){ + if ( ReadCJK(ch,t) ) + return true; + } + } + return false; + } + + bool StandardTokenizer::ReadNumber(const TCHAR* previousNumber, const TCHAR prev,Token* t) { + /* previousNumber is only non-NULL if this function already read a complete + ** number in a previous recursion, yet has been asked to read additional + ** numeric segments. For example, in the HOST "192.168.1.3", "192.168" is + ** a complete number, but this function will recurse to read the "1.3", + ** generating a single HOST token "192.168.1.3". */ + t->growBuffer(LUCENE_MAX_WORD_LEN+1);//make sure token can hold the next word + StringBuffer str(t->_termText,t->bufferLength(),true); //use stringbuffer to read data onto the termText + TokenTypes tokenType; + bool decExhausted; + if (previousNumber != NULL) { + str.prepend(previousNumber); + tokenType = CL_NS2(analysis,standard)::HOST; + decExhausted = false; + } else { + tokenType = CL_NS2(analysis,standard)::NUM; + decExhausted = (prev == '.'); + } + if ( str.len >= LUCENE_MAX_WORD_LEN ){ + //if a number is too long, i would say there is no point + //storing it, because its going to be the wrong number anyway? + //what do people think? + return false; + } + str.appendChar(prev); + + const bool signExhausted = (prev == '-'); + int ch = prev; + + CONSUME_DIGITS; + + if (str.len < 2 /* CONSUME_DIGITS didn't find any digits. */ + && ( + (signExhausted && !DECIMAL) + || (decExhausted /* && !DIGIT is implied, since CONSUME_DIGITS stopped on a non-digit. */) + ) + ) + { + /* We have either: + ** a) a negative sign that's not followed by either digit(s) or a decimal + ** b) a decimal that's not followed by digit(s) + ** so this is not a valid number. */ + if (!EOS) { + /* Unread the character that stopped CONSUME_DIGITS: */ + unReadChar(); + } + return false; + } + + /* We just read a group of digits. Is it followed by a decimal symbol, + ** implying that there might be another group of digits available? */ + if (!EOS) { + if (DECIMAL) { + if ( str.len >= LUCENE_MAX_WORD_LEN ) + return false; //read above for rationale + str.appendChar(ch); + } else { + unReadChar(); + goto SUCCESSFULLY_EXTRACTED_NUMBER; + } + + CONSUME_DIGITS; + if (!DIGIT && !DECIMAL) { + unReadChar(); + } else if (!EOS && DECIMAL && _istdigit(rd->Peek())) { + /* We just read the fractional digit group, but it's also followed by + ** a decimal symbol and at least one more digit, so this must be a + ** HOST rather than a real number. */ + return ReadNumber(str.getBuffer(), '.',t); + } + } + + SUCCESSFULLY_EXTRACTED_NUMBER: + TCHAR rightmost = RIGHTMOST(str); + /* Don't including a trailing decimal point. */ + if (rightmost == '.') { + SHAVE_RIGHTMOST(str); + unReadChar(); + rightmost = RIGHTMOST(str); + } + /* If all we have left is a negative sign, it's not a valid number. */ + if (rightmost == '-') { + CND_PRECONDITION (str.len == 1, "Number is invalid"); + return false; + } + + return setToken(t,&str,tokenType); + } + + bool StandardTokenizer::ReadAlphaNum(const TCHAR prev, Token* t) { + t->growBuffer(LUCENE_MAX_WORD_LEN+1);//make sure token can hold the next word + StringBuffer str(t->_termText,t->bufferLength(),true); //use stringbuffer to read data onto the termText + if ( str.len < LUCENE_MAX_WORD_LEN ){ + str.appendChar(prev); + int ch = prev; + + CONSUME_WORD; + if (!EOS && str.len < LUCENE_MAX_WORD_LEN-1 ) { //still have space for 1 more character? + switch(ch) { /* What follows the first alphanum segment? */ + case '.': + str.appendChar('.'); + return ReadDotted(&str, CL_NS2(analysis,standard)::UNKNOWN,t); + case '\'': + str.appendChar('\''); + return ReadApostrophe(&str,t); + case '@': + str.appendChar('@'); + return ReadAt(&str,t); + case '&': + str.appendChar('&'); + return ReadCompany(&str,t); + /* default: fall through to end of this function. */ + } + } + } + return setToken(t,&str,CL_NS2(analysis,standard)::ALPHANUM); + } + + bool StandardTokenizer::ReadCJK(const TCHAR prev, Token* t) { + t->growBuffer(LUCENE_MAX_WORD_LEN+1);//make sure token can hold the next word + StringBuffer str(t->_termText,t->bufferLength(),true); //use stringbuffer to read data onto the termText + if ( str.len < LUCENE_MAX_WORD_LEN ){ + str.appendChar(prev); + int ch = prev; + + CONSUME_CJK; + } + return setToken(t,&str,CL_NS2(analysis,standard)::CJK); + } + + + bool StandardTokenizer::ReadDotted(StringBuffer* _str, TokenTypes forcedType, Token* t) { + const int32_t specialCharPos = rdPos; + StringBuffer& str=*_str; + + /* A segment of a "dotted" is not allowed to begin with another dot or a dash. + ** Even though hosts, e-mail addresses, etc., could have a dotted-segment + ** that begins with a dot or a dash, it's far more common in source text + ** for a pattern like "abc.--def" to be intended as two tokens. */ + int ch = rd->Peek(); + if (!(DOT || DASH)) { + bool prevWasDot; + bool prevWasDash; + if (str.len == 0) { + prevWasDot = false; + prevWasDash = false; + } else { + prevWasDot = RIGHTMOST(str) == '.'; + prevWasDash = RIGHTMOST(str) == '-'; + } + while (!EOS && str.len < LUCENE_MAX_WORD_LEN-1 ) { + ch = readChar(); + const bool dot = ch == '.'; + const bool dash = ch == '-'; + + if (!(ALNUM || UNDERSCORE || dot || dash)) { + break; + } + /* Multiple dots or dashes in succession end the token. + ** Consider the following inputs: + ** "Visit windowsupdate.microsoft.com--update today!" + ** "In the U.S.A.--yes, even there!" */ + if ((dot || dash) && (prevWasDot || prevWasDash)) { + /* We're not going to append the character we just read, in any case. + ** As to the character before it (which is currently RIGHTMOST(str)): + ** Unless RIGHTMOST(str) is a dot, in which we need to save it so the + ** acronym-versus-host detection can work, we want to get rid of it. */ + if (!prevWasDot) { + SHAVE_RIGHTMOST(str); + } + break; + } + + str.appendChar(ch); + + prevWasDot = dot; + prevWasDash = dash; + } + } + + /* There's a potential StringBuffer.append call in the code above, which + ** could cause str to reallocate its internal buffer. We must wait to + ** obtain the optimization-oriented strBuf pointer until after the initial + ** potentially realloc-triggering operations on str. + ** Because there can be other such ops much later in this function, strBuf + ** is guarded within a block to prevent its use during or after the calls + ** that would potentially invalidate it. */ + { /* Begin block-guard of strBuf */ + TCHAR* strBuf = str.getBuffer(); + + bool rightmostIsDot = RIGHTMOST_IS(str, '.'); + if (CONSUMED_NOTHING_OF_VALUE) { + /* No more alphanums available for this token; shave trailing dot, if any. */ + if (rightmostIsDot) { + SHAVE_RIGHTMOST(str); + } + /* If there are no dots remaining, this is a generic ALPHANUM. */ + if (_tcschr(strBuf, '.') == NULL) { + forcedType = CL_NS2(analysis,standard)::ALPHANUM; + } + + /* Check the token to see if it's an acronym. An acronym must have a + ** letter in every even slot and a dot in every odd slot, including the + ** last slot (for example, "U.S.A."). */ + } else if (rightmostIsDot) { + bool isAcronym = true; + const int32_t upperCheckLimit = str.len - 1; /* -1 b/c we already checked the last slot. */ + + for (int32_t i = 0; i < upperCheckLimit; i++) { + const bool even = (i % 2 == 0); + ch = strBuf[i]; + if ( (even && !ALPHA) || (!even && !DOT) ) { + isAcronym = false; + break; + } + } + if (isAcronym) { + forcedType = CL_NS2(analysis,standard)::ACRONYM; + } else { + /* If it's not an acronym, we don't want the trailing dot. */ + SHAVE_RIGHTMOST(str); + /* If there are no dots remaining, this is a generic ALPHANUM. */ + if (_tcschr(strBuf, '.') == NULL) { + forcedType = CL_NS2(analysis,standard)::ALPHANUM; + } + } + } + } /* End block-guard of strBuf */ + + if (!EOS) { + if (ch == '@' && str.len < LUCENE_MAX_WORD_LEN-1) { + str.appendChar('@'); + return ReadAt(&str,t); + } else { + unReadChar(); + } + } + + return setToken(t,&str,CL_NS2(analysis,standard)::UNKNOWN + ? forcedType : CL_NS2(analysis,standard)::HOST); + } + + bool StandardTokenizer::ReadApostrophe(StringBuffer* _str, Token* t) { + StringBuffer& str=*_str; + + TokenTypes tokenType = CL_NS2(analysis,standard)::APOSTROPHE; + const int32_t specialCharPos = rdPos; + int ch=0; + + CONSUME_ALPHAS; + if (RIGHTMOST_IS(str, '\'') || CONSUMED_NOTHING_OF_VALUE) { + /* After the apostrophe, no more alphanums were available within this + ** token; shave trailing apostrophe and revert to generic ALPHANUM. */ + SHAVE_RIGHTMOST(str); + tokenType = CL_NS2(analysis,standard)::ALPHANUM; + } + if (!EOS) { + unReadChar(); + } + + return setToken(t,&str,tokenType); + } + + bool StandardTokenizer::ReadAt(StringBuffer* str, Token* t) { + ReadDotted(str, CL_NS2(analysis,standard)::EMAIL,t); + /* JLucene grammar indicates dots/digits not allowed in company name: */ + if (!CONTAINS_ANY((*str), ".0123456789")) { + setToken(t,str,CL_NS2(analysis,standard)::COMPANY); + } + return true; + } + + bool StandardTokenizer::ReadCompany(StringBuffer* _str, Token* t) { + StringBuffer& str = *_str; + const int32_t specialCharPos = rdPos; + int ch=0; + + CONSUME_WORD; + if (CONSUMED_NOTHING_OF_VALUE) { + /* After the ampersand, no more alphanums were available within this + ** token; shave trailing ampersand and revert to ALPHANUM. */ + CND_PRECONDITION(RIGHTMOST_IS(str, '&'),"ReadCompany failed"); + SHAVE_RIGHTMOST(str); + + + return setToken(t,&str,CL_NS2(analysis,standard)::ALPHANUM); + } + if (!EOS) { + unReadChar(); + } + + return setToken(t,&str,CL_NS2(analysis,standard)::COMPANY); + } + +CL_NS_END2 diff --git a/src/assistant/3rdparty/clucene/src/CLucene/analysis/standard/StandardTokenizer.h b/src/assistant/3rdparty/clucene/src/CLucene/analysis/standard/StandardTokenizer.h new file mode 100644 index 000000000..d4195be81 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/analysis/standard/StandardTokenizer.h @@ -0,0 +1,88 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _lucene_analysis_standard_StandardTokenizer +#define _lucene_analysis_standard_StandardTokenizer + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include "../AnalysisHeader.h" +#include "../Analyzers.h" +#include "StandardTokenizerConstants.h" +#include "CLucene/util/StringBuffer.h" +#include "CLucene/util/FastCharStream.h" +#include "CLucene/util/Reader.h" + + +CL_NS_DEF2(analysis,standard) + +/** A grammar-based tokenizer constructed with JavaCC. + * + * <p> This should be a good tokenizer for most European-language documents: + * + * <ul> + * <li>Splits words at punctuation characters, removing punctuation. However, a + * dot that's not followed by whitespace is considered part of a token. + * <li>Splits words at hyphens, unless there's a number in the token, in which case + * the whole token is interpreted as a product number and is not split. + * <li>Recognizes email addresses and internet hostnames as one token. + * </ul> + * + * <p>Many applications have specific tokenizer needs. If this tokenizer does + * not suit your application, please consider copying this source code + * directory to your project and maintaining your own grammar-based tokenizer. + */ + class StandardTokenizer: public Tokenizer { + private: + int32_t rdPos; + int32_t tokenStart; + + // Advance by one character, incrementing rdPos and returning the character. + int readChar(); + // Retreat by one character, decrementing rdPos. + void unReadChar(); + + // createToken centralizes token creation for auditing purposes. + //Token* createToken(CL_NS(util)::StringBuffer* sb, TokenTypes tokenCode); + inline bool setToken(Token* t, CL_NS(util)::StringBuffer* sb, TokenTypes tokenCode); + + bool ReadDotted(CL_NS(util)::StringBuffer* str, TokenTypes forcedType,Token* t); + + public: + CL_NS(util)::FastCharStream* rd; + + // Constructs a tokenizer for this Reader. + StandardTokenizer(CL_NS(util)::Reader* reader); + + ~StandardTokenizer(); + + /** Returns the next token in the stream, or false at end-of-stream. + * The returned token's type is set to an element of + * StandardTokenizerConstants::tokenImage. */ + bool next(Token* token); + + // Reads for number like "1"/"1234.567", or IP address like "192.168.1.2". + bool ReadNumber(const TCHAR* previousNumber, const TCHAR prev, Token* t); + + bool ReadAlphaNum(const TCHAR prev, Token* t); + + // Reads for apostrophe-containing word. + bool ReadApostrophe(CL_NS(util)::StringBuffer* str, Token* t); + + // Reads for something@... it may be a COMPANY name or a EMAIL address + bool ReadAt(CL_NS(util)::StringBuffer* str, Token* t); + + // Reads for COMPANY name like AT&T. + bool ReadCompany(CL_NS(util)::StringBuffer* str, Token* t); + + // Reads CJK characters + bool ReadCJK(const TCHAR prev, Token* t); + }; + +CL_NS_END2 +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/analysis/standard/StandardTokenizerConstants.h b/src/assistant/3rdparty/clucene/src/CLucene/analysis/standard/StandardTokenizerConstants.h new file mode 100644 index 000000000..3c95af45a --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/analysis/standard/StandardTokenizerConstants.h @@ -0,0 +1,30 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _lucene_analysis_standard_StandardTokenizerConstants +#define _lucene_analysis_standard_StandardTokenizerConstants + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +CL_NS_DEF2(analysis,standard) + enum TokenTypes { + _EOF, + UNKNOWN, + ALPHANUM, + APOSTROPHE, + ACRONYM, + COMPANY, + EMAIL, + HOST, + NUM, + CJK + }; + extern const TCHAR** tokenImage; + + CL_NS_END2 +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/config/CompilerAcc.h b/src/assistant/3rdparty/clucene/src/CLucene/config/CompilerAcc.h new file mode 100644 index 000000000..6ecd142be --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/config/CompilerAcc.h @@ -0,0 +1,166 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +* +* Changes are Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies). +------------------------------------------------------------------------------*/ +#if !defined(_lucene_COMPILER_ACC) +#define _lucene_COMPILER_ACC + +// It is internal CLucene header - DO NOT include it directly +#if !defined(_SUPPRESS_MAKE_BASED_CONFIG) +#if defined(_BUILD_FOR_QT_) +#include "fulltextsearch/qclucene-config_p.h" +#else +#include "CLucene/clucene-config.h" //make clucene-config.h file +#endif +#endif + +#if defined(_ASCII) +#undef _UCS2 +#elif defined(_UCS2) +// +#else +#define CL_CHARSET_GUESS +#endif + +//dont allow FS_MMAP if mmap is not available +#if defined(LUCENE_FS_MMAP) && !defined(_CL_HAVE_MMAP) +#error "LUCENE_FS_MMAP is defined and MMap doesn't appear to be available" +#endif + +#ifdef _CL_HAVE_NO_FUNCTION_TRY_BLOCKS +#undef _LUCENE_DISABLE_EXCEPTIONS +#define _LUCENE_DISABLE_EXCEPTIONS + +#error "this is bad if you made it here... your compiler seems not to have try/catch blocks." +#error "maybe you could implement an alternative solution for us? :)" +#endif + +#ifndef _CL_HAVE_NAMESPACES +#define DISABLE_NAMESPACE +#endif + +#define LUCENE_DISABLE_HASHING //we could enable this, but so far test show that the hashing is slower :( + +//define the file functions +#define fileSeek lseek +#define fileSize _filelength +#define fileStat stat +#define fileHandleStat fstat +#ifdef _CL_HAVE_TELL +#define fileTell tell +#else +//ftell (and probably soon ftell64) are POSIX standard functions, but tell and +//tell64 are not, so we define fileTell in terms of fileSeek. +#define fileTell(fhandle) fileSeek(fhandle, 0, SEEK_CUR) +#endif + +//this is needed early on so that CL_MAX_PATH can be correctly determined +//in the StdHeader.h. This was earlier causing problems with macosx. +//:: crash was due to realpath() that expects an output arguments that +//has at least the size of PATH_MAX (even if the result has a lower size) +#include <limits.h> + +#ifndef _CL_HAVE_WCHAR_T + typedef unsigned short wchar_t; +#endif + +#if defined(__CYGWIN__) +//cygwin seems to incorrectly define that it has wprintf??? +#undef _CL_HAVE_WPRINTF +#elif defined(__MINGW32__) +# ifndef _CL_HAVE_WINDOWS_H +# define _CL_HAVE_WINDOWS_H +# endif +#endif + + +/////////////////////////////////////////////////////////////////////////////// +//end _lucene_COMPILER_ACC +#elif !defined(_lucene_COMPILER_ACC2) +#define _lucene_COMPILER_ACC2 +//second inclusion + + //types + #if defined(_CL_HAVE_SYS_TYPES_H) + #include <sys/types.h> + #endif + #if defined(_CL_HAVE_INTTYPES_H) + #include <inttypes.h> + #elif defined(_CL_HAVE_STDINT_H) + #include <stdint.h> + #else + #if _CL_SIZEOF_UNSIGNED_LONG_LONG==8 + typedef unsigned long long uint64_t; + typedef long long int64_t; + #elif _CL_SIZEOF_UNSIGNED_LONG==8 + typedef unsigned long uint64_t; + typedef long int64_t; + #else + #error I do not know what to use for a uint64_t. + #endif + + /* Give us an unsigned 32-bit data type. */ + #if _CL_SIZEOF_UNSIGNED_LONG==4 + typedef unsigned long uint32_t; + typedef long int32_t; + #elif _CL_SIZEOF_UNSIGNED_INT==4 + typedef unsigned int uint32_t; + typedef int int32_t; + #else + #error I do not know what to use for a uint32_t. + #endif + + /* An unsigned 8-bit data type */ + #if _CL_SIZEOF_UNSIGNED_CHAR==1 + typedef unsigned char uint8_t; + #else + #error I do not know what to use for a uint8_t. + #endif + #endif + + //second chance to fix default settings + //this must be defined later, otherwise it messes up + //the standard libraries + #if !defined(__MINGW32__) + #define _close ::close + #define _read ::read + #endif + + //now that int64_t is defined, we can define this... + #ifndef _CL_HAVE_FILELENGTH + #undef fileSize + #define fileSize lucene_filelength + int64_t lucene_filelength(int handle); + #endif + +#elif !defined(_lucene_COMPILER_ACC3) +#define _lucene_COMPILER_ACC3 + //third inclusion + + #if !defined(__MINGW32__) + //define replacements + #define O_RANDOM 0 + #undef O_BINARY + #define O_BINARY 0 + #define _S_IREAD 0444 + #define _S_IWRITE 0333 // write and execute permissions + + //some functions that are needed - not charset dependent and not tchar type functions + #define _open open + #define _write write + #define _snprintf snprintf + + //clucene uses ascii for filename interactions + #define _realpath(rel,abs) realpath(rel,abs) + #define _mkdir(x) mkdir(x,0777) + #define _unlink unlink + #else + #define _realpath(rel,abs) _fullpath(abs,rel,CL_MAX_PATH) + #endif + //also required by mingw + #define _rename rename +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/config/CompilerBcb.h b/src/assistant/3rdparty/clucene/src/CLucene/config/CompilerBcb.h new file mode 100644 index 000000000..f1b423b50 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/config/CompilerBcb.h @@ -0,0 +1,68 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#if !defined(_lucene_COMPILER_BCB) +#define _lucene_COMPILER_BCB + +// It is internal CLucene header - DO NOT include it directly + +#include "CLucene/config/define_std.h" +#undef _CL_HAVE_STRTOLL +#undef _CL_HAVE_WCSTOLL + +#define _LUCENE_PRAGMA_ONCE +#define _LUCENE_PRAGMA_WARNINGS //tell lucene to display warnings using pragmas instead of #warning +#define LUCENE_DISABLE_HASHING +#define LUCENE_STATIC_CONSTANT(type, assignment) enum { assignment } + +#undef LUCENE_ENABLE_MEMLEAKTRACKING //it has been reported that this causes problems + +#define fileSize filelength +#define fileSeek lseek +#define fileTell tell +#define fileStat stat +#define fileHandleStat fstat + +#define O_RANDOM 0 + +//java long type +typedef __int64 int64_t; +typedef unsigned __int64 uint64_t; + +//java int type +typedef int int32_t; +typedef unsigned int uint32_t; + +//java byte type +typedef unsigned char uint8_t; + +//floating point type +//we are going to use qreal now +//typedef double float_t; + +//required type +typedef int intptr_t; + +#define _CL_ILONG(x) x ## L +#define _ILONGLONG(x) x ## i64 + + +#elif !defined(_lucene_COMPILER_BCB2) +#define _lucene_COMPILER_BCB2 + //second inclusion + + #define _open open + #define _timeb timeb + #define _ftime ::ftime + #define _rename rename + + #define _realpath(rel,abs) _fullpath(abs,rel,CL_MAX_PATH) + +#elif !defined(_lucene_COMPILER_BCB3) +#define _lucene_COMPILER_BCB3 + //third inclusion + +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/config/CompilerGcc.h b/src/assistant/3rdparty/clucene/src/CLucene/config/CompilerGcc.h new file mode 100644 index 000000000..41310cefe --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/config/CompilerGcc.h @@ -0,0 +1,175 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +* +* Changes are Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies). +------------------------------------------------------------------------------*/ +#if !defined(_lucene_COMPILER_GCC) +#define _lucene_COMPILER_GCC + +// It is internal CLucene header - DO NOT include it directly +#if !defined(_SUPPRESS_MAKE_BASED_CONFIG) + #if defined(_BUILD_FOR_QT_) + #include "qclucene-config_p.h" + #else + #include "CLucene/clucene-config.h" //make clucene-config.h file + #endif +#endif + +#if defined(_ASCII) + #undef _UCS2 +#elif defined(_UCS2) +// +#else + #define CL_CHARSET_GUESS +#endif + +//dont allow FS_MMAP if mmap is not available +#if defined(LUCENE_FS_MMAP) && !defined(_CL_HAVE_MMAP) + #error "LUCENE_FS_MMAP is defined and MMap doesn't appear to be available" +#endif + +#ifdef _CL_HAVE_NO_FUNCTION_TRY_BLOCKS + #undef _LUCENE_DISABLE_EXCEPTIONS + #define _LUCENE_DISABLE_EXCEPTIONS + + #error "this is bad if you made it here... your compiler seems not to have try/catch blocks." + #error "maybe you could implement an alternative solution for us? :)" +#endif + +#ifndef _CL_HAVE_NAMESPACES + #define DISABLE_NAMESPACE +#endif + +#define CL_NS_HASHING(func) __gnu_cxx::func +#define LUCENE_DISABLE_HASHING //we could enable this, but so far test show that the hashing is slower :( + +//define the file functions +#define fileSeek lseek +#define fileSize _filelength +#define fileStat stat +#define fileHandleStat fstat +#ifdef _CL_HAVE_TELL + #define fileTell tell +#else + //ftell (and probably soon ftell64) are POSIX standard functions, but tell and + //tell64 are not, so we define fileTell in terms of fileSeek. + #define fileTell(fhandle) fileSeek(fhandle, 0, SEEK_CUR) +#endif + +//this is needed early on so that CL_MAX_PATH can be correctly determined +//in the StdHeader.h. This was earlier causing problems with macosx. +//:: crash was due to realpath() that expects an output arguments that +//has at least the size of PATH_MAX (even if the result has a lower size) +#include <limits.h> + +#ifndef _CL_HAVE_WCHAR_T + typedef unsigned short wchar_t; +#endif + +#if defined(__CYGWIN__) + //cygwin seems to incorrectly define that it has wprintf??? + #undef _CL_HAVE_WPRINTF +#elif defined(__MINGW32__) + #ifndef _CL_HAVE_WINDOWS_H + #define _CL_HAVE_WINDOWS_H + #endif +#endif + + +/////////////////////////////////////////////////////////////////////////////// +//end _lucene_COMPILER_GCC1 +#elif !defined(_lucene_COMPILER_GCC2) +#define _lucene_COMPILER_GCC2 + //second inclusion + + //types + #if defined(_CL_HAVE_SYS_TYPES_H) + #include <sys/types.h> + #endif + #if defined(_CL_HAVE_INTTYPES_H) + #include <inttypes.h> + #elif defined(_CL_HAVE_STDINT_H) + #include <stdint.h> + #else + #if _CL_SIZEOF_UNSIGNED_LONG_LONG==8 + typedef unsigned long long uint64_t; + typedef long long int64_t; + #elif _CL_SIZEOF_UNSIGNED_LONG==8 + typedef unsigned long uint64_t; + typedef long int64_t; + #else + #error I do not know what to use for a uint64_t. + #endif + + /* Give us an unsigned 32-bit data type. */ + #if _CL_SIZEOF_UNSIGNED_LONG==4 + typedef unsigned long uint32_t; + typedef long int32_t; + #elif _CL_SIZEOF_UNSIGNED_INT==4 + typedef unsigned int uint32_t; + typedef int int32_t; + #else + #error I do not know what to use for a uint32_t. + #endif + + /* An unsigned 8-bit data type */ + #if _CL_SIZEOF_UNSIGNED_CHAR==1 + typedef unsigned char uint8_t; + #else + #error I do not know what to use for a uint8_t. + #endif + #endif + + //second chance to fix default settings + //this must be defined later, otherwise it messes up + //the standard libraries + #if !defined(__MINGW32__) + #define _close ::close + #define _read ::read + #endif + + //now that int64_t is defined, we can define this... + #ifndef _CL_HAVE_FILELENGTH + #undef fileSize + #define fileSize lucene_filelength + int64_t lucene_filelength(int handle); + #endif + +#elif !defined(_lucene_COMPILER_GCC3) +#define _lucene_COMPILER_GCC3 + //third inclusion + + #if !defined(__MINGW32__) + //define replacements + #define O_RANDOM 0 + #undef O_BINARY + #define O_BINARY 0 + #define _S_IREAD 0444 + #define _S_IWRITE 0333 // write and execute permissions + + //some functions that are needed - not charset dependent and not tchar type functions + #define _open open + #define _write write + #define _snprintf snprintf + + //clucene uses ascii for filename interactions + #define _realpath(rel,abs) realpath(rel,abs) + #define _mkdir(x) mkdir(x,0777) + #define _unlink unlink + #else + #define _realpath(rel,abs) _fullpath(abs,rel,CL_MAX_PATH) + #endif + //also required by mingw + #define _rename rename +#endif + +#if defined(__GNUC__) && (defined(__sgi) || (defined(Q_OS_SOLARIS) && Q_SOLARIS_VERSION < 10)) + #undef _CL_HAVE_FLOAT_T +#endif + +#if defined(__GNUC__) && defined(Q_OS_SOLARIS) && Q_SOLARIS_VERSION < 10 + #undef _CL_HAVE_WCSTOLL +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/config/CompilerMsvc.h b/src/assistant/3rdparty/clucene/src/CLucene/config/CompilerMsvc.h new file mode 100644 index 000000000..0021ea368 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/config/CompilerMsvc.h @@ -0,0 +1,136 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +* +* Changes are Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies). +------------------------------------------------------------------------------*/ +#if !defined(_lucene_COMPILER_MSVC) +#define _lucene_COMPILER_MSVC + +// It is internal CLucene header - DO NOT include it directly + +#include "CLucene/config/define_std.h" + +#if (_MSC_VER >= 1300) +//>= 7.0 + #if defined(_BUILD_FOR_QT_) + # pragma warning(disable: 4100) // disable unreferenced formal parameter + # pragma warning(disable: 4189) // disable local variable is initialized but not referenced + #endif + # pragma warning(disable: 4512) // This would be very annoying + # pragma warning(disable: 4290) // Ignore exception specification warning + # pragma warning(disable: 4250) // Ignore 'class1' : inherits 'class2::member' via dominance (e.g. in MultiReader) + // Check for STLport presence + #include <string> + #if (_MSC_VER < 1310) || defined(_STLPORT_VERSION) + #define CL_NS_HASHING(func) std::func //the namespace is different on VC 7.0 + #else + #define CL_NS_HASHING(func) stdext::func + #endif + #define LUCENE_STATIC_CONSTANT_SYNTAX 1 + + #if _MSC_FULL_VER >= 140050320 + #define _CL_DEPRECATE_TEXT(_Text) __declspec(deprecated(_Text)) + #else + #define _CL_DEPRECATE_TEXT(_Text) __declspec(deprecated) + #endif + +#elif (_MSC_VER >= 1200) +//6.0 +#ifdef LUCENE_ENABLE_MEMLEAKTRACKING + #define _CLDELETE_CARRAY(x) if (x!=NULL){CL_NS(debug)::LuceneBase::__cl_voidpremove((void*)x,__FILE__,__LINE__);delete[] __CONST_CAST(TCHAR*,x); x=NULL;} + #define _CLDELETE_CaARRAY(x) if (x!=NULL){CL_NS(debug)::LuceneBase::__cl_voidpremove((void*)x,__FILE__,__LINE__);delete[] __CONST_CAST(char*,x); x=NULL;} + #define _CLDELETE_LCARRAY(x) if (x!=NULL){CL_NS(debug)::LuceneBase::__cl_voidpremove((void*)x,__FILE__,__LINE__);delete[] __CONST_CAST(TCHAR*,x);} + #define _CLDELETE_LCaARRAY(x) if (x!=NULL){CL_NS(debug)::LuceneBase::__cl_voidpremove((void*)x,__FILE__,__LINE__);delete[] __CONST_CAST(char*,x);} +#else + #define _CLDELETE_CARRAY(x) if (x!=NULL){delete[] __CONST_CAST(TCHAR*,x); x=NULL;} + #define _CLDELETE_CaARRAY(x) if (x!=NULL){delete[] __CONST_CAST(char*,x); x=NULL;} + #define _CLDELETE_LCARRAY(x) if (x!=NULL){delete[] __CONST_CAST(TCHAR*,x);} + #define _CLDELETE_LCaARRAY(x) if (x!=NULL){delete[] __CONST_CAST(char*,x);} + +#endif + #define LUCENE_STATIC_CONSTANT_SYNTAX 2 + + # pragma warning(disable: 4786) // This would be very annoying + namespace std{ + # undef min // just in case + # undef max // just in case + + #define min(a,b) (a>b?b:a) + #define max(a,b) (a>b?a:b) + } + + //only 7.0+ has these function + #undef _CL_HAVE_LLTOA + #undef _CL_HAVE_LLTOAW + #undef _CL_HAVE_INTPTR_T + #undef _CL_HAVE_WCSTOLL + #undef _CL_HAVE_STRTOLL + #undef _CL_HAVE_HASH_MAP + #undef _CL_HAVE_HASH_SET + +#else +# error "This version of MSVC has not been tested. Please uncomment this line to try anyway. Please send a report to the Clucene's administration if successful" +#endif + +#if _MSC_VER >= 1020 + #define _LUCENE_PRAGMA_ONCE +#endif +#define _LUCENE_PRAGMA_WARNINGS //tell lucene to display warnings using pragmas instead of #warning + +//if we are compiling using single-threaded libraries, we can disable multi-threading stuff +#if !defined(_MT) && !defined(_CL_DISABLE_MULTITHREADING) + #define _CL_DISABLE_MULTITHREADING +#endif + +//msvc supports large files +#ifdef _LARGE_FILES +# define fileSize _filelengthi64 +# define fileSeek _lseeki64 +# define fileTell _telli64 +# define fileStat _stati64 +# define fileHandleStat _fstati64 +#else +# define fileSize _filelength +# define fileSeek _lseek +# define fileTell _tell +# define fileStat _stat +# define fileHandleStat _fstat +#endif + +//_rename is not defined??? +#define _rename rename + +#define CL_MAX_PATH 260 //give the windef.h value for this... +#define _realpath(rel,abs) _fullpath(abs,rel,CL_MAX_PATH) + +//java long type +typedef __int64 int64_t; +typedef unsigned __int64 uint64_t; + +//java int type +typedef int int32_t; +typedef unsigned int uint32_t; + +//java byte type +typedef unsigned char uint8_t; + +//floating point type +//we are going to use qreal now +//typedef double float_t; + +#define _CL_ILONG(x) x ## L +#define _ILONGLONG(x) x ## i64 + + +#elif !defined(_lucene_COMPILER_MSVC2) +#define _lucene_COMPILER_MSVC2 + //second inclusion + + +#elif !defined(_lucene_COMPILER_MSVC3) +#define _lucene_COMPILER_MSVC3 + //third inclusion +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/config/PlatformMac.h b/src/assistant/3rdparty/clucene/src/CLucene/config/PlatformMac.h new file mode 100644 index 000000000..9f6d6f421 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/config/PlatformMac.h @@ -0,0 +1,19 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +// It is internal CLucene header - DO NOT include it directly + +# define PATH_DELIMITER _T("/") +# define PATH_DELIMITERA "/" +# define PATH_DELIMITERC '/' + +# if (__GNUC__ < 3) && !defined( __APPLE_CC__) +// GCC strange "ignore std" mode works better if you pretend everything +// is in the std namespace, for the most part. +# define LUCENE_NO_STDC_NAMESPACE +# endif + +#undef _T //apple has something else strange here... diff --git a/src/assistant/3rdparty/clucene/src/CLucene/config/PlatformUnix.h b/src/assistant/3rdparty/clucene/src/CLucene/config/PlatformUnix.h new file mode 100644 index 000000000..202a894bd --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/config/PlatformUnix.h @@ -0,0 +1,12 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +// It is internal CLucene header - DO NOT include it directly + +# define PATH_DELIMITER _T("/") +# define PATH_DELIMITERA "/" +# define PATH_DELIMITERC '/' + diff --git a/src/assistant/3rdparty/clucene/src/CLucene/config/PlatformWin32.h b/src/assistant/3rdparty/clucene/src/CLucene/config/PlatformWin32.h new file mode 100644 index 000000000..8b8a1132e --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/config/PlatformWin32.h @@ -0,0 +1,11 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +// It is internal CLucene header - DO NOT include it directly + +# define PATH_DELIMITER _T("\\") +# define PATH_DELIMITERA "\\" +# define PATH_DELIMITERC '\\' diff --git a/src/assistant/3rdparty/clucene/src/CLucene/config/compiler.h b/src/assistant/3rdparty/clucene/src/CLucene/config/compiler.h new file mode 100644 index 000000000..68f93b6e4 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/config/compiler.h @@ -0,0 +1,259 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#if !defined(lucene_compiler_h) +#define lucene_compiler_h + +#if defined(_MBCS) || defined(_ASCII) +#undef _ASCII +#undef _UCS2 +#define _ASCII +#elif defined(_UNICODE) +#define _UCS2 +#elif !defined(_UCS2) +#define _UCS2 +#endif + +//msvc needs unicode define so that it uses unicode library +#ifdef _UCS2 +#undef _UNICODE +#define _UNICODE +#undef _ASCII +#else +#undef _UNICODE +#undef _UCS2 +#endif + + +//////////////////////////////////////////////////////////////////// +// Figure out what compiler we are using +//////////////////////////////////////////////////////////////////// + +#if defined(_MSC_VER) && !defined(__MWERKS__) && !defined (__COMO__) +#define _CLCOMPILER_MSVC _MSC_VER +#endif + +#if defined(__GNUC__) || defined(__SUNPRO_CC) || defined(__xlC__) || defined(__sgi) && defined(__EDG__) +#include "CLucene/config/CompilerGcc.h" + +#elif defined(_CLCOMPILER_MSVC) +/* Microsoft Visual C++ */ +#include "CLucene/config/CompilerMsvc.h" + +#elif defined (__BORLANDC__) +#include "CLucene/config/CompilerBcb.h" + +#elif defined (__HP_aCC) +#include "CLucene/config/CompilerAcc.h" + +#else + //Unable to identify the compiler, issue error diagnostic. + //Edit <CLucene/config/LuceneMycomp.h> to set STLport up for your compiler. + //Uncomment this next line +#error "Unable to identify the compiler, issue error diagnostic. Edit <CLucene/config/CompilerMycomp.h> to set Lucene up for your compiler." +#include "CLucene/config/LuceneMycomp.h" +#endif /* end of compiler choice */ +//////////////////////////////////////////////////////////////////// + + + +//////////////////////////////////////////////////////////////////// +// Now include platform specific definitions +//////////////////////////////////////////////////////////////////// + +/* Operating system recognition (basic) */ +#if defined (__unix) || defined (__linux__) || defined (__QNX__) || defined (_AIX) || defined (__NetBSD__) || defined (__OpenBSD__) || defined (__Lynx__) || defined(hpux) || defined(__hpux) +#undef _UNIX +#define _UNIX 1 +#include "CLucene/config/PlatformUnix.h" + +#elif defined(macintosh) || defined (_MAC) || defined(__APPLE__) +#undef _MAC +#define _MAC 1 +#include "CLucene/config/PlatformMac.h" + +#elif defined (_WIN32) || defined (__WIN32) || defined (WIN32) || defined (__WIN32__) +#undef _WIN32 +#define _WIN32 +#include "CLucene/config/PlatformWin32.h" + +#elif defined (__WIN16) || defined (WIN16) || defined (_WIN16) +#undef _WIN16 +#define _WIN16 +#error "CLucene has not been tested on this platform. Please send a report to the lucene administrators if you are able to successfully compile" +#else +#error "CLucene could not identify the platform." +#endif /* platforms */ + + + +//////////////////////////////////////////////////////////////////// +// Now we take all that we have learnt, and define some things +//////////////////////////////////////////////////////////////////// + +//lets just say that we can always do unicode! :) +#ifdef CL_CHARSET_GUESS +#define _UCS2 +#endif + +#if defined(_ASCII) +#undef _UCS2 +#elif defined(_UCS2) +#undef _ASCII +#endif + +#ifndef _LUCENE_NO_NEW_STYLE_CASTS +#define __CONST_CAST(typ,var) const_cast<typ>(var) +#define __REINTERPRET_CAST(typ,var) reinterpret_cast<typ>(var) +#else +#define __CONST_CAST(typ,var) ((typ)(var)) +#define __REINTERPRET_CAST,var) ((typ)(var)) +#endif + +#ifndef _CL_DEPRECATE_TEXT +#define _CL_DEPRECATE_TEXT(_Text) +#endif +#define _CL_DEPRECATED(_NewItem) _CL_DEPRECATE_TEXT("This function or variable has been superceded by newer library or operating system functionality. Consider using" #_NewItem "instead. See online help for details.") + + +//cnd-debug exit command +#ifndef debugFatalExit +#define debugFatalExit(ret) exit(ret) +#endif + +#ifndef _CL_ILONG +#define _CL_ILONG(x) x ## L +#endif +#ifndef _ILONGLONG +#define _ILONGLONG(x) x ## LL +#endif + +//define whats the values of item intergers *should* be. we can check this in a test +#define LUCENE_INT64_MAX_SHOULDBE _ILONGLONG(0x7FFFFFFFFFFFFFFF) +#define LUCENE_INT32_MAX_SHOULDBE 0x7FFFFFFFL +#define LUCENE_UINT8_MAX_SHOULDBE 0xff + +//maximum path length. only used for buffers that use fullpath. +//anything else should use a dynamic length. +#if defined(CL_MAX_PATH) +//do nothing... +#elif defined(PATH_MAX) +#define CL_MAX_PATH PATH_MAX +#elif defined(MAX_PATH) +#define CL_MAX_PATH MAX_PATH +#elif defined(_MAX_PATH) +#define CL_MAX_PATH _MAX_PATH +#else + #error "CL_MAX_PATH could not be determined" +#endif + +//this is the max filename... for now its just the same, +//but this could change, so we use a different name +#define CL_MAX_NAME CL_MAX_PATH +//this used to be CL_MAX_NAME * 32, but as Alex Hudson points out, this could come to be 128kb. +//the above logic for CL_MAX_NAME should be correct enough to handle all file names +#define CL_MAX_DIR CL_MAX_PATH + +#ifdef _LARGE_FILES +#define LUCENE_MAX_FILELENGTH LUCENE_INT64_MAX_SHOULDBE +#else +#define LUCENE_MAX_FILELENGTH LUCENE_INT32_MAX_SHOULDBE +#endif + +//use the LUCENE_STATIC_CONSTANT_SYNTAX to determine LUCENE_STATIC_CONSTANT +#ifndef LUCENE_STATIC_CONSTANT + //autoconf is not properly detecting the correct method for this, and since there's no real big + //harm in always using an enum, we'll probably just make this the default. + /*#if LUCENE_STATIC_CONSTANT_SYNTAX == 1 + #define LUCENE_STATIC_CONSTANT(type, assignment) static const type assignment + #elif LUCENE_STATIC_CONSTANT_SYNTAX == 2*/ + #define LUCENE_STATIC_CONSTANT(type, assignment) enum { assignment } + /*#else + #error "LUCENE_STATIC_CONSTANT not defined, and/or LUCENE_STATIC_CONSTANT_SYNTAX is not defined to a valid value" + #endif*/ +#endif + +//end of lucene_compiler_h +#elif !defined(lucene_compiler_h2) +#define lucene_compiler_h2 +//here we include the compiler header again, this gives the header a +//second chance at including stuff, after the main inclusions are complete + +#if defined (__GNUC__) || defined(__SUNPRO_CC) || defined(__xlC__) || defined(__sgi) && defined(__EDG__) +#include "CLucene/config/CompilerGcc.h" + +#elif defined(_CLCOMPILER_MSVC) +/* Microsoft Visual C++ */ +#include "CLucene/config/CompilerMsvc.h" + +#elif defined __BORLANDC__ +#include "CLucene/config/CompilerBcb.h" + +#elif defined (__HP_aCC) +#include "CLucene/config/CompilerAcc.h" + +#else +//Unable to identify the compiler, issue error diagnostic. +//Edit <CLucene/config/LuceneMycomp.h> to set STLport up for your compiler. +//Uncomment this next line +#error "Unable to identify the compiler, issue error diagnostic. Edit <CLucene/config/CompilerMycomp.h> to set Lucene up for your compiler." +#include "CLucene/config/LuceneMycomp.h" +#endif /* end of compiler choice */ + +#ifndef _CL_HAVE_FLOAT_T +//#ifdef _CL_HAVE_LONG_DOUBLE +// long double's are not working (reported by Mark Ashworth on Solaris 64) +// typedef long double float_t; /* `float' expressions are evaluated as `long double'. */ +//#else +// we are going to use qreal now +// typedef double float_t; +//#endif +#endif + +/*todo: but need to define SIZEOF_VOID_P #if (SIZEOF_VOID_P > 4 && SIZEOF_VOID_P <= 8) +#ifndef _CL_HAVE_INTPTR_T + typedef int64_t intptr_t; +#endif +#elif (SIZEOF_VOID_P > 2 && SIZEOF_VOID_P <= 4) +# ifndef _CL_HAVE_INTPTR_T + typedef int32_t intptr_t; +# endif +#else +#error "void * is either >8 bytes or <= 2. In either case, I am confused." +#endif*/ + +#ifndef _CL_HAVE_INTPTR_T + typedef int intptr_t; +#endif + +//end of lucene_compiler_h2 +#elif !defined(lucene_compiler_h3) +#define lucene_compiler_h3 +//here we include the compiler header again, this gives the header a +//third chance at including stuff, after the main inclusions are complete + +#if defined (__GNUC__ ) || defined(__SUNPRO_CC) || defined(__xlC__) || defined(__sgi) && defined(__EDG__) +#include "CLucene/config/CompilerGcc.h" + +#elif defined(_CLCOMPILER_MSVC) +/* Microsoft Visual C++ */ +#include "CLucene/config/CompilerMsvc.h" + +#elif defined __BORLANDC__ +#include "CLucene/config/CompilerBcb.h" + +#elif defined (__HP_aCC) +#include "CLucene/config/CompilerAcc.h" + +#else +//Unable to identify the compiler, issue error diagnostic. +//Edit <CLucene/config/LuceneMycomp.h> to set STLport up for your compiler. +//Uncomment this next line +#error "Unable to identify the compiler, issue error diagnostic. Edit <CLucene/config/CompilerMycomp.h> to set Lucene up for your compiler." +#include "CLucene/config/LuceneMycomp.h" +#endif /* end of compiler choice */ + +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/config/define_std.h b/src/assistant/3rdparty/clucene/src/CLucene/config/define_std.h new file mode 100644 index 000000000..22a079053 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/config/define_std.h @@ -0,0 +1,113 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef lucene_define_std +#define lucene_define_std +//define a standard list of defines. +//These defines represents a fairly complete compiler. +//Of course it is preferable to use the autoconf generated +//list, but then not all systems can do this :) + +//we support long files - 64 bit file functions +#define _LARGE_FILES + +//support namespaces +#define _CL_HAVE_NAMESPACES + +//support try/catch blocks +#define _CL_HAVE_FUNCTION_TRY_BLOCKS + +//the normal headers +#define _CL_STDC_HEADERS +#define _CL_HAVE_STDARG_H +#define _CL_HAVE_ALGORITHM +#define _CL_HAVE_FUNCTIONAL +#define _CL_HAVE_MATH_H +#define _CL_HAVE_STL +#define _CL_HAVE_HASH_MAP +#define _CL_HAVE_HASH_SET +#define _CL_HAVE_MAP +#define _CL_HAVE_SET +#define _CL_HAVE_LIST +#define _CL_HAVE_VECTOR +#define _CL_HAVE_STDEXCEPT +#define _CL_HAVE_ERRNO_H +#define _CL_HAVE_SYS_STAT_H +#define _CL_HAVE_FCNTL_H + +//character & std tchar support +#define _CL_HAVE_TCHAR_H +#ifdef _UCS2 + #define _CL_HAVE_WCTYPE_H + + #define _CL_HAVE_WCSCPY + #define _CL_HAVE_WCSNCPY + #define _CL_HAVE_WCSCAT + #define _CL_HAVE_WCSCHR + #define _CL_HAVE_WCSSTR + #define _CL_HAVE_WCSLEN + #define _CL_HAVE_WCSCMP + #define _CL_HAVE_WCSNCMP + #define _CL_HAVE_WCSCSPN +#else + #define _CL_HAVE_CTYPE_H +#endif + +//already have the normal structures +#define _CL_HAVE_FLOAT_T +#define _CL_HAVE_INTPTR_T + +//system dependant: +#define _CL_HAVE_STRING_H //could be HAVE_STRINGS_H && HAVE_STRCHR +#define _CL_HAVE_SYS_TIMEB_H +#define _CL_HAVE_TIME_H + +#if defined (_WIN32) || defined (__WIN32) || defined (WIN32) || defined (__WIN32__) + #define _CL_HAVE_IO_H + #define _CL_HAVE_DIRECT_H + #define _CL_HAVE_WINDOWS_H +#else + #define _CL_HAVE_UNISTD_H +#endif +#ifdef UNDER_CE +#undef _CL_HAVE_SYS_TIMEB_H +#endif + +//////////////////////////////////////////////// +//now for individual functions. some compilers +//miss these, so must individually define what +//we have +//////////////////////////////////////////////// + +//string functions +#define _CL_HAVE_STRLWR +#define _CL_HAVE_WCSLWR +#define _CL_HAVE_WCSCASECMP +#define _CL_HAVE_STRCASECMP + +//formatting functions +#define _CL_HAVE_SNWPRINTF +#define _CL_HAVE_VSNWPRINTF +#define _CL_HAVE_WPRINTF +#define _CL_HAVE_SNPRINTF +#define _CL_HAVE_PRINTF + + +//conversion functions +#define _CL_HAVE_STRTOLL +#define _CL_HAVE_WCSTOLL +#define _CL_HAVE_WCSTOD +#define _CL_HAVE_LLTOA +#define _CL_HAVE_LLTOW +#define _CL_HAVE_INTPTR_T + +//these ones are not standard (msvc) +//so you will probably need to undefine +//if you are not using msvc +#define _CL_HAVE_FILELENGTH + + +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/config/gunichartables.cpp b/src/assistant/3rdparty/clucene/src/CLucene/config/gunichartables.cpp new file mode 100644 index 000000000..5463936f6 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/config/gunichartables.cpp @@ -0,0 +1,386 @@ +/* + * Copyright (C) 1999 Tom Tromey + * Copyright (C) 2000 Red Hat, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + * + ************************************************ + * Also licensed with permission from Tom Tromey + * and Owen Taylor under the Apache license. + * Original location: + * http://cvs.gnome.org/viewcvs/glib/glib/guniprop.c?view=log + ************************************************ + * + * Copyright 2003-2006 The Apache Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Changes are Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies). +*/ + +#include "CLucene/StdHeader.h" + +typedef unsigned long gunichar; +typedef unsigned short guint16; +typedef short gint16; +typedef char gchar; +typedef unsigned char guchar; + +/* These are the possible character classifications. + * See http://www.unicode.org/Public/UNIDATA/UnicodeData.txt + or http://www.unicode.org/Public/UNIDATA/UCD.html. + + todo: i think there is a new version of the unicode, which we should use. + data is licensed like this: http://www.unicode.org/copyright.html... not sure but looks apache compatible + */ +typedef enum +{ + G_UNICODE_CONTROL, + G_UNICODE_FORMAT, + G_UNICODE_UNASSIGNED, + G_UNICODE_PRIVATE_USE, + G_UNICODE_SURROGATE, + G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_MODIFIER_LETTER, + G_UNICODE_OTHER_LETTER, + G_UNICODE_TITLECASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_COMBINING_MARK, + G_UNICODE_ENCLOSING_MARK, + G_UNICODE_NON_SPACING_MARK, + G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_LETTER_NUMBER, + G_UNICODE_OTHER_NUMBER, + G_UNICODE_CONNECT_PUNCTUATION, + G_UNICODE_DASH_PUNCTUATION, + G_UNICODE_CLOSE_PUNCTUATION, + G_UNICODE_FINAL_PUNCTUATION, + G_UNICODE_INITIAL_PUNCTUATION, + G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_OPEN_PUNCTUATION, + G_UNICODE_CURRENCY_SYMBOL, + G_UNICODE_MODIFIER_SYMBOL, + G_UNICODE_MATH_SYMBOL, + G_UNICODE_OTHER_SYMBOL, + G_UNICODE_LINE_SEPARATOR, + G_UNICODE_PARAGRAPH_SEPARATOR, + G_UNICODE_SPACE_SEPARATOR +} GUnicodeType; + + +#include "gunichartables.h" + +#define ATTR_TABLE(Page) (((Page) <= G_UNICODE_LAST_PAGE_PART1) \ + ? attr_table_part1[Page] \ + : attr_table_part2[(Page) - 0xe00]) + +#define ATTTABLE(Page, Char) \ + ((ATTR_TABLE(Page) == G_UNICODE_MAX_TABLE_INDEX) ? 0 : (attr_data[ATTR_TABLE(Page)][Char])) + + +#define TTYPE_PART1(Page, Char) \ + ((type_table_part1[Page] >= G_UNICODE_MAX_TABLE_INDEX) \ + ? (type_table_part1[Page] - G_UNICODE_MAX_TABLE_INDEX) \ + : (type_data[type_table_part1[Page]][Char])) + +#define TTYPE_PART2(Page, Char) \ + ((type_table_part2[Page] >= G_UNICODE_MAX_TABLE_INDEX) \ + ? (type_table_part2[Page] - G_UNICODE_MAX_TABLE_INDEX) \ + : (type_data[type_table_part2[Page]][Char])) + +#define TYPE(Char) \ + (((Char) <= G_UNICODE_LAST_CHAR_PART1) \ + ? TTYPE_PART1 ((Char) >> 8, (Char) & 0xff) \ + : (((Char) >= 0xe0000 && (Char) <= G_UNICODE_LAST_CHAR) \ + ? TTYPE_PART2 (((Char) - 0xe0000) >> 8, (Char) & 0xff) \ + : G_UNICODE_UNASSIGNED)) + +/* Count the number of elements in an array. The array must be defined + * as such; using this with a dynamically allocated array will give + * incorrect results. + */ +#define G_N_ELEMENTS(arr) (sizeof (arr) / sizeof ((arr)[0])) + + + + +#if defined(LUCENE_USE_INTERNAL_CHAR_FUNCTIONS) +#ifdef _LUCENE_PRAGMA_WARNINGS + #pragma message ("===== Using internal character function =====") +#else +#if !(defined(Q_OS_SOLARIS) || defined(Q_CC_MIPS)) +#warning "===== Using internal character function =====" +#endif +#endif + +bool cl_isletter(gunichar c) +{ + int t = TYPE (c); + switch(t) + { + case G_UNICODE_LOWERCASE_LETTER: return true; + case G_UNICODE_TITLECASE_LETTER: return true; + case G_UNICODE_UPPERCASE_LETTER: return true; + case G_UNICODE_MODIFIER_LETTER: return true; + case G_UNICODE_OTHER_LETTER: return true; + default: return false; + } +} + +bool cl_isalnum(gunichar c) +{ + int t = TYPE (c); + switch(t) + { + case G_UNICODE_LOWERCASE_LETTER: return true; + case G_UNICODE_TITLECASE_LETTER: return true; + case G_UNICODE_UPPERCASE_LETTER: return true; + case G_UNICODE_MODIFIER_LETTER: return true; + case G_UNICODE_OTHER_LETTER: return true; + case G_UNICODE_DECIMAL_NUMBER: return true; + case G_UNICODE_LETTER_NUMBER: return true; + case G_UNICODE_OTHER_NUMBER: return true; + default: return false; + } +} + +bool cl_isdigit(gunichar c) +{ + int t = TYPE (c); + switch(t) + { + case G_UNICODE_DECIMAL_NUMBER: return true; + case G_UNICODE_LETTER_NUMBER: return true; + case G_UNICODE_OTHER_NUMBER: return true; + default: return false; + } +} + +/** + * cl_isspace: + * @c: a Unicode character + * + * Determines whether a character is a space, tab, or line separator + * (newline, carriage return, etc.). Given some UTF-8 text, obtain a + * character value with lucene_utf8towc(). + * + * (Note: don't use this to do word breaking; you have to use + * Pango or equivalent to get word breaking right, the algorithm + * is fairly complex.) + * + * Return value: %TRUE if @c is a punctuation character + **/ +bool cl_isspace (gunichar c) +{ + switch (c) + { + /* special-case these since Unicode thinks they are not spaces */ + case '\t': + case '\n': + case '\r': + case '\f': + return true; + + default: + { + int t = TYPE ((gunichar)c); + return (t == G_UNICODE_SPACE_SEPARATOR || t == G_UNICODE_LINE_SEPARATOR + || t == G_UNICODE_PARAGRAPH_SEPARATOR); + } + } +} + + + +/** + * cl_tolower: + * @c: a Unicode character. + * + * Converts a character to lower case. + * + * Return value: the result of converting @c to lower case. + * If @c is not an upperlower or titlecase character, + * or has no lowercase equivalent @c is returned unchanged. + **/ +TCHAR cl_tolower (TCHAR ch) +{ + gunichar c=ch; + int t = TYPE ((gunichar)c); + if (t == G_UNICODE_UPPERCASE_LETTER) + { + gunichar val = ATTTABLE (c >> 8, c & 0xff); + if (val >= 0x1000000) + { + const gchar *p = special_case_table + val - 0x1000000; + int len=0; + wchar_t ret=0; + lucene_utf8towc(&ret,p,6); +#ifdef _UCS2 + return ret; +#else + return LUCENE_OOR_CHAR(ret); +#endif + //return cl_utf8_get_char (p, &len); + }else + return val ? val : c; + }else if (t == G_UNICODE_TITLECASE_LETTER){ + unsigned int i; + for (i = 0; i < G_N_ELEMENTS (title_table); ++i) + { + if (title_table[i][0] == c) + return title_table[i][2]; + } + } + return c; +} + +/** + * cl_toupper: + * @c: a Unicode character + * + * Converts a character to uppercase. + * + * Return value: the result of converting @c to uppercase. + * If @c is not an lowercase or titlecase character, + * or has no upper case equivalent @c is returned unchanged. + **/ +TCHAR cl_toupper (TCHAR ch) +{ + gunichar c=ch; + int t = TYPE (c); + if (t == G_UNICODE_LOWERCASE_LETTER) + { + gunichar val = ATTTABLE (c >> 8, c & 0xff); + if (val >= 0x1000000) + { + const gchar *p = special_case_table + val - 0x1000000; + + wchar_t ret=0; + lucene_utf8towc(&ret,p,6); +#ifdef _UCS2 + return ret; +#else + return LUCENE_OOR_CHAR(ret); +#endif + //return lucene_utf8towc (p); + } + else + return val ? val : c; + } + else if (t == G_UNICODE_TITLECASE_LETTER) + { + unsigned int i; + for (i = 0; i < G_N_ELEMENTS (title_table); ++i) + { + if (title_table[i][0] == c) + return title_table[i][1]; + } + } + return c; +} + + + +/** + * cl_tcasefold: + * @str: a unicode string + * + * Converts a string into a form that is independent of case. The + * result will not correspond to any particular case, but can be + * compared for equality or ordered with the results of calling + * cl_tcasefold() on other strings. + * + * Note that calling cl_tcasefold() followed by g_utf8_collate() is + * only an approximation to the correct linguistic case insensitive + * ordering, though it is a fairly good one. Getting this exactly + * right would require a more sophisticated collation function that + * takes case sensitivity into account. GLib does not currently + * provide such a function. + * + * Return value: a newly allocated string, that is a + * case independent form of @str. + **/ +TCHAR cl_tcasefold(const TCHAR ch){ + int start = 0; + int end = G_N_ELEMENTS (casefold_table); + + if (ch >= casefold_table[start].ch && + ch <= casefold_table[end - 1].ch) + { + while (1) + { + int half = (start + end) / 2; + if (ch == casefold_table[half].ch) + { + wchar_t ret=0; + lucene_utf8towc(&ret,casefold_table[half].data,6); + + #ifdef _UCS2 + return ret; + #else + LUCENE_OOR_CHAR(ret) + #endif + }else if (half == start){ + break; + }else if (ch > casefold_table[half].ch){ + start = half; + }else{ + end = half; + } + } + } + return cl_tolower(ch); + +} + + +//this function was not taken from gnome +TCHAR* cl_tcscasefold( TCHAR * str, int len ) //len default is -1 +{ + TCHAR *p = str; + while ((len < 0 || p < str + len) && *p) + { + *p = cl_tcasefold(*p); + p++; + } + return str; +} +//this function was not taken from gnome +int cl_tcscasefoldcmp(const TCHAR * dst, const TCHAR * src){ + TCHAR f,l; + + do{ + f = cl_tcasefold( (*(dst++)) ); + l = cl_tcasefold( (*(src++)) ); + } while ( (f) && (f == l) ); + + return (int)(f - l); +} + +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/config/gunichartables.h b/src/assistant/3rdparty/clucene/src/CLucene/config/gunichartables.h new file mode 100644 index 000000000..182a87054 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/config/gunichartables.h @@ -0,0 +1,11264 @@ +/* This file is automatically generated. DO NOT EDIT! + Instead, edit gen-unicode-tables.pl and re-run. */ + +#ifndef CHARTABLES_H +#define CHARTABLES_H + +#define G_UNICODE_DATA_VERSION "4.0" + +#define G_UNICODE_LAST_CHAR 0x10ffff + +#define G_UNICODE_MAX_TABLE_INDEX 10000 + +#define G_UNICODE_LAST_CHAR_PART1 0x2FAFF + +#define G_UNICODE_LAST_PAGE_PART1 762 + +static const char type_data[][256] = { + { /* page 0, index 0 */ + G_UNICODE_CONTROL, G_UNICODE_CONTROL, G_UNICODE_CONTROL, + G_UNICODE_CONTROL, G_UNICODE_CONTROL, G_UNICODE_CONTROL, + G_UNICODE_CONTROL, G_UNICODE_CONTROL, G_UNICODE_CONTROL, + G_UNICODE_CONTROL, G_UNICODE_CONTROL, G_UNICODE_CONTROL, + G_UNICODE_CONTROL, G_UNICODE_CONTROL, G_UNICODE_CONTROL, + G_UNICODE_CONTROL, G_UNICODE_CONTROL, G_UNICODE_CONTROL, + G_UNICODE_CONTROL, G_UNICODE_CONTROL, G_UNICODE_CONTROL, + G_UNICODE_CONTROL, G_UNICODE_CONTROL, G_UNICODE_CONTROL, + G_UNICODE_CONTROL, G_UNICODE_CONTROL, G_UNICODE_CONTROL, + G_UNICODE_CONTROL, G_UNICODE_CONTROL, G_UNICODE_CONTROL, + G_UNICODE_CONTROL, G_UNICODE_CONTROL, G_UNICODE_SPACE_SEPARATOR, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_CURRENCY_SYMBOL, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_OPEN_PUNCTUATION, + G_UNICODE_CLOSE_PUNCTUATION, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_MATH_SYMBOL, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_DASH_PUNCTUATION, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_OPEN_PUNCTUATION, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_CLOSE_PUNCTUATION, G_UNICODE_MODIFIER_SYMBOL, + G_UNICODE_CONNECT_PUNCTUATION, G_UNICODE_MODIFIER_SYMBOL, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_OPEN_PUNCTUATION, G_UNICODE_MATH_SYMBOL, + G_UNICODE_CLOSE_PUNCTUATION, G_UNICODE_MATH_SYMBOL, G_UNICODE_CONTROL, + G_UNICODE_CONTROL, G_UNICODE_CONTROL, G_UNICODE_CONTROL, + G_UNICODE_CONTROL, G_UNICODE_CONTROL, G_UNICODE_CONTROL, + G_UNICODE_CONTROL, G_UNICODE_CONTROL, G_UNICODE_CONTROL, + G_UNICODE_CONTROL, G_UNICODE_CONTROL, G_UNICODE_CONTROL, + G_UNICODE_CONTROL, G_UNICODE_CONTROL, G_UNICODE_CONTROL, + G_UNICODE_CONTROL, G_UNICODE_CONTROL, G_UNICODE_CONTROL, + G_UNICODE_CONTROL, G_UNICODE_CONTROL, G_UNICODE_CONTROL, + G_UNICODE_CONTROL, G_UNICODE_CONTROL, G_UNICODE_CONTROL, + G_UNICODE_CONTROL, G_UNICODE_CONTROL, G_UNICODE_CONTROL, + G_UNICODE_CONTROL, G_UNICODE_CONTROL, G_UNICODE_CONTROL, + G_UNICODE_CONTROL, G_UNICODE_CONTROL, G_UNICODE_SPACE_SEPARATOR, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_CURRENCY_SYMBOL, + G_UNICODE_CURRENCY_SYMBOL, G_UNICODE_CURRENCY_SYMBOL, + G_UNICODE_CURRENCY_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_MODIFIER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_INITIAL_PUNCTUATION, G_UNICODE_MATH_SYMBOL, G_UNICODE_FORMAT, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_MODIFIER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_MODIFIER_SYMBOL, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_MODIFIER_SYMBOL, + G_UNICODE_OTHER_NUMBER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_FINAL_PUNCTUATION, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_MATH_SYMBOL, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_MATH_SYMBOL, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER + }, + { /* page 1, index 1 */ + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_TITLECASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_TITLECASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_TITLECASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_TITLECASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER + }, + { /* page 2, index 2 */ + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_MODIFIER_LETTER, + G_UNICODE_MODIFIER_LETTER, G_UNICODE_MODIFIER_LETTER, + G_UNICODE_MODIFIER_LETTER, G_UNICODE_MODIFIER_LETTER, + G_UNICODE_MODIFIER_LETTER, G_UNICODE_MODIFIER_LETTER, + G_UNICODE_MODIFIER_LETTER, G_UNICODE_MODIFIER_LETTER, + G_UNICODE_MODIFIER_LETTER, G_UNICODE_MODIFIER_LETTER, + G_UNICODE_MODIFIER_LETTER, G_UNICODE_MODIFIER_LETTER, + G_UNICODE_MODIFIER_LETTER, G_UNICODE_MODIFIER_LETTER, + G_UNICODE_MODIFIER_LETTER, G_UNICODE_MODIFIER_LETTER, + G_UNICODE_MODIFIER_LETTER, G_UNICODE_MODIFIER_SYMBOL, + G_UNICODE_MODIFIER_SYMBOL, G_UNICODE_MODIFIER_SYMBOL, + G_UNICODE_MODIFIER_SYMBOL, G_UNICODE_MODIFIER_LETTER, + G_UNICODE_MODIFIER_LETTER, G_UNICODE_MODIFIER_LETTER, + G_UNICODE_MODIFIER_LETTER, G_UNICODE_MODIFIER_LETTER, + G_UNICODE_MODIFIER_LETTER, G_UNICODE_MODIFIER_LETTER, + G_UNICODE_MODIFIER_LETTER, G_UNICODE_MODIFIER_LETTER, + G_UNICODE_MODIFIER_LETTER, G_UNICODE_MODIFIER_LETTER, + G_UNICODE_MODIFIER_LETTER, G_UNICODE_MODIFIER_SYMBOL, + G_UNICODE_MODIFIER_SYMBOL, G_UNICODE_MODIFIER_SYMBOL, + G_UNICODE_MODIFIER_SYMBOL, G_UNICODE_MODIFIER_SYMBOL, + G_UNICODE_MODIFIER_SYMBOL, G_UNICODE_MODIFIER_SYMBOL, + G_UNICODE_MODIFIER_SYMBOL, G_UNICODE_MODIFIER_SYMBOL, + G_UNICODE_MODIFIER_SYMBOL, G_UNICODE_MODIFIER_SYMBOL, + G_UNICODE_MODIFIER_SYMBOL, G_UNICODE_MODIFIER_SYMBOL, + G_UNICODE_MODIFIER_SYMBOL, G_UNICODE_MODIFIER_LETTER, + G_UNICODE_MODIFIER_LETTER, G_UNICODE_MODIFIER_LETTER, + G_UNICODE_MODIFIER_LETTER, G_UNICODE_MODIFIER_LETTER, + G_UNICODE_MODIFIER_SYMBOL, G_UNICODE_MODIFIER_SYMBOL, + G_UNICODE_MODIFIER_SYMBOL, G_UNICODE_MODIFIER_SYMBOL, + G_UNICODE_MODIFIER_SYMBOL, G_UNICODE_MODIFIER_SYMBOL, + G_UNICODE_MODIFIER_SYMBOL, G_UNICODE_MODIFIER_SYMBOL, + G_UNICODE_MODIFIER_SYMBOL, G_UNICODE_MODIFIER_LETTER, + G_UNICODE_MODIFIER_SYMBOL, G_UNICODE_MODIFIER_SYMBOL, + G_UNICODE_MODIFIER_SYMBOL, G_UNICODE_MODIFIER_SYMBOL, + G_UNICODE_MODIFIER_SYMBOL, G_UNICODE_MODIFIER_SYMBOL, + G_UNICODE_MODIFIER_SYMBOL, G_UNICODE_MODIFIER_SYMBOL, + G_UNICODE_MODIFIER_SYMBOL, G_UNICODE_MODIFIER_SYMBOL, + G_UNICODE_MODIFIER_SYMBOL, G_UNICODE_MODIFIER_SYMBOL, + G_UNICODE_MODIFIER_SYMBOL, G_UNICODE_MODIFIER_SYMBOL, + G_UNICODE_MODIFIER_SYMBOL, G_UNICODE_MODIFIER_SYMBOL, + G_UNICODE_MODIFIER_SYMBOL + }, + { /* page 3, index 3 */ + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_MODIFIER_SYMBOL, + G_UNICODE_MODIFIER_SYMBOL, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_MODIFIER_LETTER, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_MODIFIER_SYMBOL, G_UNICODE_MODIFIER_SYMBOL, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UNASSIGNED, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UNASSIGNED, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UNASSIGNED, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UNASSIGNED, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_MATH_SYMBOL, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED + }, + { /* page 4, index 4 */ + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_UNASSIGNED, + G_UNICODE_ENCLOSING_MARK, G_UNICODE_ENCLOSING_MARK, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UNASSIGNED, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED + }, + { /* page 5, index 5 */ + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_MODIFIER_LETTER, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_UNASSIGNED, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_DASH_PUNCTUATION, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_UNASSIGNED, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_UNASSIGNED, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED + }, + { /* page 6, index 6 */ + G_UNICODE_FORMAT, G_UNICODE_FORMAT, G_UNICODE_FORMAT, G_UNICODE_FORMAT, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_UNASSIGNED, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_MODIFIER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_OTHER_LETTER, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_FORMAT, G_UNICODE_ENCLOSING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_MODIFIER_LETTER, G_UNICODE_MODIFIER_LETTER, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_LETTER + }, + { /* page 7, index 7 */ + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_UNASSIGNED, G_UNICODE_FORMAT, G_UNICODE_OTHER_LETTER, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED + }, + { /* page 9, index 8 */ + G_UNICODE_UNASSIGNED, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_COMBINING_MARK, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_OTHER_LETTER, G_UNICODE_COMBINING_MARK, + G_UNICODE_COMBINING_MARK, G_UNICODE_COMBINING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_COMBINING_MARK, G_UNICODE_COMBINING_MARK, + G_UNICODE_COMBINING_MARK, G_UNICODE_COMBINING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_OTHER_LETTER, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_COMBINING_MARK, G_UNICODE_COMBINING_MARK, G_UNICODE_UNASSIGNED, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, + G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_NON_SPACING_MARK, G_UNICODE_OTHER_LETTER, + G_UNICODE_COMBINING_MARK, G_UNICODE_COMBINING_MARK, + G_UNICODE_COMBINING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_COMBINING_MARK, G_UNICODE_COMBINING_MARK, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_COMBINING_MARK, G_UNICODE_COMBINING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_COMBINING_MARK, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_CURRENCY_SYMBOL, G_UNICODE_CURRENCY_SYMBOL, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED + }, + { /* page 10, index 9 */ + G_UNICODE_UNASSIGNED, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_COMBINING_MARK, + G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_UNASSIGNED, G_UNICODE_COMBINING_MARK, G_UNICODE_COMBINING_MARK, + G_UNICODE_COMBINING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_COMBINING_MARK, G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_OTHER_LETTER, + G_UNICODE_COMBINING_MARK, G_UNICODE_COMBINING_MARK, + G_UNICODE_COMBINING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_UNASSIGNED, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_COMBINING_MARK, + G_UNICODE_UNASSIGNED, G_UNICODE_COMBINING_MARK, G_UNICODE_COMBINING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_UNASSIGNED, + G_UNICODE_CURRENCY_SYMBOL, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED + }, + { /* page 11, index 10 */ + G_UNICODE_UNASSIGNED, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_COMBINING_MARK, G_UNICODE_COMBINING_MARK, G_UNICODE_UNASSIGNED, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_NON_SPACING_MARK, G_UNICODE_OTHER_LETTER, + G_UNICODE_COMBINING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_COMBINING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_COMBINING_MARK, G_UNICODE_COMBINING_MARK, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_COMBINING_MARK, G_UNICODE_COMBINING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_COMBINING_MARK, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_LETTER, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_NON_SPACING_MARK, G_UNICODE_OTHER_LETTER, + G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, + G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_COMBINING_MARK, G_UNICODE_COMBINING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_COMBINING_MARK, + G_UNICODE_COMBINING_MARK, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_COMBINING_MARK, G_UNICODE_COMBINING_MARK, + G_UNICODE_COMBINING_MARK, G_UNICODE_UNASSIGNED, G_UNICODE_COMBINING_MARK, + G_UNICODE_COMBINING_MARK, G_UNICODE_COMBINING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_COMBINING_MARK, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_CURRENCY_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED + }, + { /* page 12, index 11 */ + G_UNICODE_UNASSIGNED, G_UNICODE_COMBINING_MARK, G_UNICODE_COMBINING_MARK, + G_UNICODE_COMBINING_MARK, G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_COMBINING_MARK, G_UNICODE_COMBINING_MARK, + G_UNICODE_COMBINING_MARK, G_UNICODE_COMBINING_MARK, G_UNICODE_UNASSIGNED, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_UNASSIGNED, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_COMBINING_MARK, + G_UNICODE_COMBINING_MARK, G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_OTHER_LETTER, + G_UNICODE_COMBINING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_COMBINING_MARK, G_UNICODE_COMBINING_MARK, + G_UNICODE_COMBINING_MARK, G_UNICODE_COMBINING_MARK, + G_UNICODE_COMBINING_MARK, G_UNICODE_UNASSIGNED, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_COMBINING_MARK, + G_UNICODE_COMBINING_MARK, G_UNICODE_UNASSIGNED, G_UNICODE_COMBINING_MARK, + G_UNICODE_COMBINING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_COMBINING_MARK, + G_UNICODE_COMBINING_MARK, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, + G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED + }, + { /* page 13, index 12 */ + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_COMBINING_MARK, + G_UNICODE_COMBINING_MARK, G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_COMBINING_MARK, + G_UNICODE_COMBINING_MARK, G_UNICODE_COMBINING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_COMBINING_MARK, G_UNICODE_COMBINING_MARK, + G_UNICODE_COMBINING_MARK, G_UNICODE_UNASSIGNED, G_UNICODE_COMBINING_MARK, + G_UNICODE_COMBINING_MARK, G_UNICODE_COMBINING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_COMBINING_MARK, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_COMBINING_MARK, + G_UNICODE_COMBINING_MARK, G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_COMBINING_MARK, G_UNICODE_COMBINING_MARK, + G_UNICODE_COMBINING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_UNASSIGNED, G_UNICODE_NON_SPACING_MARK, G_UNICODE_UNASSIGNED, + G_UNICODE_COMBINING_MARK, G_UNICODE_COMBINING_MARK, + G_UNICODE_COMBINING_MARK, G_UNICODE_COMBINING_MARK, + G_UNICODE_COMBINING_MARK, G_UNICODE_COMBINING_MARK, + G_UNICODE_COMBINING_MARK, G_UNICODE_COMBINING_MARK, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_COMBINING_MARK, + G_UNICODE_COMBINING_MARK, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED + }, + { /* page 14, index 13 */ + G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_CURRENCY_SYMBOL, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_MODIFIER_LETTER, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, + G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_UNASSIGNED, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_UNASSIGNED, G_UNICODE_MODIFIER_LETTER, G_UNICODE_UNASSIGNED, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED + }, + { /* page 15, index 14 */ + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_OPEN_PUNCTUATION, + G_UNICODE_CLOSE_PUNCTUATION, G_UNICODE_OPEN_PUNCTUATION, + G_UNICODE_CLOSE_PUNCTUATION, G_UNICODE_COMBINING_MARK, + G_UNICODE_COMBINING_MARK, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_COMBINING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_UNASSIGNED, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_SYMBOL, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED + }, + { /* page 16, index 15 */ + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, G_UNICODE_COMBINING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_COMBINING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_COMBINING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_COMBINING_MARK, G_UNICODE_COMBINING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED + }, + { /* page 17, index 16 */ + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED + }, + { /* page 18, index 17 */ + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, + G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, + G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, + G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER + }, + { /* page 19, index 18 */ + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED + }, + { /* page 20, index 19 */ + G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER + }, + { /* page 22, index 20 */ + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_SPACE_SEPARATOR, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OPEN_PUNCTUATION, + G_UNICODE_CLOSE_PUNCTUATION, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_LETTER_NUMBER, G_UNICODE_LETTER_NUMBER, + G_UNICODE_LETTER_NUMBER, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED + }, + { /* page 23, index 21 */ + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_UNASSIGNED, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_FORMAT, + G_UNICODE_FORMAT, G_UNICODE_COMBINING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_COMBINING_MARK, G_UNICODE_COMBINING_MARK, + G_UNICODE_COMBINING_MARK, G_UNICODE_COMBINING_MARK, + G_UNICODE_COMBINING_MARK, G_UNICODE_COMBINING_MARK, + G_UNICODE_COMBINING_MARK, G_UNICODE_COMBINING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_COMBINING_MARK, + G_UNICODE_COMBINING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_MODIFIER_LETTER, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_CURRENCY_SYMBOL, + G_UNICODE_OTHER_LETTER, G_UNICODE_NON_SPACING_MARK, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED + }, + { /* page 24, index 22 */ + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_DASH_PUNCTUATION, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_SPACE_SEPARATOR, G_UNICODE_UNASSIGNED, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_MODIFIER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED + }, + { /* page 25, index 23 */ + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_COMBINING_MARK, G_UNICODE_COMBINING_MARK, + G_UNICODE_COMBINING_MARK, G_UNICODE_COMBINING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_COMBINING_MARK, G_UNICODE_COMBINING_MARK, + G_UNICODE_COMBINING_MARK, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_COMBINING_MARK, + G_UNICODE_COMBINING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_COMBINING_MARK, G_UNICODE_COMBINING_MARK, + G_UNICODE_COMBINING_MARK, G_UNICODE_COMBINING_MARK, + G_UNICODE_COMBINING_MARK, G_UNICODE_COMBINING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL + }, + { /* page 29, index 24 */ + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_MODIFIER_LETTER, G_UNICODE_MODIFIER_LETTER, + G_UNICODE_MODIFIER_LETTER, G_UNICODE_MODIFIER_LETTER, + G_UNICODE_MODIFIER_LETTER, G_UNICODE_MODIFIER_LETTER, + G_UNICODE_MODIFIER_LETTER, G_UNICODE_MODIFIER_LETTER, + G_UNICODE_MODIFIER_LETTER, G_UNICODE_MODIFIER_LETTER, + G_UNICODE_MODIFIER_LETTER, G_UNICODE_MODIFIER_LETTER, + G_UNICODE_MODIFIER_LETTER, G_UNICODE_MODIFIER_LETTER, + G_UNICODE_MODIFIER_LETTER, G_UNICODE_MODIFIER_LETTER, + G_UNICODE_MODIFIER_LETTER, G_UNICODE_MODIFIER_LETTER, + G_UNICODE_MODIFIER_LETTER, G_UNICODE_MODIFIER_LETTER, + G_UNICODE_MODIFIER_LETTER, G_UNICODE_MODIFIER_LETTER, + G_UNICODE_MODIFIER_LETTER, G_UNICODE_MODIFIER_LETTER, + G_UNICODE_MODIFIER_LETTER, G_UNICODE_MODIFIER_LETTER, + G_UNICODE_MODIFIER_LETTER, G_UNICODE_MODIFIER_LETTER, + G_UNICODE_MODIFIER_LETTER, G_UNICODE_MODIFIER_LETTER, + G_UNICODE_MODIFIER_LETTER, G_UNICODE_MODIFIER_LETTER, + G_UNICODE_MODIFIER_LETTER, G_UNICODE_MODIFIER_LETTER, + G_UNICODE_MODIFIER_LETTER, G_UNICODE_MODIFIER_LETTER, + G_UNICODE_MODIFIER_LETTER, G_UNICODE_MODIFIER_LETTER, + G_UNICODE_MODIFIER_LETTER, G_UNICODE_MODIFIER_LETTER, + G_UNICODE_MODIFIER_LETTER, G_UNICODE_MODIFIER_LETTER, + G_UNICODE_MODIFIER_LETTER, G_UNICODE_MODIFIER_LETTER, + G_UNICODE_MODIFIER_LETTER, G_UNICODE_MODIFIER_LETTER, + G_UNICODE_MODIFIER_LETTER, G_UNICODE_MODIFIER_LETTER, + G_UNICODE_MODIFIER_LETTER, G_UNICODE_MODIFIER_LETTER, + G_UNICODE_MODIFIER_LETTER, G_UNICODE_MODIFIER_LETTER, + G_UNICODE_MODIFIER_LETTER, G_UNICODE_MODIFIER_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED + }, + { /* page 30, index 25 */ + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED + }, + { /* page 31, index 26 */ + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UNASSIGNED, G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UNASSIGNED, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UNASSIGNED, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UNASSIGNED, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_TITLECASE_LETTER, G_UNICODE_TITLECASE_LETTER, + G_UNICODE_TITLECASE_LETTER, G_UNICODE_TITLECASE_LETTER, + G_UNICODE_TITLECASE_LETTER, G_UNICODE_TITLECASE_LETTER, + G_UNICODE_TITLECASE_LETTER, G_UNICODE_TITLECASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_TITLECASE_LETTER, G_UNICODE_TITLECASE_LETTER, + G_UNICODE_TITLECASE_LETTER, G_UNICODE_TITLECASE_LETTER, + G_UNICODE_TITLECASE_LETTER, G_UNICODE_TITLECASE_LETTER, + G_UNICODE_TITLECASE_LETTER, G_UNICODE_TITLECASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_TITLECASE_LETTER, G_UNICODE_TITLECASE_LETTER, + G_UNICODE_TITLECASE_LETTER, G_UNICODE_TITLECASE_LETTER, + G_UNICODE_TITLECASE_LETTER, G_UNICODE_TITLECASE_LETTER, + G_UNICODE_TITLECASE_LETTER, G_UNICODE_TITLECASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UNASSIGNED, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_TITLECASE_LETTER, G_UNICODE_MODIFIER_SYMBOL, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_MODIFIER_SYMBOL, + G_UNICODE_MODIFIER_SYMBOL, G_UNICODE_MODIFIER_SYMBOL, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UNASSIGNED, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_TITLECASE_LETTER, G_UNICODE_MODIFIER_SYMBOL, + G_UNICODE_MODIFIER_SYMBOL, G_UNICODE_MODIFIER_SYMBOL, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UNASSIGNED, + G_UNICODE_MODIFIER_SYMBOL, G_UNICODE_MODIFIER_SYMBOL, + G_UNICODE_MODIFIER_SYMBOL, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_MODIFIER_SYMBOL, G_UNICODE_MODIFIER_SYMBOL, + G_UNICODE_MODIFIER_SYMBOL, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UNASSIGNED, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_TITLECASE_LETTER, G_UNICODE_MODIFIER_SYMBOL, + G_UNICODE_MODIFIER_SYMBOL, G_UNICODE_UNASSIGNED + }, + { /* page 32, index 27 */ + G_UNICODE_SPACE_SEPARATOR, G_UNICODE_SPACE_SEPARATOR, + G_UNICODE_SPACE_SEPARATOR, G_UNICODE_SPACE_SEPARATOR, + G_UNICODE_SPACE_SEPARATOR, G_UNICODE_SPACE_SEPARATOR, + G_UNICODE_SPACE_SEPARATOR, G_UNICODE_SPACE_SEPARATOR, + G_UNICODE_SPACE_SEPARATOR, G_UNICODE_SPACE_SEPARATOR, + G_UNICODE_SPACE_SEPARATOR, G_UNICODE_SPACE_SEPARATOR, G_UNICODE_FORMAT, + G_UNICODE_FORMAT, G_UNICODE_FORMAT, G_UNICODE_FORMAT, + G_UNICODE_DASH_PUNCTUATION, G_UNICODE_DASH_PUNCTUATION, + G_UNICODE_DASH_PUNCTUATION, G_UNICODE_DASH_PUNCTUATION, + G_UNICODE_DASH_PUNCTUATION, G_UNICODE_DASH_PUNCTUATION, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_INITIAL_PUNCTUATION, G_UNICODE_FINAL_PUNCTUATION, + G_UNICODE_OPEN_PUNCTUATION, G_UNICODE_INITIAL_PUNCTUATION, + G_UNICODE_INITIAL_PUNCTUATION, G_UNICODE_FINAL_PUNCTUATION, + G_UNICODE_OPEN_PUNCTUATION, G_UNICODE_INITIAL_PUNCTUATION, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_LINE_SEPARATOR, G_UNICODE_PARAGRAPH_SEPARATOR, + G_UNICODE_FORMAT, G_UNICODE_FORMAT, G_UNICODE_FORMAT, G_UNICODE_FORMAT, + G_UNICODE_FORMAT, G_UNICODE_SPACE_SEPARATOR, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_INITIAL_PUNCTUATION, G_UNICODE_FINAL_PUNCTUATION, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_CONNECT_PUNCTUATION, G_UNICODE_CONNECT_PUNCTUATION, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_MATH_SYMBOL, + G_UNICODE_OPEN_PUNCTUATION, G_UNICODE_CLOSE_PUNCTUATION, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_MATH_SYMBOL, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_CONNECT_PUNCTUATION, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_SPACE_SEPARATOR, G_UNICODE_FORMAT, + G_UNICODE_FORMAT, G_UNICODE_FORMAT, G_UNICODE_FORMAT, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_FORMAT, G_UNICODE_FORMAT, G_UNICODE_FORMAT, G_UNICODE_FORMAT, + G_UNICODE_FORMAT, G_UNICODE_FORMAT, G_UNICODE_OTHER_NUMBER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_OPEN_PUNCTUATION, G_UNICODE_CLOSE_PUNCTUATION, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_OPEN_PUNCTUATION, G_UNICODE_CLOSE_PUNCTUATION, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_CURRENCY_SYMBOL, + G_UNICODE_CURRENCY_SYMBOL, G_UNICODE_CURRENCY_SYMBOL, + G_UNICODE_CURRENCY_SYMBOL, G_UNICODE_CURRENCY_SYMBOL, + G_UNICODE_CURRENCY_SYMBOL, G_UNICODE_CURRENCY_SYMBOL, + G_UNICODE_CURRENCY_SYMBOL, G_UNICODE_CURRENCY_SYMBOL, + G_UNICODE_CURRENCY_SYMBOL, G_UNICODE_CURRENCY_SYMBOL, + G_UNICODE_CURRENCY_SYMBOL, G_UNICODE_CURRENCY_SYMBOL, + G_UNICODE_CURRENCY_SYMBOL, G_UNICODE_CURRENCY_SYMBOL, + G_UNICODE_CURRENCY_SYMBOL, G_UNICODE_CURRENCY_SYMBOL, + G_UNICODE_CURRENCY_SYMBOL, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_ENCLOSING_MARK, G_UNICODE_ENCLOSING_MARK, + G_UNICODE_ENCLOSING_MARK, G_UNICODE_ENCLOSING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_ENCLOSING_MARK, + G_UNICODE_ENCLOSING_MARK, G_UNICODE_ENCLOSING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED + }, + { /* page 33, index 28 */ + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_UNASSIGNED, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_LETTER_NUMBER, G_UNICODE_LETTER_NUMBER, + G_UNICODE_LETTER_NUMBER, G_UNICODE_LETTER_NUMBER, + G_UNICODE_LETTER_NUMBER, G_UNICODE_LETTER_NUMBER, + G_UNICODE_LETTER_NUMBER, G_UNICODE_LETTER_NUMBER, + G_UNICODE_LETTER_NUMBER, G_UNICODE_LETTER_NUMBER, + G_UNICODE_LETTER_NUMBER, G_UNICODE_LETTER_NUMBER, + G_UNICODE_LETTER_NUMBER, G_UNICODE_LETTER_NUMBER, + G_UNICODE_LETTER_NUMBER, G_UNICODE_LETTER_NUMBER, + G_UNICODE_LETTER_NUMBER, G_UNICODE_LETTER_NUMBER, + G_UNICODE_LETTER_NUMBER, G_UNICODE_LETTER_NUMBER, + G_UNICODE_LETTER_NUMBER, G_UNICODE_LETTER_NUMBER, + G_UNICODE_LETTER_NUMBER, G_UNICODE_LETTER_NUMBER, + G_UNICODE_LETTER_NUMBER, G_UNICODE_LETTER_NUMBER, + G_UNICODE_LETTER_NUMBER, G_UNICODE_LETTER_NUMBER, + G_UNICODE_LETTER_NUMBER, G_UNICODE_LETTER_NUMBER, + G_UNICODE_LETTER_NUMBER, G_UNICODE_LETTER_NUMBER, + G_UNICODE_LETTER_NUMBER, G_UNICODE_LETTER_NUMBER, + G_UNICODE_LETTER_NUMBER, G_UNICODE_LETTER_NUMBER, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL + }, + { /* page 35, index 29 */ + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OPEN_PUNCTUATION, G_UNICODE_CLOSE_PUNCTUATION, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_OPEN_PUNCTUATION, + G_UNICODE_CLOSE_PUNCTUATION, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED + }, + { /* page 36, index 30 */ + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER + }, + { /* page 37, index 31 */ + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL + }, + { /* page 38, index 32 */ + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED + }, + { /* page 39, index 33 */ + G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_UNASSIGNED, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OPEN_PUNCTUATION, G_UNICODE_CLOSE_PUNCTUATION, + G_UNICODE_OPEN_PUNCTUATION, G_UNICODE_CLOSE_PUNCTUATION, + G_UNICODE_OPEN_PUNCTUATION, G_UNICODE_CLOSE_PUNCTUATION, + G_UNICODE_OPEN_PUNCTUATION, G_UNICODE_CLOSE_PUNCTUATION, + G_UNICODE_OPEN_PUNCTUATION, G_UNICODE_CLOSE_PUNCTUATION, + G_UNICODE_OPEN_PUNCTUATION, G_UNICODE_CLOSE_PUNCTUATION, + G_UNICODE_OPEN_PUNCTUATION, G_UNICODE_CLOSE_PUNCTUATION, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_OPEN_PUNCTUATION, + G_UNICODE_CLOSE_PUNCTUATION, G_UNICODE_OPEN_PUNCTUATION, + G_UNICODE_CLOSE_PUNCTUATION, G_UNICODE_OPEN_PUNCTUATION, + G_UNICODE_CLOSE_PUNCTUATION, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL + }, + { /* page 41, index 34 */ + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_OPEN_PUNCTUATION, + G_UNICODE_CLOSE_PUNCTUATION, G_UNICODE_OPEN_PUNCTUATION, + G_UNICODE_CLOSE_PUNCTUATION, G_UNICODE_OPEN_PUNCTUATION, + G_UNICODE_CLOSE_PUNCTUATION, G_UNICODE_OPEN_PUNCTUATION, + G_UNICODE_CLOSE_PUNCTUATION, G_UNICODE_OPEN_PUNCTUATION, + G_UNICODE_CLOSE_PUNCTUATION, G_UNICODE_OPEN_PUNCTUATION, + G_UNICODE_CLOSE_PUNCTUATION, G_UNICODE_OPEN_PUNCTUATION, + G_UNICODE_CLOSE_PUNCTUATION, G_UNICODE_OPEN_PUNCTUATION, + G_UNICODE_CLOSE_PUNCTUATION, G_UNICODE_OPEN_PUNCTUATION, + G_UNICODE_CLOSE_PUNCTUATION, G_UNICODE_OPEN_PUNCTUATION, + G_UNICODE_CLOSE_PUNCTUATION, G_UNICODE_OPEN_PUNCTUATION, + G_UNICODE_CLOSE_PUNCTUATION, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_OPEN_PUNCTUATION, + G_UNICODE_CLOSE_PUNCTUATION, G_UNICODE_OPEN_PUNCTUATION, + G_UNICODE_CLOSE_PUNCTUATION, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_OPEN_PUNCTUATION, + G_UNICODE_CLOSE_PUNCTUATION, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL + }, + { /* page 43, index 35 */ + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED + }, + { /* page 46, index 36 */ + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED + }, + { /* page 47, index 37 */ + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED + }, + { /* page 48, index 38 */ + G_UNICODE_SPACE_SEPARATOR, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_MODIFIER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_LETTER_NUMBER, + G_UNICODE_OPEN_PUNCTUATION, G_UNICODE_CLOSE_PUNCTUATION, + G_UNICODE_OPEN_PUNCTUATION, G_UNICODE_CLOSE_PUNCTUATION, + G_UNICODE_OPEN_PUNCTUATION, G_UNICODE_CLOSE_PUNCTUATION, + G_UNICODE_OPEN_PUNCTUATION, G_UNICODE_CLOSE_PUNCTUATION, + G_UNICODE_OPEN_PUNCTUATION, G_UNICODE_CLOSE_PUNCTUATION, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OPEN_PUNCTUATION, G_UNICODE_CLOSE_PUNCTUATION, + G_UNICODE_OPEN_PUNCTUATION, G_UNICODE_CLOSE_PUNCTUATION, + G_UNICODE_OPEN_PUNCTUATION, G_UNICODE_CLOSE_PUNCTUATION, + G_UNICODE_OPEN_PUNCTUATION, G_UNICODE_CLOSE_PUNCTUATION, + G_UNICODE_DASH_PUNCTUATION, G_UNICODE_OPEN_PUNCTUATION, + G_UNICODE_CLOSE_PUNCTUATION, G_UNICODE_CLOSE_PUNCTUATION, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_LETTER_NUMBER, G_UNICODE_LETTER_NUMBER, + G_UNICODE_LETTER_NUMBER, G_UNICODE_LETTER_NUMBER, + G_UNICODE_LETTER_NUMBER, G_UNICODE_LETTER_NUMBER, + G_UNICODE_LETTER_NUMBER, G_UNICODE_LETTER_NUMBER, + G_UNICODE_LETTER_NUMBER, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_DASH_PUNCTUATION, + G_UNICODE_MODIFIER_LETTER, G_UNICODE_MODIFIER_LETTER, + G_UNICODE_MODIFIER_LETTER, G_UNICODE_MODIFIER_LETTER, + G_UNICODE_MODIFIER_LETTER, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_LETTER_NUMBER, G_UNICODE_LETTER_NUMBER, + G_UNICODE_LETTER_NUMBER, G_UNICODE_MODIFIER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_UNASSIGNED, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_MODIFIER_SYMBOL, + G_UNICODE_MODIFIER_SYMBOL, G_UNICODE_MODIFIER_LETTER, + G_UNICODE_MODIFIER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_DASH_PUNCTUATION, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_CONNECT_PUNCTUATION, G_UNICODE_MODIFIER_LETTER, + G_UNICODE_MODIFIER_LETTER, G_UNICODE_MODIFIER_LETTER, + G_UNICODE_OTHER_LETTER + }, + { /* page 49, index 39 */ + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER + }, + { /* page 50, index 40 */ + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_UNASSIGNED + }, + { /* page 77, index 41 */ + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL + }, + { /* page 159, index 42 */ + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED + }, + { /* page 164, index 43 */ + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED + }, + { /* page 215, index 44 */ + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED + }, + { /* page 250, index 45 */ + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED + }, + { /* page 251, index 46 */ + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_MATH_SYMBOL, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, + G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER + }, + { /* page 253, index 47 */ + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OPEN_PUNCTUATION, G_UNICODE_CLOSE_PUNCTUATION, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_CURRENCY_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED + }, + { /* page 254, index 48 */ + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_DASH_PUNCTUATION, G_UNICODE_DASH_PUNCTUATION, + G_UNICODE_CONNECT_PUNCTUATION, G_UNICODE_CONNECT_PUNCTUATION, + G_UNICODE_OPEN_PUNCTUATION, G_UNICODE_CLOSE_PUNCTUATION, + G_UNICODE_OPEN_PUNCTUATION, G_UNICODE_CLOSE_PUNCTUATION, + G_UNICODE_OPEN_PUNCTUATION, G_UNICODE_CLOSE_PUNCTUATION, + G_UNICODE_OPEN_PUNCTUATION, G_UNICODE_CLOSE_PUNCTUATION, + G_UNICODE_OPEN_PUNCTUATION, G_UNICODE_CLOSE_PUNCTUATION, + G_UNICODE_OPEN_PUNCTUATION, G_UNICODE_CLOSE_PUNCTUATION, + G_UNICODE_OPEN_PUNCTUATION, G_UNICODE_CLOSE_PUNCTUATION, + G_UNICODE_OPEN_PUNCTUATION, G_UNICODE_CLOSE_PUNCTUATION, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_OPEN_PUNCTUATION, G_UNICODE_CLOSE_PUNCTUATION, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_CONNECT_PUNCTUATION, G_UNICODE_CONNECT_PUNCTUATION, + G_UNICODE_CONNECT_PUNCTUATION, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_DASH_PUNCTUATION, + G_UNICODE_OPEN_PUNCTUATION, G_UNICODE_CLOSE_PUNCTUATION, + G_UNICODE_OPEN_PUNCTUATION, G_UNICODE_CLOSE_PUNCTUATION, + G_UNICODE_OPEN_PUNCTUATION, G_UNICODE_CLOSE_PUNCTUATION, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_MATH_SYMBOL, + G_UNICODE_DASH_PUNCTUATION, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_CURRENCY_SYMBOL, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_FORMAT + }, + { /* page 255, index 49 */ + G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_CURRENCY_SYMBOL, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_OPEN_PUNCTUATION, G_UNICODE_CLOSE_PUNCTUATION, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_MATH_SYMBOL, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_DASH_PUNCTUATION, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_OPEN_PUNCTUATION, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_CLOSE_PUNCTUATION, G_UNICODE_MODIFIER_SYMBOL, + G_UNICODE_CONNECT_PUNCTUATION, G_UNICODE_MODIFIER_SYMBOL, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_OPEN_PUNCTUATION, G_UNICODE_MATH_SYMBOL, + G_UNICODE_CLOSE_PUNCTUATION, G_UNICODE_MATH_SYMBOL, + G_UNICODE_OPEN_PUNCTUATION, G_UNICODE_CLOSE_PUNCTUATION, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_OPEN_PUNCTUATION, + G_UNICODE_CLOSE_PUNCTUATION, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_CONNECT_PUNCTUATION, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_MODIFIER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_MODIFIER_LETTER, G_UNICODE_MODIFIER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_CURRENCY_SYMBOL, + G_UNICODE_CURRENCY_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MODIFIER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_CURRENCY_SYMBOL, G_UNICODE_CURRENCY_SYMBOL, + G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, G_UNICODE_MATH_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_FORMAT, G_UNICODE_FORMAT, G_UNICODE_FORMAT, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED + }, + { /* page 256, index 50 */ + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED + }, + { /* page 257, index 51 */ + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_OTHER_PUNCTUATION, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED + }, + { /* page 259, index 52 */ + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_NUMBER, + G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, G_UNICODE_OTHER_NUMBER, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_LETTER_NUMBER, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, + G_UNICODE_OTHER_PUNCTUATION, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED + }, + { /* page 260, index 53 */ + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED + }, + { /* page 264, index 54 */ + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, + G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_UNASSIGNED, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED + }, + { /* page 464, index 55 */ + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED + }, + { /* page 465, index 56 */ + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_COMBINING_MARK, + G_UNICODE_COMBINING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_COMBINING_MARK, G_UNICODE_COMBINING_MARK, + G_UNICODE_COMBINING_MARK, G_UNICODE_COMBINING_MARK, + G_UNICODE_COMBINING_MARK, G_UNICODE_COMBINING_MARK, G_UNICODE_FORMAT, + G_UNICODE_FORMAT, G_UNICODE_FORMAT, G_UNICODE_FORMAT, G_UNICODE_FORMAT, + G_UNICODE_FORMAT, G_UNICODE_FORMAT, G_UNICODE_FORMAT, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED + }, + { /* page 467, index 57 */ + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, G_UNICODE_OTHER_SYMBOL, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED + }, + { /* page 468, index 58 */ + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UNASSIGNED, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UNASSIGNED, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UNASSIGNED, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UNASSIGNED, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UNASSIGNED, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UNASSIGNED, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER + }, + { /* page 469, index 59 */ + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UNASSIGNED, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UNASSIGNED, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UNASSIGNED, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UNASSIGNED, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UNASSIGNED, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UNASSIGNED, G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UNASSIGNED, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER + }, + { /* page 470, index 60 */ + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_MATH_SYMBOL, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_MATH_SYMBOL, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_MATH_SYMBOL, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER + }, + { /* page 471, index 61 */ + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_MATH_SYMBOL, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_MATH_SYMBOL, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_MATH_SYMBOL, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_MATH_SYMBOL, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_MATH_SYMBOL, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_UPPERCASE_LETTER, + G_UNICODE_UPPERCASE_LETTER, G_UNICODE_MATH_SYMBOL, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_MATH_SYMBOL, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_LOWERCASE_LETTER, G_UNICODE_LOWERCASE_LETTER, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER, + G_UNICODE_DECIMAL_NUMBER, G_UNICODE_DECIMAL_NUMBER + }, + { /* page 678, index 62 */ + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED + }, + { /* page 762, index 63 */ + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, G_UNICODE_OTHER_LETTER, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED + }, + { /* page 3584, index 64 */ + G_UNICODE_UNASSIGNED, G_UNICODE_FORMAT, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_FORMAT, + G_UNICODE_FORMAT, G_UNICODE_FORMAT, G_UNICODE_FORMAT, G_UNICODE_FORMAT, + G_UNICODE_FORMAT, G_UNICODE_FORMAT, G_UNICODE_FORMAT, G_UNICODE_FORMAT, + G_UNICODE_FORMAT, G_UNICODE_FORMAT, G_UNICODE_FORMAT, G_UNICODE_FORMAT, + G_UNICODE_FORMAT, G_UNICODE_FORMAT, G_UNICODE_FORMAT, G_UNICODE_FORMAT, + G_UNICODE_FORMAT, G_UNICODE_FORMAT, G_UNICODE_FORMAT, G_UNICODE_FORMAT, + G_UNICODE_FORMAT, G_UNICODE_FORMAT, G_UNICODE_FORMAT, G_UNICODE_FORMAT, + G_UNICODE_FORMAT, G_UNICODE_FORMAT, G_UNICODE_FORMAT, G_UNICODE_FORMAT, + G_UNICODE_FORMAT, G_UNICODE_FORMAT, G_UNICODE_FORMAT, G_UNICODE_FORMAT, + G_UNICODE_FORMAT, G_UNICODE_FORMAT, G_UNICODE_FORMAT, G_UNICODE_FORMAT, + G_UNICODE_FORMAT, G_UNICODE_FORMAT, G_UNICODE_FORMAT, G_UNICODE_FORMAT, + G_UNICODE_FORMAT, G_UNICODE_FORMAT, G_UNICODE_FORMAT, G_UNICODE_FORMAT, + G_UNICODE_FORMAT, G_UNICODE_FORMAT, G_UNICODE_FORMAT, G_UNICODE_FORMAT, + G_UNICODE_FORMAT, G_UNICODE_FORMAT, G_UNICODE_FORMAT, G_UNICODE_FORMAT, + G_UNICODE_FORMAT, G_UNICODE_FORMAT, G_UNICODE_FORMAT, G_UNICODE_FORMAT, + G_UNICODE_FORMAT, G_UNICODE_FORMAT, G_UNICODE_FORMAT, G_UNICODE_FORMAT, + G_UNICODE_FORMAT, G_UNICODE_FORMAT, G_UNICODE_FORMAT, G_UNICODE_FORMAT, + G_UNICODE_FORMAT, G_UNICODE_FORMAT, G_UNICODE_FORMAT, G_UNICODE_FORMAT, + G_UNICODE_FORMAT, G_UNICODE_FORMAT, G_UNICODE_FORMAT, G_UNICODE_FORMAT, + G_UNICODE_FORMAT, G_UNICODE_FORMAT, G_UNICODE_FORMAT, G_UNICODE_FORMAT, + G_UNICODE_FORMAT, G_UNICODE_FORMAT, G_UNICODE_FORMAT, G_UNICODE_FORMAT, + G_UNICODE_FORMAT, G_UNICODE_FORMAT, G_UNICODE_FORMAT, G_UNICODE_FORMAT, + G_UNICODE_FORMAT, G_UNICODE_FORMAT, G_UNICODE_FORMAT, G_UNICODE_FORMAT, + G_UNICODE_FORMAT, G_UNICODE_FORMAT, G_UNICODE_FORMAT, G_UNICODE_FORMAT, + G_UNICODE_FORMAT, G_UNICODE_FORMAT, G_UNICODE_FORMAT, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED + }, + { /* page 3585, index 65 */ + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_NON_SPACING_MARK, G_UNICODE_NON_SPACING_MARK, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED + }, + { /* page 4095, index 66 */ + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED + }, + { /* page 4351, index 67 */ + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, + G_UNICODE_PRIVATE_USE, G_UNICODE_PRIVATE_USE, G_UNICODE_UNASSIGNED, + G_UNICODE_UNASSIGNED + } +}; + +/* U+0000 through U+2FAFF */ +static const gint16 type_table_part1[763] = { + 0 /* page 0 */, + 1 /* page 1 */, + 2 /* page 2 */, + 3 /* page 3 */, + 4 /* page 4 */, + 5 /* page 5 */, + 6 /* page 6 */, + 7 /* page 7 */, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + 8 /* page 9 */, + 9 /* page 10 */, + 10 /* page 11 */, + 11 /* page 12 */, + 12 /* page 13 */, + 13 /* page 14 */, + 14 /* page 15 */, + 15 /* page 16 */, + 16 /* page 17 */, + 17 /* page 18 */, + 18 /* page 19 */, + 19 /* page 20 */, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + 20 /* page 22 */, + 21 /* page 23 */, + 22 /* page 24 */, + 23 /* page 25 */, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + 24 /* page 29 */, + 25 /* page 30 */, + 26 /* page 31 */, + 27 /* page 32 */, + 28 /* page 33 */, + G_UNICODE_MATH_SYMBOL + G_UNICODE_MAX_TABLE_INDEX, + 29 /* page 35 */, + 30 /* page 36 */, + 31 /* page 37 */, + 32 /* page 38 */, + 33 /* page 39 */, + G_UNICODE_OTHER_SYMBOL + G_UNICODE_MAX_TABLE_INDEX, + 34 /* page 41 */, + G_UNICODE_MATH_SYMBOL + G_UNICODE_MAX_TABLE_INDEX, + 35 /* page 43 */, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + 36 /* page 46 */, + 37 /* page 47 */, + 38 /* page 48 */, + 39 /* page 49 */, + 40 /* page 50 */, + G_UNICODE_OTHER_SYMBOL + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + 41 /* page 77 */, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + 42 /* page 159 */, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + 43 /* page 164 */, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + 44 /* page 215 */, + G_UNICODE_SURROGATE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_SURROGATE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_SURROGATE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_SURROGATE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_SURROGATE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_SURROGATE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_SURROGATE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_SURROGATE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + 45 /* page 250 */, + 46 /* page 251 */, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + 47 /* page 253 */, + 48 /* page 254 */, + 49 /* page 255 */, + 50 /* page 256 */, + 51 /* page 257 */, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + 52 /* page 259 */, + 53 /* page 260 */, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + 54 /* page 264 */, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + 55 /* page 464 */, + 56 /* page 465 */, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + 57 /* page 467 */, + 58 /* page 468 */, + 59 /* page 469 */, + 60 /* page 470 */, + 61 /* page 471 */, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + 62 /* page 678 */, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_OTHER_LETTER + G_UNICODE_MAX_TABLE_INDEX, + 63 /* page 762 */ +}; + +/* U+E0000 through U+10FFFF */ +static const gint16 type_table_part2[768] = { + 64 /* page 3584 */, + 65 /* page 3585 */, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_UNASSIGNED + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + 66 /* page 4095 */, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + G_UNICODE_PRIVATE_USE + G_UNICODE_MAX_TABLE_INDEX, + 67 /* page 4351 */ +}; + +static const gunichar attr_data[][256] = { + { /* page 0, index 0 */ + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, + 0x0006, 0x0007, 0x0008, 0x0009, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, 0x0070, + 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, 0x0078, 0x0079, + 0x007a, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0041, 0x0042, + 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, 0x0048, 0x0049, 0x004a, 0x004b, + 0x004c, 0x004d, 0x004e, 0x004f, 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, + 0x0055, 0x0056, 0x0057, 0x0058, 0x0059, 0x005a, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x039c, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x00e0, 0x00e1, 0x00e2, 0x00e3, 0x00e4, 0x00e5, + 0x00e6, 0x00e7, 0x00e8, 0x00e9, 0x00ea, 0x00eb, 0x00ec, 0x00ed, 0x00ee, + 0x00ef, 0x00f0, 0x00f1, 0x00f2, 0x00f3, 0x00f4, 0x00f5, 0x00f6, 0x0000, + 0x00f8, 0x00f9, 0x00fa, 0x00fb, 0x00fc, 0x00fd, 0x00fe, 0x1000000, + 0x00c0, 0x00c1, 0x00c2, 0x00c3, 0x00c4, 0x00c5, 0x00c6, 0x00c7, 0x00c8, + 0x00c9, 0x00ca, 0x00cb, 0x00cc, 0x00cd, 0x00ce, 0x00cf, 0x00d0, 0x00d1, + 0x00d2, 0x00d3, 0x00d4, 0x00d5, 0x00d6, 0x0000, 0x00d8, 0x00d9, 0x00da, + 0x00db, 0x00dc, 0x00dd, 0x00de, 0x0178 + }, + { /* page 1, index 1 */ + 0x0101, 0x0100, 0x0103, 0x0102, 0x0105, 0x0104, 0x0107, 0x0106, 0x0109, + 0x0108, 0x010b, 0x010a, 0x010d, 0x010c, 0x010f, 0x010e, 0x0111, 0x0110, + 0x0113, 0x0112, 0x0115, 0x0114, 0x0117, 0x0116, 0x0119, 0x0118, 0x011b, + 0x011a, 0x011d, 0x011c, 0x011f, 0x011e, 0x0121, 0x0120, 0x0123, 0x0122, + 0x0125, 0x0124, 0x0127, 0x0126, 0x0129, 0x0128, 0x012b, 0x012a, 0x012d, + 0x012c, 0x012f, 0x012e, 0x1000007, 0x0049, 0x0133, 0x0132, 0x0135, + 0x0134, 0x0137, 0x0136, 0x0000, 0x013a, 0x0139, 0x013c, 0x013b, 0x013e, + 0x013d, 0x0140, 0x013f, 0x0142, 0x0141, 0x0144, 0x0143, 0x0146, 0x0145, + 0x0148, 0x0147, 0x1000086, 0x014b, 0x014a, 0x014d, 0x014c, 0x014f, + 0x014e, 0x0151, 0x0150, 0x0153, 0x0152, 0x0155, 0x0154, 0x0157, 0x0156, + 0x0159, 0x0158, 0x015b, 0x015a, 0x015d, 0x015c, 0x015f, 0x015e, 0x0161, + 0x0160, 0x0163, 0x0162, 0x0165, 0x0164, 0x0167, 0x0166, 0x0169, 0x0168, + 0x016b, 0x016a, 0x016d, 0x016c, 0x016f, 0x016e, 0x0171, 0x0170, 0x0173, + 0x0172, 0x0175, 0x0174, 0x0177, 0x0176, 0x00ff, 0x017a, 0x0179, 0x017c, + 0x017b, 0x017e, 0x017d, 0x0053, 0x0000, 0x0253, 0x0183, 0x0182, 0x0185, + 0x0184, 0x0254, 0x0188, 0x0187, 0x0256, 0x0257, 0x018c, 0x018b, 0x0000, + 0x01dd, 0x0259, 0x025b, 0x0192, 0x0191, 0x0260, 0x0263, 0x01f6, 0x0269, + 0x0268, 0x0199, 0x0198, 0x0000, 0x0000, 0x026f, 0x0272, 0x0220, 0x0275, + 0x01a1, 0x01a0, 0x01a3, 0x01a2, 0x01a5, 0x01a4, 0x0280, 0x01a8, 0x01a7, + 0x0283, 0x0000, 0x0000, 0x01ad, 0x01ac, 0x0288, 0x01b0, 0x01af, 0x028a, + 0x028b, 0x01b4, 0x01b3, 0x01b6, 0x01b5, 0x0292, 0x01b9, 0x01b8, 0x0000, + 0x0000, 0x01bd, 0x01bc, 0x0000, 0x01f7, 0x0000, 0x0000, 0x0000, 0x0000, + 0x01c6, 0x0000, 0x01c4, 0x01c9, 0x0000, 0x01c7, 0x01cc, 0x0000, 0x01ca, + 0x01ce, 0x01cd, 0x01d0, 0x01cf, 0x01d2, 0x01d1, 0x01d4, 0x01d3, 0x01d6, + 0x01d5, 0x01d8, 0x01d7, 0x01da, 0x01d9, 0x01dc, 0x01db, 0x018e, 0x01df, + 0x01de, 0x01e1, 0x01e0, 0x01e3, 0x01e2, 0x01e5, 0x01e4, 0x01e7, 0x01e6, + 0x01e9, 0x01e8, 0x01eb, 0x01ea, 0x01ed, 0x01ec, 0x01ef, 0x01ee, + 0x10000ad, 0x01f3, 0x0000, 0x01f1, 0x01f5, 0x01f4, 0x0195, 0x01bf, + 0x01f9, 0x01f8, 0x01fb, 0x01fa, 0x01fd, 0x01fc, 0x01ff, 0x01fe + }, + { /* page 2, index 2 */ + 0x0201, 0x0200, 0x0203, 0x0202, 0x0205, 0x0204, 0x0207, 0x0206, 0x0209, + 0x0208, 0x020b, 0x020a, 0x020d, 0x020c, 0x020f, 0x020e, 0x0211, 0x0210, + 0x0213, 0x0212, 0x0215, 0x0214, 0x0217, 0x0216, 0x0219, 0x0218, 0x021b, + 0x021a, 0x021d, 0x021c, 0x021f, 0x021e, 0x019e, 0x0000, 0x0223, 0x0222, + 0x0225, 0x0224, 0x0227, 0x0226, 0x0229, 0x0228, 0x022b, 0x022a, 0x022d, + 0x022c, 0x022f, 0x022e, 0x0231, 0x0230, 0x0233, 0x0232, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0181, 0x0186, 0x0000, 0x0189, 0x018a, 0x0000, 0x018f, + 0x0000, 0x0190, 0x0000, 0x0000, 0x0000, 0x0000, 0x0193, 0x0000, 0x0000, + 0x0194, 0x0000, 0x0000, 0x0000, 0x0000, 0x0197, 0x0196, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x019c, 0x0000, 0x0000, 0x019d, 0x0000, 0x0000, + 0x019f, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x01a6, 0x0000, 0x0000, 0x01a9, 0x0000, 0x0000, 0x0000, + 0x0000, 0x01ae, 0x0000, 0x01b1, 0x01b2, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x01b7, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000 + }, + { /* page 3, index 3 */ + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x03ac, + 0x0000, 0x03ad, 0x03ae, 0x03af, 0x0000, 0x03cc, 0x0000, 0x03cd, 0x03ce, + 0x100008f, 0x03b1, 0x03b2, 0x03b3, 0x03b4, 0x03b5, 0x03b6, 0x03b7, + 0x03b8, 0x03b9, 0x03ba, 0x03bb, 0x03bc, 0x03bd, 0x03be, 0x03bf, 0x03c0, + 0x03c1, 0x0000, 0x03c3, 0x03c4, 0x03c5, 0x03c6, 0x03c7, 0x03c8, 0x03c9, + 0x03ca, 0x03cb, 0x0386, 0x0388, 0x0389, 0x038a, 0x100009e, 0x0391, + 0x0392, 0x0393, 0x0394, 0x0395, 0x0396, 0x0397, 0x0398, 0x0399, 0x039a, + 0x039b, 0x039c, 0x039d, 0x039e, 0x039f, 0x03a0, 0x03a1, 0x03a3, 0x03a3, + 0x03a4, 0x03a5, 0x03a6, 0x03a7, 0x03a8, 0x03a9, 0x03aa, 0x03ab, 0x038c, + 0x038e, 0x038f, 0x0000, 0x0392, 0x0398, 0x0000, 0x0000, 0x0000, 0x03a6, + 0x03a0, 0x0000, 0x03d9, 0x03d8, 0x03db, 0x03da, 0x03dd, 0x03dc, 0x03df, + 0x03de, 0x03e1, 0x03e0, 0x03e3, 0x03e2, 0x03e5, 0x03e4, 0x03e7, 0x03e6, + 0x03e9, 0x03e8, 0x03eb, 0x03ea, 0x03ed, 0x03ec, 0x03ef, 0x03ee, 0x039a, + 0x03a1, 0x03f9, 0x0000, 0x03b8, 0x0395, 0x0000, 0x03f8, 0x03f7, 0x03f2, + 0x03fb, 0x03fa, 0x0000, 0x0000, 0x0000, 0x0000 + }, + { /* page 4, index 4 */ + 0x0450, 0x0451, 0x0452, 0x0453, 0x0454, 0x0455, 0x0456, 0x0457, 0x0458, + 0x0459, 0x045a, 0x045b, 0x045c, 0x045d, 0x045e, 0x045f, 0x0430, 0x0431, + 0x0432, 0x0433, 0x0434, 0x0435, 0x0436, 0x0437, 0x0438, 0x0439, 0x043a, + 0x043b, 0x043c, 0x043d, 0x043e, 0x043f, 0x0440, 0x0441, 0x0442, 0x0443, + 0x0444, 0x0445, 0x0446, 0x0447, 0x0448, 0x0449, 0x044a, 0x044b, 0x044c, + 0x044d, 0x044e, 0x044f, 0x0410, 0x0411, 0x0412, 0x0413, 0x0414, 0x0415, + 0x0416, 0x0417, 0x0418, 0x0419, 0x041a, 0x041b, 0x041c, 0x041d, 0x041e, + 0x041f, 0x0420, 0x0421, 0x0422, 0x0423, 0x0424, 0x0425, 0x0426, 0x0427, + 0x0428, 0x0429, 0x042a, 0x042b, 0x042c, 0x042d, 0x042e, 0x042f, 0x0400, + 0x0401, 0x0402, 0x0403, 0x0404, 0x0405, 0x0406, 0x0407, 0x0408, 0x0409, + 0x040a, 0x040b, 0x040c, 0x040d, 0x040e, 0x040f, 0x0461, 0x0460, 0x0463, + 0x0462, 0x0465, 0x0464, 0x0467, 0x0466, 0x0469, 0x0468, 0x046b, 0x046a, + 0x046d, 0x046c, 0x046f, 0x046e, 0x0471, 0x0470, 0x0473, 0x0472, 0x0475, + 0x0474, 0x0477, 0x0476, 0x0479, 0x0478, 0x047b, 0x047a, 0x047d, 0x047c, + 0x047f, 0x047e, 0x0481, 0x0480, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x048b, 0x048a, 0x048d, 0x048c, 0x048f, 0x048e, + 0x0491, 0x0490, 0x0493, 0x0492, 0x0495, 0x0494, 0x0497, 0x0496, 0x0499, + 0x0498, 0x049b, 0x049a, 0x049d, 0x049c, 0x049f, 0x049e, 0x04a1, 0x04a0, + 0x04a3, 0x04a2, 0x04a5, 0x04a4, 0x04a7, 0x04a6, 0x04a9, 0x04a8, 0x04ab, + 0x04aa, 0x04ad, 0x04ac, 0x04af, 0x04ae, 0x04b1, 0x04b0, 0x04b3, 0x04b2, + 0x04b5, 0x04b4, 0x04b7, 0x04b6, 0x04b9, 0x04b8, 0x04bb, 0x04ba, 0x04bd, + 0x04bc, 0x04bf, 0x04be, 0x0000, 0x04c2, 0x04c1, 0x04c4, 0x04c3, 0x04c6, + 0x04c5, 0x04c8, 0x04c7, 0x04ca, 0x04c9, 0x04cc, 0x04cb, 0x04ce, 0x04cd, + 0x0000, 0x04d1, 0x04d0, 0x04d3, 0x04d2, 0x04d5, 0x04d4, 0x04d7, 0x04d6, + 0x04d9, 0x04d8, 0x04db, 0x04da, 0x04dd, 0x04dc, 0x04df, 0x04de, 0x04e1, + 0x04e0, 0x04e3, 0x04e2, 0x04e5, 0x04e4, 0x04e7, 0x04e6, 0x04e9, 0x04e8, + 0x04eb, 0x04ea, 0x04ed, 0x04ec, 0x04ef, 0x04ee, 0x04f1, 0x04f0, 0x04f3, + 0x04f2, 0x04f5, 0x04f4, 0x0000, 0x0000, 0x04f9, 0x04f8, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000 + }, + { /* page 5, index 5 */ + 0x0501, 0x0500, 0x0503, 0x0502, 0x0505, 0x0504, 0x0507, 0x0506, 0x0509, + 0x0508, 0x050b, 0x050a, 0x050d, 0x050c, 0x050f, 0x050e, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0561, 0x0562, 0x0563, 0x0564, 0x0565, + 0x0566, 0x0567, 0x0568, 0x0569, 0x056a, 0x056b, 0x056c, 0x056d, 0x056e, + 0x056f, 0x0570, 0x0571, 0x0572, 0x0573, 0x0574, 0x0575, 0x0576, 0x0577, + 0x0578, 0x0579, 0x057a, 0x057b, 0x057c, 0x057d, 0x057e, 0x057f, 0x0580, + 0x0581, 0x0582, 0x0583, 0x0584, 0x0585, 0x0586, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0531, 0x0532, + 0x0533, 0x0534, 0x0535, 0x0536, 0x0537, 0x0538, 0x0539, 0x053a, 0x053b, + 0x053c, 0x053d, 0x053e, 0x053f, 0x0540, 0x0541, 0x0542, 0x0543, 0x0544, + 0x0545, 0x0546, 0x0547, 0x0548, 0x0549, 0x054a, 0x054b, 0x054c, 0x054d, + 0x054e, 0x054f, 0x0550, 0x0551, 0x0552, 0x0553, 0x0554, 0x0555, 0x0556, + 0x1000044, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000 + }, + { /* page 6, index 6 */ + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0001, 0x0002, + 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0001, 0x0002, + 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000 + }, + { /* page 9, index 7 */ + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, + 0x0006, 0x0007, 0x0008, 0x0009, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0001, 0x0002, 0x0003, + 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000 + }, + { /* page 10, index 8 */ + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, + 0x0006, 0x0007, 0x0008, 0x0009, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0001, 0x0002, 0x0003, + 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000 + }, + { /* page 11, index 9 */ + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, + 0x0006, 0x0007, 0x0008, 0x0009, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0001, 0x0002, 0x0003, + 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000 + }, + { /* page 12, index 10 */ + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, + 0x0006, 0x0007, 0x0008, 0x0009, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0001, 0x0002, 0x0003, + 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000 + }, + { /* page 13, index 11 */ + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, + 0x0006, 0x0007, 0x0008, 0x0009, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000 + }, + { /* page 14, index 12 */ + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000 + }, + { /* page 15, index 13 */ + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0001, 0x0002, 0x0003, + 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000 + }, + { /* page 16, index 14 */ + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000 + }, + { /* page 19, index 15 */ + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0001, 0x0002, 0x0003, + 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000 + }, + { /* page 23, index 16 */ + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000 + }, + { /* page 24, index 17 */ + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0001, + 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000 + }, + { /* page 25, index 18 */ + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0001, + 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000 + }, + { /* page 30, index 19 */ + 0x1e01, 0x1e00, 0x1e03, 0x1e02, 0x1e05, 0x1e04, 0x1e07, 0x1e06, 0x1e09, + 0x1e08, 0x1e0b, 0x1e0a, 0x1e0d, 0x1e0c, 0x1e0f, 0x1e0e, 0x1e11, 0x1e10, + 0x1e13, 0x1e12, 0x1e15, 0x1e14, 0x1e17, 0x1e16, 0x1e19, 0x1e18, 0x1e1b, + 0x1e1a, 0x1e1d, 0x1e1c, 0x1e1f, 0x1e1e, 0x1e21, 0x1e20, 0x1e23, 0x1e22, + 0x1e25, 0x1e24, 0x1e27, 0x1e26, 0x1e29, 0x1e28, 0x1e2b, 0x1e2a, 0x1e2d, + 0x1e2c, 0x1e2f, 0x1e2e, 0x1e31, 0x1e30, 0x1e33, 0x1e32, 0x1e35, 0x1e34, + 0x1e37, 0x1e36, 0x1e39, 0x1e38, 0x1e3b, 0x1e3a, 0x1e3d, 0x1e3c, 0x1e3f, + 0x1e3e, 0x1e41, 0x1e40, 0x1e43, 0x1e42, 0x1e45, 0x1e44, 0x1e47, 0x1e46, + 0x1e49, 0x1e48, 0x1e4b, 0x1e4a, 0x1e4d, 0x1e4c, 0x1e4f, 0x1e4e, 0x1e51, + 0x1e50, 0x1e53, 0x1e52, 0x1e55, 0x1e54, 0x1e57, 0x1e56, 0x1e59, 0x1e58, + 0x1e5b, 0x1e5a, 0x1e5d, 0x1e5c, 0x1e5f, 0x1e5e, 0x1e61, 0x1e60, 0x1e63, + 0x1e62, 0x1e65, 0x1e64, 0x1e67, 0x1e66, 0x1e69, 0x1e68, 0x1e6b, 0x1e6a, + 0x1e6d, 0x1e6c, 0x1e6f, 0x1e6e, 0x1e71, 0x1e70, 0x1e73, 0x1e72, 0x1e75, + 0x1e74, 0x1e77, 0x1e76, 0x1e79, 0x1e78, 0x1e7b, 0x1e7a, 0x1e7d, 0x1e7c, + 0x1e7f, 0x1e7e, 0x1e81, 0x1e80, 0x1e83, 0x1e82, 0x1e85, 0x1e84, 0x1e87, + 0x1e86, 0x1e89, 0x1e88, 0x1e8b, 0x1e8a, 0x1e8d, 0x1e8c, 0x1e8f, 0x1e8e, + 0x1e91, 0x1e90, 0x1e93, 0x1e92, 0x1e95, 0x1e94, 0x10000b6, 0x10000bf, + 0x10000c8, 0x10000d1, 0x10000da, 0x1e60, 0x0000, 0x0000, 0x0000, 0x0000, + 0x1ea1, 0x1ea0, 0x1ea3, 0x1ea2, 0x1ea5, 0x1ea4, 0x1ea7, 0x1ea6, 0x1ea9, + 0x1ea8, 0x1eab, 0x1eaa, 0x1ead, 0x1eac, 0x1eaf, 0x1eae, 0x1eb1, 0x1eb0, + 0x1eb3, 0x1eb2, 0x1eb5, 0x1eb4, 0x1eb7, 0x1eb6, 0x1eb9, 0x1eb8, 0x1ebb, + 0x1eba, 0x1ebd, 0x1ebc, 0x1ebf, 0x1ebe, 0x1ec1, 0x1ec0, 0x1ec3, 0x1ec2, + 0x1ec5, 0x1ec4, 0x1ec7, 0x1ec6, 0x1ec9, 0x1ec8, 0x1ecb, 0x1eca, 0x1ecd, + 0x1ecc, 0x1ecf, 0x1ece, 0x1ed1, 0x1ed0, 0x1ed3, 0x1ed2, 0x1ed5, 0x1ed4, + 0x1ed7, 0x1ed6, 0x1ed9, 0x1ed8, 0x1edb, 0x1eda, 0x1edd, 0x1edc, 0x1edf, + 0x1ede, 0x1ee1, 0x1ee0, 0x1ee3, 0x1ee2, 0x1ee5, 0x1ee4, 0x1ee7, 0x1ee6, + 0x1ee9, 0x1ee8, 0x1eeb, 0x1eea, 0x1eed, 0x1eec, 0x1eef, 0x1eee, 0x1ef1, + 0x1ef0, 0x1ef3, 0x1ef2, 0x1ef5, 0x1ef4, 0x1ef7, 0x1ef6, 0x1ef9, 0x1ef8, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000 + }, + { /* page 31, index 20 */ + 0x1f08, 0x1f09, 0x1f0a, 0x1f0b, 0x1f0c, 0x1f0d, 0x1f0e, 0x1f0f, 0x1f00, + 0x1f01, 0x1f02, 0x1f03, 0x1f04, 0x1f05, 0x1f06, 0x1f07, 0x1f18, 0x1f19, + 0x1f1a, 0x1f1b, 0x1f1c, 0x1f1d, 0x0000, 0x0000, 0x1f10, 0x1f11, 0x1f12, + 0x1f13, 0x1f14, 0x1f15, 0x0000, 0x0000, 0x1f28, 0x1f29, 0x1f2a, 0x1f2b, + 0x1f2c, 0x1f2d, 0x1f2e, 0x1f2f, 0x1f20, 0x1f21, 0x1f22, 0x1f23, 0x1f24, + 0x1f25, 0x1f26, 0x1f27, 0x1f38, 0x1f39, 0x1f3a, 0x1f3b, 0x1f3c, 0x1f3d, + 0x1f3e, 0x1f3f, 0x1f30, 0x1f31, 0x1f32, 0x1f33, 0x1f34, 0x1f35, 0x1f36, + 0x1f37, 0x1f48, 0x1f49, 0x1f4a, 0x1f4b, 0x1f4c, 0x1f4d, 0x0000, 0x0000, + 0x1f40, 0x1f41, 0x1f42, 0x1f43, 0x1f44, 0x1f45, 0x0000, 0x0000, + 0x10000e3, 0x1f59, 0x10000ee, 0x1f5b, 0x10000fd, 0x1f5d, 0x100010c, + 0x1f5f, 0x0000, 0x1f51, 0x0000, 0x1f53, 0x0000, 0x1f55, 0x0000, 0x1f57, + 0x1f68, 0x1f69, 0x1f6a, 0x1f6b, 0x1f6c, 0x1f6d, 0x1f6e, 0x1f6f, 0x1f60, + 0x1f61, 0x1f62, 0x1f63, 0x1f64, 0x1f65, 0x1f66, 0x1f67, 0x1fba, 0x1fbb, + 0x1fc8, 0x1fc9, 0x1fca, 0x1fcb, 0x1fda, 0x1fdb, 0x1ff8, 0x1ff9, 0x1fea, + 0x1feb, 0x1ffa, 0x1ffb, 0x0000, 0x0000, 0x10001b7, 0x10001c4, 0x10001d1, + 0x10001de, 0x10001eb, 0x10001f8, 0x1000205, 0x1000212, 0x100021f, + 0x1000229, 0x1000233, 0x100023d, 0x1000247, 0x1000251, 0x100025b, + 0x1000265, 0x100026f, 0x100027c, 0x1000289, 0x1000296, 0x10002a3, + 0x10002b0, 0x10002bd, 0x10002ca, 0x10002d7, 0x10002e1, 0x10002eb, + 0x10002f5, 0x10002ff, 0x1000309, 0x1000313, 0x100031d, 0x1000327, + 0x1000334, 0x1000341, 0x100034e, 0x100035b, 0x1000368, 0x1000375, + 0x1000382, 0x100038f, 0x1000399, 0x10003a3, 0x10003ad, 0x10003b7, + 0x10003c1, 0x10003cb, 0x10003d5, 0x1fb8, 0x1fb9, 0x100041e, 0x10003df, + 0x100042b, 0x0000, 0x100011b, 0x1000466, 0x1fb0, 0x1fb1, 0x1f70, 0x1f71, + 0x10003eb, 0x0000, 0x0399, 0x0000, 0x0000, 0x0000, 0x1000436, 0x10003f4, + 0x1000443, 0x0000, 0x1000126, 0x1000475, 0x1f72, 0x1f73, 0x1f74, 0x1f75, + 0x1000400, 0x0000, 0x0000, 0x0000, 0x1fd8, 0x1fd9, 0x1000131, 0x1000140, + 0x0000, 0x0000, 0x100014f, 0x100015a, 0x1fd0, 0x1fd1, 0x1f76, 0x1f77, + 0x0000, 0x0000, 0x0000, 0x0000, 0x1fe8, 0x1fe9, 0x1000169, 0x1000178, + 0x1000187, 0x1fec, 0x1000192, 0x100019d, 0x1fe0, 0x1fe1, 0x1f7a, 0x1f7b, + 0x1fe5, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x100044e, 0x1000409, + 0x100045b, 0x0000, 0x10001ac, 0x1000484, 0x1f78, 0x1f79, 0x1f7c, 0x1f7d, + 0x1000415, 0x0000, 0x0000, 0x0000 + }, + { /* page 33, index 21 */ + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x03c9, 0x0000, 0x0000, 0x0000, 0x006b, 0x00e5, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000 + }, + { /* page 251, index 22 */ + 0x100000f, 0x1000016, 0x100001d, 0x1000024, 0x100002d, 0x1000036, + 0x100003d, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x100004f, 0x100005a, 0x1000065, + 0x1000070, 0x100007b, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000 + }, + { /* page 255, index 23 */ + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0001, + 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0xff41, 0xff42, 0xff43, + 0xff44, 0xff45, 0xff46, 0xff47, 0xff48, 0xff49, 0xff4a, 0xff4b, 0xff4c, + 0xff4d, 0xff4e, 0xff4f, 0xff50, 0xff51, 0xff52, 0xff53, 0xff54, 0xff55, + 0xff56, 0xff57, 0xff58, 0xff59, 0xff5a, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0xff21, 0xff22, 0xff23, 0xff24, 0xff25, 0xff26, 0xff27, + 0xff28, 0xff29, 0xff2a, 0xff2b, 0xff2c, 0xff2d, 0xff2e, 0xff2f, 0xff30, + 0xff31, 0xff32, 0xff33, 0xff34, 0xff35, 0xff36, 0xff37, 0xff38, 0xff39, + 0xff3a, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000 + }, + { /* page 260, index 24 */ + 0x10428, 0x10429, 0x1042a, 0x1042b, 0x1042c, 0x1042d, 0x1042e, 0x1042f, + 0x10430, 0x10431, 0x10432, 0x10433, 0x10434, 0x10435, 0x10436, 0x10437, + 0x10438, 0x10439, 0x1043a, 0x1043b, 0x1043c, 0x1043d, 0x1043e, 0x1043f, + 0x10440, 0x10441, 0x10442, 0x10443, 0x10444, 0x10445, 0x10446, 0x10447, + 0x10448, 0x10449, 0x1044a, 0x1044b, 0x1044c, 0x1044d, 0x1044e, 0x1044f, + 0x10400, 0x10401, 0x10402, 0x10403, 0x10404, 0x10405, 0x10406, 0x10407, + 0x10408, 0x10409, 0x1040a, 0x1040b, 0x1040c, 0x1040d, 0x1040e, 0x1040f, + 0x10410, 0x10411, 0x10412, 0x10413, 0x10414, 0x10415, 0x10416, 0x10417, + 0x10418, 0x10419, 0x1041a, 0x1041b, 0x1041c, 0x1041d, 0x1041e, 0x1041f, + 0x10420, 0x10421, 0x10422, 0x10423, 0x10424, 0x10425, 0x10426, 0x10427, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000 + }, + { /* page 471, index 25 */ + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, + 0x0009, 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, + 0x0007, 0x0008, 0x0009, 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, + 0x0006, 0x0007, 0x0008, 0x0009 + } +}; + +/* U+0000 through U+2FAFF */ +static const gint16 attr_table_part1[763] = { + 0 /* page 0 */, + 1 /* page 1 */, + 2 /* page 2 */, + 3 /* page 3 */, + 4 /* page 4 */, + 5 /* page 5 */, + 6 /* page 6 */, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 7 /* page 9 */, + 8 /* page 10 */, + 9 /* page 11 */, + 10 /* page 12 */, + 11 /* page 13 */, + 12 /* page 14 */, + 13 /* page 15 */, + 14 /* page 16 */, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 15 /* page 19 */, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 16 /* page 23 */, + 17 /* page 24 */, + 18 /* page 25 */, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 19 /* page 30 */, + 20 /* page 31 */, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 21 /* page 33 */, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 22 /* page 251 */, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 23 /* page 255 */, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 24 /* page 260 */, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 25 /* page 471 */, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX +}; + +/* U+E0000 through U+10FFFF */ +static const gint16 attr_table_part2[768] = { + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX, + 0x0000 + G_UNICODE_MAX_TABLE_INDEX +}; + +static const gunichar title_table[][3] = { + { 0x01c5, 0x01c4, 0x01c6 }, + { 0x01c8, 0x01c7, 0x01c9 }, + { 0x01cb, 0x01ca, 0x01cc }, + { 0x01f2, 0x01f1, 0x01f3 }, + { 0x1f88, 0x0000, 0x1f80 }, + { 0x1f89, 0x0000, 0x1f81 }, + { 0x1f8a, 0x0000, 0x1f82 }, + { 0x1f8b, 0x0000, 0x1f83 }, + { 0x1f8c, 0x0000, 0x1f84 }, + { 0x1f8d, 0x0000, 0x1f85 }, + { 0x1f8e, 0x0000, 0x1f86 }, + { 0x1f8f, 0x0000, 0x1f87 }, + { 0x1f98, 0x0000, 0x1f90 }, + { 0x1f99, 0x0000, 0x1f91 }, + { 0x1f9a, 0x0000, 0x1f92 }, + { 0x1f9b, 0x0000, 0x1f93 }, + { 0x1f9c, 0x0000, 0x1f94 }, + { 0x1f9d, 0x0000, 0x1f95 }, + { 0x1f9e, 0x0000, 0x1f96 }, + { 0x1f9f, 0x0000, 0x1f97 }, + { 0x1fa8, 0x0000, 0x1fa0 }, + { 0x1fa9, 0x0000, 0x1fa1 }, + { 0x1faa, 0x0000, 0x1fa2 }, + { 0x1fab, 0x0000, 0x1fa3 }, + { 0x1fac, 0x0000, 0x1fa4 }, + { 0x1fad, 0x0000, 0x1fa5 }, + { 0x1fae, 0x0000, 0x1fa6 }, + { 0x1faf, 0x0000, 0x1fa7 }, + { 0x1fbc, 0x0000, 0x1fb3 }, + { 0x1fcc, 0x0000, 0x1fc3 }, + { 0x1ffc, 0x0000, 0x1ff3 } +}; + + +/* Table of special cases for case conversion; each record contains + * First, the best single character mapping to lowercase if Lu, + * and to uppercase if Ll, followed by the output mapping for the two cases + * other than the case of the codepoint, in the order [Ll],[Lu],[Lt], + * encoded in UTF-8, separated and terminated by a null character. + */ +static const gchar special_case_table[] = { + "\x00\x53\x53\x00\x53\x73\0" /* offset 0 */ + "\x69\x69\xcc\x87\x00\xc4\xb0\0" /* offset 7 */ + "\x00\x46\x46\x00\x46\x66\0" /* offset 15 */ + "\x00\x46\x49\x00\x46\x69\0" /* offset 22 */ + "\x00\x46\x4c\x00\x46\x6c\0" /* offset 29 */ + "\x00\x46\x46\x49\x00\x46\x66\x69\0" /* offset 36 */ + "\x00\x46\x46\x4c\x00\x46\x66\x6c\0" /* offset 45 */ + "\x00\x53\x54\x00\x53\x74\0" /* offset 54 */ + "\x00\x53\x54\x00\x53\x74\0" /* offset 61 */ + "\x00\xd4\xb5\xd5\x92\x00\xd4\xb5\xd6\x82\0" /* offset 68 */ + "\x00\xd5\x84\xd5\x86\x00\xd5\x84\xd5\xb6\0" /* offset 79 */ + "\x00\xd5\x84\xd4\xb5\x00\xd5\x84\xd5\xa5\0" /* offset 90 */ + "\x00\xd5\x84\xd4\xbb\x00\xd5\x84\xd5\xab\0" /* offset 101 */ + "\x00\xd5\x8e\xd5\x86\x00\xd5\x8e\xd5\xb6\0" /* offset 112 */ + "\x00\xd5\x84\xd4\xbd\x00\xd5\x84\xd5\xad\0" /* offset 123 */ + "\x00\xca\xbc\x4e\x00\xca\xbc\x4e\0" /* offset 134 */ + "\x00\xce\x99\xcc\x88\xcc\x81\x00\xce\x99\xcc\x88\xcc\x81\0" /* offset 143 */ + "\x00\xce\xa5\xcc\x88\xcc\x81\x00\xce\xa5\xcc\x88\xcc\x81\0" /* offset 158 */ + "\x00\x4a\xcc\x8c\x00\x4a\xcc\x8c\0" /* offset 173 */ + "\x00\x48\xcc\xb1\x00\x48\xcc\xb1\0" /* offset 182 */ + "\x00\x54\xcc\x88\x00\x54\xcc\x88\0" /* offset 191 */ + "\x00\x57\xcc\x8a\x00\x57\xcc\x8a\0" /* offset 200 */ + "\x00\x59\xcc\x8a\x00\x59\xcc\x8a\0" /* offset 209 */ + "\x00\x41\xca\xbe\x00\x41\xca\xbe\0" /* offset 218 */ + "\x00\xce\xa5\xcc\x93\x00\xce\xa5\xcc\x93\0" /* offset 227 */ + "\x00\xce\xa5\xcc\x93\xcc\x80\x00\xce\xa5\xcc\x93\xcc\x80\0" /* offset 238 */ + "\x00\xce\xa5\xcc\x93\xcc\x81\x00\xce\xa5\xcc\x93\xcc\x81\0" /* offset 253 */ + "\x00\xce\xa5\xcc\x93\xcd\x82\x00\xce\xa5\xcc\x93\xcd\x82\0" /* offset 268 */ + "\x00\xce\x91\xcd\x82\x00\xce\x91\xcd\x82\0" /* offset 283 */ + "\x00\xce\x97\xcd\x82\x00\xce\x97\xcd\x82\0" /* offset 294 */ + "\x00\xce\x99\xcc\x88\xcc\x80\x00\xce\x99\xcc\x88\xcc\x80\0" /* offset 305 */ + "\x00\xce\x99\xcc\x88\xcc\x81\x00\xce\x99\xcc\x88\xcc\x81\0" /* offset 320 */ + "\x00\xce\x99\xcd\x82\x00\xce\x99\xcd\x82\0" /* offset 335 */ + "\x00\xce\x99\xcc\x88\xcd\x82\x00\xce\x99\xcc\x88\xcd\x82\0" /* offset 346 */ + "\x00\xce\xa5\xcc\x88\xcc\x80\x00\xce\xa5\xcc\x88\xcc\x80\0" /* offset 361 */ + "\x00\xce\xa5\xcc\x88\xcc\x81\x00\xce\xa5\xcc\x88\xcc\x81\0" /* offset 376 */ + "\x00\xce\xa1\xcc\x93\x00\xce\xa1\xcc\x93\0" /* offset 391 */ + "\x00\xce\xa5\xcd\x82\x00\xce\xa5\xcd\x82\0" /* offset 402 */ + "\x00\xce\xa5\xcc\x88\xcd\x82\x00\xce\xa5\xcc\x88\xcd\x82\0" /* offset 413 */ + "\x00\xce\xa9\xcd\x82\x00\xce\xa9\xcd\x82\0" /* offset 428 */ + "\xe1\xbe\x88\xe1\xbc\x88\xce\x99\x00\xe1\xbe\x88\0" /* offset 439 */ + "\xe1\xbe\x89\xe1\xbc\x89\xce\x99\x00\xe1\xbe\x89\0" /* offset 452 */ + "\xe1\xbe\x8a\xe1\xbc\x8a\xce\x99\x00\xe1\xbe\x8a\0" /* offset 465 */ + "\xe1\xbe\x8b\xe1\xbc\x8b\xce\x99\x00\xe1\xbe\x8b\0" /* offset 478 */ + "\xe1\xbe\x8c\xe1\xbc\x8c\xce\x99\x00\xe1\xbe\x8c\0" /* offset 491 */ + "\xe1\xbe\x8d\xe1\xbc\x8d\xce\x99\x00\xe1\xbe\x8d\0" /* offset 504 */ + "\xe1\xbe\x8e\xe1\xbc\x8e\xce\x99\x00\xe1\xbe\x8e\0" /* offset 517 */ + "\xe1\xbe\x8f\xe1\xbc\x8f\xce\x99\x00\xe1\xbe\x8f\0" /* offset 530 */ + "\xe1\xbe\x80\x00\xe1\xbc\x88\xce\x99\0" /* offset 543 */ + "\xe1\xbe\x81\x00\xe1\xbc\x89\xce\x99\0" /* offset 553 */ + "\xe1\xbe\x82\x00\xe1\xbc\x8a\xce\x99\0" /* offset 563 */ + "\xe1\xbe\x83\x00\xe1\xbc\x8b\xce\x99\0" /* offset 573 */ + "\xe1\xbe\x84\x00\xe1\xbc\x8c\xce\x99\0" /* offset 583 */ + "\xe1\xbe\x85\x00\xe1\xbc\x8d\xce\x99\0" /* offset 593 */ + "\xe1\xbe\x86\x00\xe1\xbc\x8e\xce\x99\0" /* offset 603 */ + "\xe1\xbe\x87\x00\xe1\xbc\x8f\xce\x99\0" /* offset 613 */ + "\xe1\xbe\x98\xe1\xbc\xa8\xce\x99\x00\xe1\xbe\x98\0" /* offset 623 */ + "\xe1\xbe\x99\xe1\xbc\xa9\xce\x99\x00\xe1\xbe\x99\0" /* offset 636 */ + "\xe1\xbe\x9a\xe1\xbc\xaa\xce\x99\x00\xe1\xbe\x9a\0" /* offset 649 */ + "\xe1\xbe\x9b\xe1\xbc\xab\xce\x99\x00\xe1\xbe\x9b\0" /* offset 662 */ + "\xe1\xbe\x9c\xe1\xbc\xac\xce\x99\x00\xe1\xbe\x9c\0" /* offset 675 */ + "\xe1\xbe\x9d\xe1\xbc\xad\xce\x99\x00\xe1\xbe\x9d\0" /* offset 688 */ + "\xe1\xbe\x9e\xe1\xbc\xae\xce\x99\x00\xe1\xbe\x9e\0" /* offset 701 */ + "\xe1\xbe\x9f\xe1\xbc\xaf\xce\x99\x00\xe1\xbe\x9f\0" /* offset 714 */ + "\xe1\xbe\x90\x00\xe1\xbc\xa8\xce\x99\0" /* offset 727 */ + "\xe1\xbe\x91\x00\xe1\xbc\xa9\xce\x99\0" /* offset 737 */ + "\xe1\xbe\x92\x00\xe1\xbc\xaa\xce\x99\0" /* offset 747 */ + "\xe1\xbe\x93\x00\xe1\xbc\xab\xce\x99\0" /* offset 757 */ + "\xe1\xbe\x94\x00\xe1\xbc\xac\xce\x99\0" /* offset 767 */ + "\xe1\xbe\x95\x00\xe1\xbc\xad\xce\x99\0" /* offset 777 */ + "\xe1\xbe\x96\x00\xe1\xbc\xae\xce\x99\0" /* offset 787 */ + "\xe1\xbe\x97\x00\xe1\xbc\xaf\xce\x99\0" /* offset 797 */ + "\xe1\xbe\xa8\xe1\xbd\xa8\xce\x99\x00\xe1\xbe\xa8\0" /* offset 807 */ + "\xe1\xbe\xa9\xe1\xbd\xa9\xce\x99\x00\xe1\xbe\xa9\0" /* offset 820 */ + "\xe1\xbe\xaa\xe1\xbd\xaa\xce\x99\x00\xe1\xbe\xaa\0" /* offset 833 */ + "\xe1\xbe\xab\xe1\xbd\xab\xce\x99\x00\xe1\xbe\xab\0" /* offset 846 */ + "\xe1\xbe\xac\xe1\xbd\xac\xce\x99\x00\xe1\xbe\xac\0" /* offset 859 */ + "\xe1\xbe\xad\xe1\xbd\xad\xce\x99\x00\xe1\xbe\xad\0" /* offset 872 */ + "\xe1\xbe\xae\xe1\xbd\xae\xce\x99\x00\xe1\xbe\xae\0" /* offset 885 */ + "\xe1\xbe\xaf\xe1\xbd\xaf\xce\x99\x00\xe1\xbe\xaf\0" /* offset 898 */ + "\xe1\xbe\xa0\x00\xe1\xbd\xa8\xce\x99\0" /* offset 911 */ + "\xe1\xbe\xa1\x00\xe1\xbd\xa9\xce\x99\0" /* offset 921 */ + "\xe1\xbe\xa2\x00\xe1\xbd\xaa\xce\x99\0" /* offset 931 */ + "\xe1\xbe\xa3\x00\xe1\xbd\xab\xce\x99\0" /* offset 941 */ + "\xe1\xbe\xa4\x00\xe1\xbd\xac\xce\x99\0" /* offset 951 */ + "\xe1\xbe\xa5\x00\xe1\xbd\xad\xce\x99\0" /* offset 961 */ + "\xe1\xbe\xa6\x00\xe1\xbd\xae\xce\x99\0" /* offset 971 */ + "\xe1\xbe\xa7\x00\xe1\xbd\xaf\xce\x99\0" /* offset 981 */ + "\xe1\xbe\xbc\xce\x91\xce\x99\x00\xe1\xbe\xbc\0" /* offset 991 */ + "\xe1\xbe\xb3\x00\xce\x91\xce\x99\0" /* offset 1003 */ + "\xe1\xbf\x8c\xce\x97\xce\x99\x00\xe1\xbf\x8c\0" /* offset 1012 */ + "\xe1\xbf\x83\x00\xce\x97\xce\x99\0" /* offset 1024 */ + "\xe1\xbf\xbc\xce\xa9\xce\x99\x00\xe1\xbf\xbc\0" /* offset 1033 */ + "\xe1\xbf\xb3\x00\xce\xa9\xce\x99\0" /* offset 1045 */ + "\x00\xe1\xbe\xba\xce\x99\x00\xe1\xbe\xba\xcd\x85\0" /* offset 1054 */ + "\x00\xce\x86\xce\x99\x00\xce\x86\xcd\x85\0" /* offset 1067 */ + "\x00\xe1\xbf\x8a\xce\x99\x00\xe1\xbf\x8a\xcd\x85\0" /* offset 1078 */ + "\x00\xce\x89\xce\x99\x00\xce\x89\xcd\x85\0" /* offset 1091 */ + "\x00\xe1\xbf\xba\xce\x99\x00\xe1\xbf\xba\xcd\x85\0" /* offset 1102 */ + "\x00\xce\x8f\xce\x99\x00\xce\x8f\xcd\x85\0" /* offset 1115 */ + "\x00\xce\x91\xcd\x82\xce\x99\x00\xce\x91\xcd\x82\xcd\x85\0" /* offset 1126 */ + "\x00\xce\x97\xcd\x82\xce\x99\x00\xce\x97\xcd\x82\xcd\x85\0" /* offset 1141 */ + "\x00\xce\xa9\xcd\x82\xce\x99\x00\xce\xa9\xcd\x82\xcd\x85\0" /* offset 1156 */ +}; + + +/* Table of casefolding cases that can't be derived by lowercasing + */ +static const struct { + guint16 ch; + gchar data[7]; +} casefold_table[] = { + { 0x00b5, "\xce\xbc" }, + { 0x00df, "\x73\x73" }, + { 0x0130, "\x69\xcc\x87" }, + { 0x0149, "\xca\xbc\x6e" }, + { 0x017f, "\x73" }, + { 0x01f0, "\x6a\xcc\x8c" }, + { 0x0345, "\xce\xb9" }, + { 0x0390, "\xce\xb9\xcc\x88\xcc\x81" }, + { 0x03b0, "\xcf\x85\xcc\x88\xcc\x81" }, + { 0x03c2, "\xcf\x83" }, + { 0x03d0, "\xce\xb2" }, + { 0x03d1, "\xce\xb8" }, + { 0x03d5, "\xcf\x86" }, + { 0x03d6, "\xcf\x80" }, + { 0x03f0, "\xce\xba" }, + { 0x03f1, "\xcf\x81" }, + { 0x03f5, "\xce\xb5" }, + { 0x0587, "\xd5\xa5\xd6\x82" }, + { 0x1e96, "\x68\xcc\xb1" }, + { 0x1e97, "\x74\xcc\x88" }, + { 0x1e98, "\x77\xcc\x8a" }, + { 0x1e99, "\x79\xcc\x8a" }, + { 0x1e9a, "\x61\xca\xbe" }, + { 0x1e9b, "\xe1\xb9\xa1" }, + { 0x1f50, "\xcf\x85\xcc\x93" }, + { 0x1f52, "\xcf\x85\xcc\x93\xcc\x80" }, + { 0x1f54, "\xcf\x85\xcc\x93\xcc\x81" }, + { 0x1f56, "\xcf\x85\xcc\x93\xcd\x82" }, + { 0x1f80, "\xe1\xbc\x80\xce\xb9" }, + { 0x1f81, "\xe1\xbc\x81\xce\xb9" }, + { 0x1f82, "\xe1\xbc\x82\xce\xb9" }, + { 0x1f83, "\xe1\xbc\x83\xce\xb9" }, + { 0x1f84, "\xe1\xbc\x84\xce\xb9" }, + { 0x1f85, "\xe1\xbc\x85\xce\xb9" }, + { 0x1f86, "\xe1\xbc\x86\xce\xb9" }, + { 0x1f87, "\xe1\xbc\x87\xce\xb9" }, + { 0x1f88, "\xe1\xbc\x80\xce\xb9" }, + { 0x1f89, "\xe1\xbc\x81\xce\xb9" }, + { 0x1f8a, "\xe1\xbc\x82\xce\xb9" }, + { 0x1f8b, "\xe1\xbc\x83\xce\xb9" }, + { 0x1f8c, "\xe1\xbc\x84\xce\xb9" }, + { 0x1f8d, "\xe1\xbc\x85\xce\xb9" }, + { 0x1f8e, "\xe1\xbc\x86\xce\xb9" }, + { 0x1f8f, "\xe1\xbc\x87\xce\xb9" }, + { 0x1f90, "\xe1\xbc\xa0\xce\xb9" }, + { 0x1f91, "\xe1\xbc\xa1\xce\xb9" }, + { 0x1f92, "\xe1\xbc\xa2\xce\xb9" }, + { 0x1f93, "\xe1\xbc\xa3\xce\xb9" }, + { 0x1f94, "\xe1\xbc\xa4\xce\xb9" }, + { 0x1f95, "\xe1\xbc\xa5\xce\xb9" }, + { 0x1f96, "\xe1\xbc\xa6\xce\xb9" }, + { 0x1f97, "\xe1\xbc\xa7\xce\xb9" }, + { 0x1f98, "\xe1\xbc\xa0\xce\xb9" }, + { 0x1f99, "\xe1\xbc\xa1\xce\xb9" }, + { 0x1f9a, "\xe1\xbc\xa2\xce\xb9" }, + { 0x1f9b, "\xe1\xbc\xa3\xce\xb9" }, + { 0x1f9c, "\xe1\xbc\xa4\xce\xb9" }, + { 0x1f9d, "\xe1\xbc\xa5\xce\xb9" }, + { 0x1f9e, "\xe1\xbc\xa6\xce\xb9" }, + { 0x1f9f, "\xe1\xbc\xa7\xce\xb9" }, + { 0x1fa0, "\xe1\xbd\xa0\xce\xb9" }, + { 0x1fa1, "\xe1\xbd\xa1\xce\xb9" }, + { 0x1fa2, "\xe1\xbd\xa2\xce\xb9" }, + { 0x1fa3, "\xe1\xbd\xa3\xce\xb9" }, + { 0x1fa4, "\xe1\xbd\xa4\xce\xb9" }, + { 0x1fa5, "\xe1\xbd\xa5\xce\xb9" }, + { 0x1fa6, "\xe1\xbd\xa6\xce\xb9" }, + { 0x1fa7, "\xe1\xbd\xa7\xce\xb9" }, + { 0x1fa8, "\xe1\xbd\xa0\xce\xb9" }, + { 0x1fa9, "\xe1\xbd\xa1\xce\xb9" }, + { 0x1faa, "\xe1\xbd\xa2\xce\xb9" }, + { 0x1fab, "\xe1\xbd\xa3\xce\xb9" }, + { 0x1fac, "\xe1\xbd\xa4\xce\xb9" }, + { 0x1fad, "\xe1\xbd\xa5\xce\xb9" }, + { 0x1fae, "\xe1\xbd\xa6\xce\xb9" }, + { 0x1faf, "\xe1\xbd\xa7\xce\xb9" }, + { 0x1fb2, "\xe1\xbd\xb0\xce\xb9" }, + { 0x1fb3, "\xce\xb1\xce\xb9" }, + { 0x1fb4, "\xce\xac\xce\xb9" }, + { 0x1fb6, "\xce\xb1\xcd\x82" }, + { 0x1fb7, "\xce\xb1\xcd\x82\xce\xb9" }, + { 0x1fbc, "\xce\xb1\xce\xb9" }, + { 0x1fbe, "\xce\xb9" }, + { 0x1fc2, "\xe1\xbd\xb4\xce\xb9" }, + { 0x1fc3, "\xce\xb7\xce\xb9" }, + { 0x1fc4, "\xce\xae\xce\xb9" }, + { 0x1fc6, "\xce\xb7\xcd\x82" }, + { 0x1fc7, "\xce\xb7\xcd\x82\xce\xb9" }, + { 0x1fcc, "\xce\xb7\xce\xb9" }, + { 0x1fd2, "\xce\xb9\xcc\x88\xcc\x80" }, + { 0x1fd3, "\xce\xb9\xcc\x88\xcc\x81" }, + { 0x1fd6, "\xce\xb9\xcd\x82" }, + { 0x1fd7, "\xce\xb9\xcc\x88\xcd\x82" }, + { 0x1fe2, "\xcf\x85\xcc\x88\xcc\x80" }, + { 0x1fe3, "\xcf\x85\xcc\x88\xcc\x81" }, + { 0x1fe4, "\xcf\x81\xcc\x93" }, + { 0x1fe6, "\xcf\x85\xcd\x82" }, + { 0x1fe7, "\xcf\x85\xcc\x88\xcd\x82" }, + { 0x1ff2, "\xe1\xbd\xbc\xce\xb9" }, + { 0x1ff3, "\xcf\x89\xce\xb9" }, + { 0x1ff4, "\xcf\x8e\xce\xb9" }, + { 0x1ff6, "\xcf\x89\xcd\x82" }, + { 0x1ff7, "\xcf\x89\xcd\x82\xce\xb9" }, + { 0x1ffc, "\xcf\x89\xce\xb9" }, + { 0x2160, "\xe2\x85\xb0" }, + { 0x2161, "\xe2\x85\xb1" }, + { 0x2162, "\xe2\x85\xb2" }, + { 0x2163, "\xe2\x85\xb3" }, + { 0x2164, "\xe2\x85\xb4" }, + { 0x2165, "\xe2\x85\xb5" }, + { 0x2166, "\xe2\x85\xb6" }, + { 0x2167, "\xe2\x85\xb7" }, + { 0x2168, "\xe2\x85\xb8" }, + { 0x2169, "\xe2\x85\xb9" }, + { 0x216a, "\xe2\x85\xba" }, + { 0x216b, "\xe2\x85\xbb" }, + { 0x216c, "\xe2\x85\xbc" }, + { 0x216d, "\xe2\x85\xbd" }, + { 0x216e, "\xe2\x85\xbe" }, + { 0x216f, "\xe2\x85\xbf" }, + { 0x24b6, "\xe2\x93\x90" }, + { 0x24b7, "\xe2\x93\x91" }, + { 0x24b8, "\xe2\x93\x92" }, + { 0x24b9, "\xe2\x93\x93" }, + { 0x24ba, "\xe2\x93\x94" }, + { 0x24bb, "\xe2\x93\x95" }, + { 0x24bc, "\xe2\x93\x96" }, + { 0x24bd, "\xe2\x93\x97" }, + { 0x24be, "\xe2\x93\x98" }, + { 0x24bf, "\xe2\x93\x99" }, + { 0x24c0, "\xe2\x93\x9a" }, + { 0x24c1, "\xe2\x93\x9b" }, + { 0x24c2, "\xe2\x93\x9c" }, + { 0x24c3, "\xe2\x93\x9d" }, + { 0x24c4, "\xe2\x93\x9e" }, + { 0x24c5, "\xe2\x93\x9f" }, + { 0x24c6, "\xe2\x93\xa0" }, + { 0x24c7, "\xe2\x93\xa1" }, + { 0x24c8, "\xe2\x93\xa2" }, + { 0x24c9, "\xe2\x93\xa3" }, + { 0x24ca, "\xe2\x93\xa4" }, + { 0x24cb, "\xe2\x93\xa5" }, + { 0x24cc, "\xe2\x93\xa6" }, + { 0x24cd, "\xe2\x93\xa7" }, + { 0x24ce, "\xe2\x93\xa8" }, + { 0x24cf, "\xe2\x93\xa9" }, + { 0xfb00, "\x66\x66" }, + { 0xfb01, "\x66\x69" }, + { 0xfb02, "\x66\x6c" }, + { 0xfb03, "\x66\x66\x69" }, + { 0xfb04, "\x66\x66\x6c" }, + { 0xfb05, "\x73\x74" }, + { 0xfb06, "\x73\x74" }, + { 0xfb13, "\xd5\xb4\xd5\xb6" }, + { 0xfb14, "\xd5\xb4\xd5\xa5" }, + { 0xfb15, "\xd5\xb4\xd5\xab" }, + { 0xfb16, "\xd5\xbe\xd5\xb6" }, + { 0xfb17, "\xd5\xb4\xd5\xad" }, +}; + +static const struct { + gunichar ch; + gunichar mirrored_ch; +} bidi_mirroring_table[] = +{ + { 0x0028, 0x0029 }, + { 0x0029, 0x0028 }, + { 0x003c, 0x003e }, + { 0x003e, 0x003c }, + { 0x005b, 0x005d }, + { 0x005d, 0x005b }, + { 0x007b, 0x007d }, + { 0x007d, 0x007b }, + { 0x00ab, 0x00bb }, + { 0x00bb, 0x00ab }, + { 0x2039, 0x203a }, + { 0x203a, 0x2039 }, + { 0x2045, 0x2046 }, + { 0x2046, 0x2045 }, + { 0x207d, 0x207e }, + { 0x207e, 0x207d }, + { 0x208d, 0x208e }, + { 0x208e, 0x208d }, + { 0x2208, 0x220b }, + { 0x2209, 0x220c }, + { 0x220a, 0x220d }, + { 0x220b, 0x2208 }, + { 0x220c, 0x2209 }, + { 0x220d, 0x220a }, + { 0x2215, 0x29f5 }, + { 0x223c, 0x223d }, + { 0x223d, 0x223c }, + { 0x2243, 0x22cd }, + { 0x2252, 0x2253 }, + { 0x2253, 0x2252 }, + { 0x2254, 0x2255 }, + { 0x2255, 0x2254 }, + { 0x2264, 0x2265 }, + { 0x2265, 0x2264 }, + { 0x2266, 0x2267 }, + { 0x2267, 0x2266 }, + { 0x2268, 0x2269 }, + { 0x2269, 0x2268 }, + { 0x226a, 0x226b }, + { 0x226b, 0x226a }, + { 0x226e, 0x226f }, + { 0x226f, 0x226e }, + { 0x2270, 0x2271 }, + { 0x2271, 0x2270 }, + { 0x2272, 0x2273 }, + { 0x2273, 0x2272 }, + { 0x2274, 0x2275 }, + { 0x2275, 0x2274 }, + { 0x2276, 0x2277 }, + { 0x2277, 0x2276 }, + { 0x2278, 0x2279 }, + { 0x2279, 0x2278 }, + { 0x227a, 0x227b }, + { 0x227b, 0x227a }, + { 0x227c, 0x227d }, + { 0x227d, 0x227c }, + { 0x227e, 0x227f }, + { 0x227f, 0x227e }, + { 0x2280, 0x2281 }, + { 0x2281, 0x2280 }, + { 0x2282, 0x2283 }, + { 0x2283, 0x2282 }, + { 0x2284, 0x2285 }, + { 0x2285, 0x2284 }, + { 0x2286, 0x2287 }, + { 0x2287, 0x2286 }, + { 0x2288, 0x2289 }, + { 0x2289, 0x2288 }, + { 0x228a, 0x228b }, + { 0x228b, 0x228a }, + { 0x228f, 0x2290 }, + { 0x2290, 0x228f }, + { 0x2291, 0x2292 }, + { 0x2292, 0x2291 }, + { 0x2298, 0x29b8 }, + { 0x22a2, 0x22a3 }, + { 0x22a3, 0x22a2 }, + { 0x22a6, 0x2ade }, + { 0x22a8, 0x2ae4 }, + { 0x22a9, 0x2ae3 }, + { 0x22ab, 0x2ae5 }, + { 0x22b0, 0x22b1 }, + { 0x22b1, 0x22b0 }, + { 0x22b2, 0x22b3 }, + { 0x22b3, 0x22b2 }, + { 0x22b4, 0x22b5 }, + { 0x22b5, 0x22b4 }, + { 0x22b6, 0x22b7 }, + { 0x22b7, 0x22b6 }, + { 0x22c9, 0x22ca }, + { 0x22ca, 0x22c9 }, + { 0x22cb, 0x22cc }, + { 0x22cc, 0x22cb }, + { 0x22cd, 0x2243 }, + { 0x22d0, 0x22d1 }, + { 0x22d1, 0x22d0 }, + { 0x22d6, 0x22d7 }, + { 0x22d7, 0x22d6 }, + { 0x22d8, 0x22d9 }, + { 0x22d9, 0x22d8 }, + { 0x22da, 0x22db }, + { 0x22db, 0x22da }, + { 0x22dc, 0x22dd }, + { 0x22dd, 0x22dc }, + { 0x22de, 0x22df }, + { 0x22df, 0x22de }, + { 0x22e0, 0x22e1 }, + { 0x22e1, 0x22e0 }, + { 0x22e2, 0x22e3 }, + { 0x22e3, 0x22e2 }, + { 0x22e4, 0x22e5 }, + { 0x22e5, 0x22e4 }, + { 0x22e6, 0x22e7 }, + { 0x22e7, 0x22e6 }, + { 0x22e8, 0x22e9 }, + { 0x22e9, 0x22e8 }, + { 0x22ea, 0x22eb }, + { 0x22eb, 0x22ea }, + { 0x22ec, 0x22ed }, + { 0x22ed, 0x22ec }, + { 0x22f0, 0x22f1 }, + { 0x22f1, 0x22f0 }, + { 0x22f2, 0x22fa }, + { 0x22f3, 0x22fb }, + { 0x22f4, 0x22fc }, + { 0x22f6, 0x22fd }, + { 0x22f7, 0x22fe }, + { 0x22fa, 0x22f2 }, + { 0x22fb, 0x22f3 }, + { 0x22fc, 0x22f4 }, + { 0x22fd, 0x22f6 }, + { 0x22fe, 0x22f7 }, + { 0x2308, 0x2309 }, + { 0x2309, 0x2308 }, + { 0x230a, 0x230b }, + { 0x230b, 0x230a }, + { 0x2329, 0x232a }, + { 0x232a, 0x2329 }, + { 0x2768, 0x2769 }, + { 0x2769, 0x2768 }, + { 0x276a, 0x276b }, + { 0x276b, 0x276a }, + { 0x276c, 0x276d }, + { 0x276d, 0x276c }, + { 0x276e, 0x276f }, + { 0x276f, 0x276e }, + { 0x2770, 0x2771 }, + { 0x2771, 0x2770 }, + { 0x2772, 0x2773 }, + { 0x2773, 0x2772 }, + { 0x2774, 0x2775 }, + { 0x2775, 0x2774 }, + { 0x27d5, 0x27d6 }, + { 0x27d6, 0x27d5 }, + { 0x27dd, 0x27de }, + { 0x27de, 0x27dd }, + { 0x27e2, 0x27e3 }, + { 0x27e3, 0x27e2 }, + { 0x27e4, 0x27e5 }, + { 0x27e5, 0x27e4 }, + { 0x27e6, 0x27e7 }, + { 0x27e7, 0x27e6 }, + { 0x27e8, 0x27e9 }, + { 0x27e9, 0x27e8 }, + { 0x27ea, 0x27eb }, + { 0x27eb, 0x27ea }, + { 0x2983, 0x2984 }, + { 0x2984, 0x2983 }, + { 0x2985, 0x2986 }, + { 0x2986, 0x2985 }, + { 0x2987, 0x2988 }, + { 0x2988, 0x2987 }, + { 0x2989, 0x298a }, + { 0x298a, 0x2989 }, + { 0x298b, 0x298c }, + { 0x298c, 0x298b }, + { 0x298d, 0x2990 }, + { 0x298e, 0x298f }, + { 0x298f, 0x298e }, + { 0x2990, 0x298d }, + { 0x2991, 0x2992 }, + { 0x2992, 0x2991 }, + { 0x2993, 0x2994 }, + { 0x2994, 0x2993 }, + { 0x2995, 0x2996 }, + { 0x2996, 0x2995 }, + { 0x2997, 0x2998 }, + { 0x2998, 0x2997 }, + { 0x29b8, 0x2298 }, + { 0x29c0, 0x29c1 }, + { 0x29c1, 0x29c0 }, + { 0x29c4, 0x29c5 }, + { 0x29c5, 0x29c4 }, + { 0x29cf, 0x29d0 }, + { 0x29d0, 0x29cf }, + { 0x29d1, 0x29d2 }, + { 0x29d2, 0x29d1 }, + { 0x29d4, 0x29d5 }, + { 0x29d5, 0x29d4 }, + { 0x29d8, 0x29d9 }, + { 0x29d9, 0x29d8 }, + { 0x29da, 0x29db }, + { 0x29db, 0x29da }, + { 0x29f5, 0x2215 }, + { 0x29f8, 0x29f9 }, + { 0x29f9, 0x29f8 }, + { 0x29fc, 0x29fd }, + { 0x29fd, 0x29fc }, + { 0x2a2b, 0x2a2c }, + { 0x2a2c, 0x2a2b }, + { 0x2a2d, 0x2a2c }, + { 0x2a2e, 0x2a2d }, + { 0x2a34, 0x2a35 }, + { 0x2a35, 0x2a34 }, + { 0x2a3c, 0x2a3d }, + { 0x2a3d, 0x2a3c }, + { 0x2a64, 0x2a65 }, + { 0x2a65, 0x2a64 }, + { 0x2a79, 0x2a7a }, + { 0x2a7a, 0x2a79 }, + { 0x2a7d, 0x2a7e }, + { 0x2a7e, 0x2a7d }, + { 0x2a7f, 0x2a80 }, + { 0x2a80, 0x2a7f }, + { 0x2a81, 0x2a82 }, + { 0x2a82, 0x2a81 }, + { 0x2a83, 0x2a84 }, + { 0x2a84, 0x2a83 }, + { 0x2a8b, 0x2a8c }, + { 0x2a8c, 0x2a8b }, + { 0x2a91, 0x2a92 }, + { 0x2a92, 0x2a91 }, + { 0x2a93, 0x2a94 }, + { 0x2a94, 0x2a93 }, + { 0x2a95, 0x2a96 }, + { 0x2a96, 0x2a95 }, + { 0x2a97, 0x2a98 }, + { 0x2a98, 0x2a97 }, + { 0x2a99, 0x2a9a }, + { 0x2a9a, 0x2a99 }, + { 0x2a9b, 0x2a9c }, + { 0x2a9c, 0x2a9b }, + { 0x2aa1, 0x2aa2 }, + { 0x2aa2, 0x2aa1 }, + { 0x2aa6, 0x2aa7 }, + { 0x2aa7, 0x2aa6 }, + { 0x2aa8, 0x2aa9 }, + { 0x2aa9, 0x2aa8 }, + { 0x2aaa, 0x2aab }, + { 0x2aab, 0x2aaa }, + { 0x2aac, 0x2aad }, + { 0x2aad, 0x2aac }, + { 0x2aaf, 0x2ab0 }, + { 0x2ab0, 0x2aaf }, + { 0x2ab3, 0x2ab4 }, + { 0x2ab4, 0x2ab3 }, + { 0x2abb, 0x2abc }, + { 0x2abc, 0x2abb }, + { 0x2abd, 0x2abe }, + { 0x2abe, 0x2abd }, + { 0x2abf, 0x2ac0 }, + { 0x2ac0, 0x2abf }, + { 0x2ac1, 0x2ac2 }, + { 0x2ac2, 0x2ac1 }, + { 0x2ac3, 0x2ac4 }, + { 0x2ac4, 0x2ac3 }, + { 0x2ac5, 0x2ac6 }, + { 0x2ac6, 0x2ac5 }, + { 0x2acd, 0x2ace }, + { 0x2ace, 0x2acd }, + { 0x2acf, 0x2ad0 }, + { 0x2ad0, 0x2acf }, + { 0x2ad1, 0x2ad2 }, + { 0x2ad2, 0x2ad1 }, + { 0x2ad3, 0x2ad4 }, + { 0x2ad4, 0x2ad3 }, + { 0x2ad5, 0x2ad6 }, + { 0x2ad6, 0x2ad5 }, + { 0x2ade, 0x22a6 }, + { 0x2ae3, 0x22a9 }, + { 0x2ae4, 0x22a8 }, + { 0x2ae5, 0x22ab }, + { 0x2aec, 0x2aed }, + { 0x2aed, 0x2aec }, + { 0x2af7, 0x2af8 }, + { 0x2af8, 0x2af7 }, + { 0x2af9, 0x2afa }, + { 0x2afa, 0x2af9 }, + { 0x3008, 0x3009 }, + { 0x3009, 0x3008 }, + { 0x300a, 0x300b }, + { 0x300b, 0x300a }, + { 0x300c, 0x300d }, + { 0x300d, 0x300c }, + { 0x300e, 0x300f }, + { 0x300f, 0x300e }, + { 0x3010, 0x3011 }, + { 0x3011, 0x3010 }, + { 0x3014, 0x3015 }, + { 0x3015, 0x3014 }, + { 0x3016, 0x3017 }, + { 0x3017, 0x3016 }, + { 0x3018, 0x3019 }, + { 0x3019, 0x3018 }, + { 0x301a, 0x301b }, + { 0x301b, 0x301a }, + { 0xff08, 0xff09 }, + { 0xff09, 0xff08 }, + { 0xff1c, 0xff1e }, + { 0xff1e, 0xff1c }, + { 0xff3b, 0xff3d }, + { 0xff3d, 0xff3b }, + { 0xff5b, 0xff5d }, + { 0xff5d, 0xff5b }, + { 0xff5f, 0xff60 }, + { 0xff60, 0xff5f }, + { 0xff62, 0xff63 }, + { 0xff63, 0xff62 } +}; + +#endif /* CHARTABLES_H */ diff --git a/src/assistant/3rdparty/clucene/src/CLucene/config/repl_lltot.cpp b/src/assistant/3rdparty/clucene/src/CLucene/config/repl_lltot.cpp new file mode 100644 index 000000000..05a63b887 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/config/repl_lltot.cpp @@ -0,0 +1,47 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#include "CLucene/StdHeader.h" + +TCHAR* lucene_i64tot( + int64_t value, /* [I] Value to be converted */ + TCHAR* str, /* [O] Destination for the converted value */ + int radix) /* [I] Number base for conversion */ +{ + uint64_t val; + int negative; + TCHAR buffer[65]; + TCHAR* pos; + int digit; + + if (value < 0 && radix == 10) { + negative = 1; + val = -value; + } else { + negative = 0; + val = value; + } /* if */ + + pos = &buffer[64]; + *pos = '\0'; + + do { + digit = val % radix; + val = val / radix; + if (digit < 10) { + *--pos = '0' + digit; + } else { + *--pos = 'a' + digit - 10; + } /* if */ + } while (val != 0L); + + if (negative) { + *--pos = '-'; + } /* if */ + + _tcsncpy(str,pos,&buffer[64] - pos + 1); //needed for unicode to work + return str; +} diff --git a/src/assistant/3rdparty/clucene/src/CLucene/config/repl_tchar.h b/src/assistant/3rdparty/clucene/src/CLucene/config/repl_tchar.h new file mode 100644 index 000000000..ba8aef5c6 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/config/repl_tchar.h @@ -0,0 +1,126 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _CL_HAVE_TCHAR_H +#if defined(_UCS2) + #define TCHAR wchar_t + + //note: descriptions with * in front have replacement functions + + //formatting functions + #define _sntprintf swprintf //* make a formatted a string + #define _tprintf wprintf //* print a formatted string + + //this one has no replacement functions yet, but it is only used in the tests + #define _vsntprintf vsnwprintf //* print a formatted string using variable arguments + + //we are using the internal functions of the compiler here + //if LUCENE_USE_INTERNAL_CHAR_FUNCTIONS is defined, thesse + //will be replaced by internal functions + #define _istalnum iswalnum //* alpha/numeric char check + #define _istalpha iswalpha //* alpha char check + #define _istspace iswspace //* space char check + #define _istdigit iswdigit //* digit char check + #define _totlower towlower //* convert char to lower case + #define _totupper towupper //* convert char to lower case + #define _tcslwr wcslwr //* convert string to lower case + + //these are the string handling functions + //we may need to create wide-character/multi-byte replacements for these + #define _tcscpy wcscpy //copy a string to another string + #define _tcsncpy wcsncpy //copy a specified amount of one string to another string. + #define _tcscat wcscat //copy a string onto the end of the other string + #define _tcschr wcschr //find location of one character + #define _tcsstr wcsstr //find location of a string + #define _tcslen wcslen //get length of a string + #define _tcscmp wcscmp //case sensitive compare two strings + #define _tcsncmp wcsncmp //case sensitive compare two strings + #define _tcscspn wcscspn //location of any of a set of character in a string + + #ifdef _CL_HAVE_WCSICMP + #define _tcsicmp wcsicmp //* case insensitive compare two string + #else + #define _tcsicmp wcscasecmp //* case insensitive compare two string + #endif + + //conversion functions + #define _tcstod wcstod //convert a string to a double + #ifdef _PA_RISC + #define _tcstoi64 __wcstoll //* convers a string to an 64bit bit integer + #else + #define _tcstoi64 wcstoll //* convers a string to an 64bit bit integer + #endif + #define _i64tot lltow //* converts a 64 bit integer to a string (with base) + +#else //if defined(_ASCII) + #define TCHAR char + + //formatting functions + #define _sntprintf snprintf + #define _tprintf printf + #define _vsntprintf vsnprintf + + //we are using the internal functions of the compiler here + //if LUCENE_USE_INTERNAL_CHAR_FUNCTIONS is defined, thesse + //will be replaced by internal functions + #define _istalnum isalnum + #define _istalpha isalpha + #define _istspace isspace + #define _istdigit isdigit + #define _totlower tolower + #define _totupper toupper + #define _tcslwr strlwr + + //these are the string handling functions + #define _tcscpy strcpy + #define _tcsncpy strncpy + #define _tcscat strcat + #define _tcschr strchr + #define _tcsstr strstr + #define _tcslen strlen + #define _tcscmp strcmp + #define _tcsncmp strncmp + #define _tcsicmp strcasecmp + #define _tcscspn strcspn + + //converstion methods + #define _tcstod strtod + #define _tcstoi64 strtoll + #define _i64tot lltoa +#endif +#else //HAVE_TCHAR_H + #include <tchar.h> + +#ifdef UNDER_CE +#include <QString> +#define _i64tot i64tot +inline TCHAR* i64tot(__int64 value, TCHAR* str, int radix) +{ + QT_USE_NAMESPACE + _tcscpy(str, (TCHAR *) QString::number(value, radix).utf16()); + return str; +} + +#define _tcstoi64 tcstoi64 +inline __int64 tcstoi64(const TCHAR *nptr, TCHAR **endptr, int base) +{ + QT_USE_NAMESPACE + bool ok; + return QString::fromUtf16((ushort*) nptr).toInt(&ok, base); +} + +#endif + + //some tchar headers miss these... + #ifndef _tcstoi64 + #if defined(_UCS2) + #define _tcstoi64 wcstoll //* convers a string to an 64bit bit integer + #else + #define _tcstoi64 strtoll + #endif + #endif + +#endif //HAVE_TCHAR_H diff --git a/src/assistant/3rdparty/clucene/src/CLucene/config/repl_tcscasecmp.cpp b/src/assistant/3rdparty/clucene/src/CLucene/config/repl_tcscasecmp.cpp new file mode 100644 index 000000000..1bee7b7a6 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/config/repl_tcscasecmp.cpp @@ -0,0 +1,21 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ + +#include "CLucene/StdHeader.h" + +int lucene_tcscasecmp(const TCHAR * sa, const TCHAR * sb){ + TCHAR ca,cb; + if (sa == sb) + return 0; + + do{ + ca = _totlower( (*(sa++)) ); + cb = _totlower( (*(sb++)) ); + } while ( ca != L'\0' && (ca == cb) ); + + return (int)(ca - cb); +} diff --git a/src/assistant/3rdparty/clucene/src/CLucene/config/repl_tcslwr.cpp b/src/assistant/3rdparty/clucene/src/CLucene/config/repl_tcslwr.cpp new file mode 100644 index 000000000..2ae6abca4 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/config/repl_tcslwr.cpp @@ -0,0 +1,15 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ + +#include "CLucene/StdHeader.h" + +TCHAR* lucene_tcslwr( TCHAR* str ) +{ + TCHAR* ret = str; + for ( ; *str; str++) *str = _totlower(*str); + return ret; +} diff --git a/src/assistant/3rdparty/clucene/src/CLucene/config/repl_tcstod.cpp b/src/assistant/3rdparty/clucene/src/CLucene/config/repl_tcstod.cpp new file mode 100644 index 000000000..1fd4ca770 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/config/repl_tcstod.cpp @@ -0,0 +1,23 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ + +#include "CLucene/StdHeader.h" + +#ifndef _ASCII +double lucene_tcstod(const TCHAR *value, TCHAR **end){ + int32_t len = _tcslen(value)+1; + char* avalue=_CL_NEWARRAY(char,len); + char* aend=NULL; + STRCPY_TtoA(avalue,value,len); + + double ret = strtod(avalue,&aend); + *end=(TCHAR*)value+(aend-avalue); + _CLDELETE_CaARRAY(avalue); + + return ret; +} +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/config/repl_tcstoll.cpp b/src/assistant/3rdparty/clucene/src/CLucene/config/repl_tcstoll.cpp new file mode 100644 index 000000000..246d66c80 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/config/repl_tcstoll.cpp @@ -0,0 +1,46 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ + +#include "CLucene/StdHeader.h" + +int64_t lucene_tcstoi64(const TCHAR* str, TCHAR**end, int radix){ + #define LUCENE_TCSTOI64_RADIX(x,r) ((x>=_T('0') && x<=_T('9'))?x-_T('0'):((x>=_T('a') && x<=_T('z'))?x-_T('a')+10:((x>=_T('A') && x<=_T('Z'))?x-_T('A')+10:1000))) + + if (radix < 2 || radix > 36) + return 0; + + /* Skip white space. */ + while (_istspace (*str)) + ++str; + + int sign=1; + if ( str[0] == _T('+') ) + str++; + else if ( str[0] == _T('-') ){ + sign = -1; + str++; + } + + *end=(TCHAR*)str; + long r = -1; + while ( (r=LUCENE_TCSTOI64_RADIX(*end[0],radix)) >=0 && r<radix ) + (*end)++; + + TCHAR* p = (*end)-1; + int64_t ret = 0; + int pos = 0; + for ( ;p>=str;p-- ){ + int i=LUCENE_TCSTOI64_RADIX(p[0],radix); + if ( pos == 0 ) + ret=i; + else + ret += (int64_t)pow((qreal)radix,(qreal)pos) * i; //todo: might be quicker with a different pow overload + + pos++; + } + return sign*ret; +} diff --git a/src/assistant/3rdparty/clucene/src/CLucene/config/repl_tprintf.cpp b/src/assistant/3rdparty/clucene/src/CLucene/config/repl_tprintf.cpp new file mode 100644 index 000000000..62cecb78b --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/config/repl_tprintf.cpp @@ -0,0 +1,149 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#include "CLucene/StdHeader.h" +#include "CLucene/util/StringBuffer.h" + +#ifdef __CL_INCLUDE_TPRINTF + +CL_NS_USE(util) + +//print a variable argument to a stream +//currently special number formatting is not supported. it is very minimalistic +void lucene_vfnwprintf(StringBuffer* buffer, size_t count, const wchar_t * format, va_list& valist){ + const wchar_t *iter = format; + StringBuffer* tmp = NULL; + if ( buffer == NULL ) + tmp = _CLNEW StringBuffer; + else + tmp = buffer; + + while (*iter) + { + while (*iter && *iter != '%') + { + tmp->appendChar(*iter++); + } + if (*iter == '%') + { + if (iter[1] == '%') + { + //just print a % + tmp->appendChar('%'); + iter += 2; + continue; + } + + iter++; + switch (*iter) + { + case 's': + { + //todo: this is faulty. it doesn't heed count + + //print a string or null + TCHAR *wstr = va_arg(valist, TCHAR *); + if ( !wstr ) + wstr = _T("(null)"); + + tmp->append(wstr); + iter++; + break; + } + + case 'c': + tmp->appendChar((TCHAR)va_arg(valist, int)); + iter++; + break; + + default: + { + //todo: this is faulty. it doesn't heed count + + if (*iter == 'p') + tmp->appendInt((int32_t)va_arg(valist, long)); + else + { + if (*iter == 'a' || *iter == 'A' || + *iter == 'e' || *iter == 'E' || + *iter == 'f' || *iter == 'F' || + *iter == 'g' || *iter == 'G') + tmp->appendFloat((qreal)va_arg(valist, double),8); + else if (*iter == 'd' || *iter == 'i' ){ + tmp->appendInt((int32_t)va_arg(valist, int)); + }else if (*iter == 'l' ){ + TCHAR b[100]; + _i64tot((int64_t)va_arg(valist, int64_t),b,10); + tmp->append(b); + }/*else{ + TCHAR b[100]; + _i64tot((int64_t)va_arg(valist, void*),b,10); + tmp->append(b); + }*/ + } + iter++; + break; + } + } + } + } + + + if ( buffer == NULL ){ + //we are supposed to be writing to the console +#ifdef _UCS2 + TCHAR* pointer = tmp->getBuffer(); + char ob[MB_LEN_MAX]; + size_t v; + size_t len = tmp->length(); + for (size_t i=0;i<len;i++){ + v = wctomb(ob,*pointer); + if ( v > 0 ){ + ob[v]='\0'; + fputs(ob,stdout); + } + pointer++; + } + + +#else + fputs(tmp->getBuffer(),stdout); +#endif + _CLDELETE(tmp); + } +} + +//print a list of arguments to a string +int lucene_snwprintf(wchar_t* strbuf, size_t count, const wchar_t * format, ...){ + va_list ap; + va_start(ap, format); + StringBuffer buffer; + lucene_vfnwprintf(&buffer,count,format,ap); + va_end(ap); + + size_t ret = min(count,(size_t)(buffer.length()+1)); + _tcsncpy(strbuf,buffer.getBuffer(),ret); + return ret; +} + +//print a list of arguments to the stdout +void lucene_wprintf(const wchar_t * format, ...){ + va_list ap; + va_start(ap, format); + lucene_vfnwprintf(NULL,LUCENE_INT32_MAX_SHOULDBE,format,ap); + va_end(ap); +} + +//print a variable argument to a string +int lucene_vsnwprintf(wchar_t * strbuf, size_t count, const wchar_t * format, va_list& ap){ + StringBuffer buffer; + lucene_vfnwprintf(&buffer,count,format,ap); + int ret = min((int32_t)count,buffer.length()+1); + _tcsncpy(strbuf,buffer.getBuffer(),ret); + return ret; +} + +#endif //__CL_INCLUDE_TPRINTF diff --git a/src/assistant/3rdparty/clucene/src/CLucene/config/repl_wchar.h b/src/assistant/3rdparty/clucene/src/CLucene/config/repl_wchar.h new file mode 100644 index 000000000..3e05c311c --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/config/repl_wchar.h @@ -0,0 +1,121 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _lucene_repl_wchar_h +#define _lucene_repl_wchar_h + +#ifdef _UCS2 + +#ifndef _CL_HAVE_WCSCPY + //copy a string to another string + #error wcscpy is not defined, and a licensed replacement has not been written yet +#endif + +#ifndef _CL_HAVE_WCSNCPY + //copy a specified amount of one string to another string. + #error wcsncpy is not defined, and a licensed replacement has not been written yet +#endif + +#ifndef _CL_HAVE_WCSCAT + //copy a string onto the end of the other string + #error wcscat is not defined, and a licensed replacement has not been written yet +#endif + +#ifndef _CL_HAVE_WCSCHR + //find location of one character + #error wcschr is not defined, and a licensed replacement has not been written yet +#endif + +#ifndef _CL_HAVE_WCSSTR + //find location of a string + #error wcspy is not defined, and a licensed replacement has not been written yet +#endif + +#ifndef _CL_HAVE_WCSLEN + //get length of a string + #error wcslen is not defined, and a licensed replacement has not been written yet +#endif + +#ifndef _CL_HAVE_WCSCMP + //case sensitive compare two strings + #error wcscmp is not defined, and a licensed replacement has not been written yet +#endif + +#ifndef _CL_HAVE_WCSNCMP + //case sensitive compare two strings of a specified length + #error wcsncmp is not defined, and a licensed replacement has not been written yet +#endif + +#ifndef _CL_HAVE_WCSCSPN + //Return the length of the maximum initial segment + //of WCS which contains only wide-characters not in REJECT. + #error wcscspn is not defined, and a licensed replacement has not been written yet +#endif + +#endif //_UCS2 + +//string function replacements +#if defined(LUCENE_USE_INTERNAL_CHAR_FUNCTIONS) || (defined(_UCS2) && !defined(_CL_HAVE_WCSCASECMP)) || (defined(_ASCII) && !defined(_CL_HAVE_STRCASECMP)) + int lucene_tcscasecmp(const TCHAR *, const TCHAR *); + #undef _tcsicmp + #define _tcsicmp lucene_tcscasecmp +#endif +#if defined(LUCENE_USE_INTERNAL_CHAR_FUNCTIONS) || (defined(_UCS2) && !defined(_CL_HAVE_WCSLWR)) || (defined(_ASCII) && !defined(_CL_HAVE_STRLWR)) + TCHAR* lucene_tcslwr( TCHAR* str ); + #undef _tcslwr + #define _tcslwr lucene_tcslwr +#endif + +//conversion functions +#if (defined(_ASCII) && !defined(_CL_HAVE_LLTOA)) || (defined(_UCS2) && !defined(_CL_HAVE_LLTOW)) + TCHAR* lucene_i64tot( int64_t value, TCHAR* str, int radix); + #undef _i64tot + #define _i64tot lucene_i64tot +#endif +#if (defined(_UCS2) && !defined(_CL_HAVE_WCSTOLL)) || (defined(_ASCII) && !defined(_CL_HAVE_STRTOLL)) + int64_t lucene_tcstoi64(const TCHAR* str, TCHAR**end, int radix); + #undef _tcstoi64 + #define _tcstoi64 lucene_tcstoi64 +#endif +#if defined(_UCS2) && !defined(_CL_HAVE_WCSTOD) + double lucene_tcstod(const TCHAR *value, TCHAR **end); + #undef _tcstod + #define _tcstod lucene_tcstod +#endif + +//printf functions +#if defined(_UCS2) && (!defined(_CL_HAVE_SNWPRINTF) || defined(_CL_HAVE_SWPRINTF_BUG) ) + #undef _sntprintf + #define _sntprintf lucene_snwprintf + int lucene_snwprintf(wchar_t* strbuf, size_t count, const wchar_t * format, ...); + + #ifndef __CL_INCLUDE_TPRINTF + #define __CL_INCLUDE_TPRINTF + #endif +#endif +#if defined(_UCS2) && !defined(_CL_HAVE_WPRINTF) + #undef _tprintf + #define _tprintf lucene_wprintf + void lucene_wprintf(const wchar_t * format, ...); + + #ifndef __CL_INCLUDE_TPRINTF + #define __CL_INCLUDE_TPRINTF + #endif +#endif +#if defined(_UCS2) && (!defined(_CL_HAVE_VSNWPRINTF) || defined(_CL_HAVE_SWPRINTF_BUG) ) + #undef _vsntprintf + #define _vsntprintf lucene_vsnwprintf + int lucene_vsnwprintf(wchar_t * strbuf, size_t count, const wchar_t * format, va_list& ap); + + #ifndef __CL_INCLUDE_TPRINTF + #define __CL_INCLUDE_TPRINTF + #endif +#endif + +//todo: if _CL_HAVE_SNPRINTF_BUG fails(snprintf overflow),we should use our own +//function. but we don't have it currently, and our functions are dubious anyway... + +#endif //end of _lucene_repl_wchar_h diff --git a/src/assistant/3rdparty/clucene/src/CLucene/config/threadCSection.h b/src/assistant/3rdparty/clucene/src/CLucene/config/threadCSection.h new file mode 100644 index 000000000..ab1842051 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/config/threadCSection.h @@ -0,0 +1,71 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +//NOTE: do not include this file directly, it is included from lucene internally. + +#ifndef lucene_config_threadCSection_h +#define lucene_config_threadCSection_h +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +CL_NS_DEF(util) + + #if !defined(LUCENE_USE_WINDOWS_H) && !defined(_WINDOWS_) && !defined(__MINGW32__) + //we have not explicity included windows.h and windows.h has + //not been included (check _WINDOWS_), then we must define + //our own definitions to the thread locking functions: + struct CRITICAL_SECTION + { + struct critical_section_debug * DebugInfo; + long LockCount; + long RecursionCount; + void * OwningThread; + void * LockSemaphore; + #if defined(_WIN64) + unsigned __int64 SpinCount; + #else + unsigned long SpinCount; + #endif + }; + #endif + + ///a windows implementation of the lock mutex + ///todo: boost has a InterlockedExchange way of locking too. More backwards compatible/faster??? + class mutex_win32 + { + private: + CRITICAL_SECTION mtx; + public: + mutex_win32(const mutex_win32& clone); + mutex_win32(); + ~mutex_win32(); + void lock(); + void unlock(); + }; + + class CLuceneThreadIdCompare + { + public: + + enum + { // parameters for hash table + bucket_size = 4, // 0 < bucket_size + min_buckets = 8 + }; // min_buckets = 2 ^^ N, 0 < N + + bool operator()( DWORD t1, DWORD t2 ) const{ + return t1 < t2; + } + }; + + #define _LUCENE_SLEEP(x) Sleep(x) + #define _LUCENE_THREADMUTEX CL_NS(util)::mutex_win32 + #define _LUCENE_CURRTHREADID GetCurrentThreadId() + #define _LUCENE_THREADID_TYPE DWORD +CL_NS_END + +#endif //lucene_config_threadCSection_h diff --git a/src/assistant/3rdparty/clucene/src/CLucene/config/threadPthread.h b/src/assistant/3rdparty/clucene/src/CLucene/config/threadPthread.h new file mode 100644 index 000000000..d0ed9c4c9 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/config/threadPthread.h @@ -0,0 +1,59 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +//NOTE: do not include this file directly, it is included from lucene internally. + +#ifndef lucene_config_threadPthread_h +#define lucene_config_threadPthread_h + +#include <pthread.h> + +CL_NS_DEF(util) + +///a posix implementation of the lock mutex +///todo: we need a spinlock implemenation for usage in reference counting +class mutex_pthread +{ +private: + pthread_mutex_t mtx; + +public: + mutex_pthread(const mutex_pthread& clone); + mutex_pthread(); + ~mutex_pthread(); + void lock(); + void unlock(); + +private: + #ifndef _CL_HAVE_PTHREAD_MUTEX_RECURSIVE + pthread_t lockOwner; + unsigned int lockCount; + #endif +}; + +#define _LUCENE_SLEEP(x) usleep(x*1000) //_LUCENE_SLEEP should be in millis, usleep is in micros +#define _LUCENE_THREADMUTEX CL_NS(util)::mutex_pthread +#define _LUCENE_CURRTHREADID pthread_self() +#define _LUCENE_THREADID_TYPE pthread_t + +class CLuceneThreadIdCompare +{ +public: + enum + { // parameters for hash table + bucket_size = 4, // 0 < bucket_size + min_buckets = 8 + }; // min_buckets = 2 ^^ N, 0 < N + + bool operator()( pthread_t t1, pthread_t t2 ) const{ + return t1 < t2; + } +}; + + +CL_NS_END + +#endif //lucene_config_threadPthread_h diff --git a/src/assistant/3rdparty/clucene/src/CLucene/config/threads.cpp b/src/assistant/3rdparty/clucene/src/CLucene/config/threads.cpp new file mode 100644 index 000000000..427e58092 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/config/threads.cpp @@ -0,0 +1,162 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#include "CLucene/StdHeader.h" + +#ifndef _CL_DISABLE_MULTITHREADING +CL_NS_DEF(util) + + +mutexGuard::mutexGuard(const mutexGuard& clone){ + //no autoclone + mrMutex = NULL; +} +mutexGuard::mutexGuard( _LUCENE_THREADMUTEX& rMutex ) : + mrMutex(&rMutex) +{ + mrMutex->lock(); +} +mutexGuard::~mutexGuard() +{ + mrMutex->unlock(); +} + +#if defined(_LUCENE_DONTIMPLEMENT_THREADMUTEX) + //do nothing + #if defined(_LUCENE_PRAGMA_WARNINGS) + #pragma message ("==================Not implementing any thread mutex==================") + #else + #warning "==================Not implementing any thread mutex==================" + #endif + + + +#elif defined(_CL_HAVE_WIN32_THREADS) + #include "CLucene/config/threadCSection.h" + + #if !defined(LUCENE_USE_WINDOWS_H) && !defined(_WINDOWS_) + //we have not explicity included windows.h and windows.h has + //not been included (check _WINDOWS_), then we must define + //our own definitions to the thread locking functions: + extern "C" __declspec(dllimport) void __stdcall InitializeCriticalSection(CRITICAL_SECTION *); + extern "C" __declspec(dllimport) void __stdcall EnterCriticalSection(CRITICAL_SECTION *); + extern "C" __declspec(dllimport) void __stdcall LeaveCriticalSection(CRITICAL_SECTION *); + extern "C" __declspec(dllimport) void __stdcall DeleteCriticalSection(CRITICAL_SECTION *); + extern "C" __declspec(dllimport) unsigned long __stdcall GetCurrentThreadId(); + #endif + + mutex_win32::mutex_win32(const mutex_win32& clone){ + InitializeCriticalSection(&mtx); + } + mutex_win32::mutex_win32() + { + InitializeCriticalSection(&mtx); + } + + mutex_win32::~mutex_win32() + { + DeleteCriticalSection(&mtx); + } + + void mutex_win32::lock() + { + EnterCriticalSection(&mtx); + } + + void mutex_win32::unlock() + { + LeaveCriticalSection(&mtx); + } + + + +#elif defined(_CL_HAVE_PTHREAD) + #include "CLucene/config/threadPthread.h" + + #ifdef _CL_HAVE_PTHREAD_MUTEX_RECURSIVE + bool mutex_pthread_attr_initd=false; + pthread_mutexattr_t mutex_pthread_attr; + #endif + + #ifdef _CL__CND_DEBUG + #define _CLPTHREAD_CHECK(c,m) CND_PRECONDITION(c==0,m) + #else + #define _CLPTHREAD_CHECK(c,m) c; + #endif + + mutex_pthread::mutex_pthread(const mutex_pthread& clone){ + #ifdef _CL_HAVE_PTHREAD_MUTEX_RECURSIVE + _CLPTHREAD_CHECK(pthread_mutex_init(&mtx, &mutex_pthread_attr), "mutex_pthread(clone) constructor failed") + #else + #if defined(__hpux) && defined(_DECTHREADS_) + _CLPTHREAD_CHECK(pthread_mutex_init(&mtx, pthread_mutexattr_default), "mutex_pthread(clone) constructor failed") + #else + _CLPTHREAD_CHECK(pthread_mutex_init(&mtx, 0), "mutex_pthread(clone) constructor failed") + #endif + lockCount=0; + lockOwner=0; + #endif + } + mutex_pthread::mutex_pthread() + { + #ifdef _CL_HAVE_PTHREAD_MUTEX_RECURSIVE + if ( mutex_pthread_attr_initd == false ){ + pthread_mutexattr_init(&mutex_pthread_attr); + pthread_mutexattr_settype(&mutex_pthread_attr, PTHREAD_MUTEX_RECURSIVE); + mutex_pthread_attr_initd = true; + } + _CLPTHREAD_CHECK(pthread_mutex_init(&mtx, &mutex_pthread_attr), "mutex_pthread(clone) constructor failed") + #else + #if defined(__hpux) && defined(_DECTHREADS_) + _CLPTHREAD_CHECK(pthread_mutex_init(&mtx, pthread_mutexattr_default), "mutex_pthread(clone) constructor failed") + #else + _CLPTHREAD_CHECK(pthread_mutex_init(&mtx, 0), "mutex_pthread(clone) constructor failed") + #endif + lockCount=0; + lockOwner=0; + #endif + } + + mutex_pthread::~mutex_pthread() + { + _CLPTHREAD_CHECK(pthread_mutex_destroy(&mtx), "~mutex_pthread destructor failed") + } + + void mutex_pthread::lock() + { + #ifndef _CL_HAVE_PTHREAD_MUTEX_RECURSIVE + pthread_t currentThread = pthread_self(); + if( pthread_equal( lockOwner, currentThread ) ) { + ++lockCount; + } else { + _CLPTHREAD_CHECK(pthread_mutex_lock(&mtx), "mutex_pthread::lock") + lockOwner = currentThread; + lockCount = 1; + } + #else + _CLPTHREAD_CHECK(pthread_mutex_lock(&mtx), "mutex_pthread::lock") + #endif + } + + void mutex_pthread::unlock() + { + #ifndef _CL_HAVE_PTHREAD_MUTEX_RECURSIVE + --lockCount; + if( lockCount == 0 ) + { + lockOwner = 0; + _CLPTHREAD_CHECK(pthread_mutex_unlock(&mtx), "mutex_pthread::unlock") + } + #else + _CLPTHREAD_CHECK(pthread_mutex_unlock(&mtx), "mutex_pthread::unlock") + #endif + } + +#endif //thread impl choice + + +CL_NS_END +#endif //!_CL_DISABLE_MULTITHREADING diff --git a/src/assistant/3rdparty/clucene/src/CLucene/config/utf8.cpp b/src/assistant/3rdparty/clucene/src/CLucene/config/utf8.cpp new file mode 100644 index 000000000..14ccf5aa9 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/config/utf8.cpp @@ -0,0 +1,237 @@ +/* + * Copyright (C) 1999 Tom Tromey + * Copyright (C) 2000 Red Hat, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + * + ************************************************ + * Also licensed with permission from Tom Tromey + * and Owen Taylor under the Apache license. + * Original location: + * http://cvs.gnome.org/viewcvs/glib/glib/gutf8.c?rev=1.50&view=log + ************************************************ + * + * Copyright 2003-2006 The Apache Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + #include "CLucene/StdHeader.h" + +typedef unsigned long gunichar; +typedef unsigned char guchar; + +#define UTF8_COMPUTE(Char, Mask, Len) \ + if (Char < 128) \ + { \ + Len = 1; \ + Mask = 0x7f; \ + } \ + else if ((Char & 0xe0) == 0xc0) \ + { \ + Len = 2; \ + Mask = 0x1f; \ + } \ + else if ((Char & 0xf0) == 0xe0) \ + { \ + Len = 3; \ + Mask = 0x0f; \ + } \ + else if ((Char & 0xf8) == 0xf0) \ + { \ + Len = 4; \ + Mask = 0x07; \ + } \ + else if ((Char & 0xfc) == 0xf8) \ + { \ + Len = 5; \ + Mask = 0x03; \ + } \ + else if ((Char & 0xfe) == 0xfc) \ + { \ + Len = 6; \ + Mask = 0x01; \ + } \ + else \ + Len = -1; + +/*#define UTF8_LENGTH(Char) \ + ((Char) < 0x80 ? 1 : \ + ((Char) < 0x800 ? 2 : \ + ((Char) < 0x10000 ? 3 : \ + ((Char) < 0x200000 ? 4 : \ + ((Char) < 0x4000000 ? 5 : 6)))))*/ + + +#define UTF8_GET(Result, Chars, Count, Mask, Len) \ + (Result) = (Chars)[0] & (Mask); \ + for ((Count) = 1; (Count) < (Len); ++(Count)) \ + { \ + if (((Chars)[(Count)] & 0xc0) != 0x80) \ + { \ + (Result) = -1; \ + break; \ + } \ + (Result) <<= 6; \ + (Result) |= ((Chars)[(Count)] & 0x3f); \ + } + + +/** + * lucene_wctoutf8: + * @c: a ISO10646 character code + * @outbuf: output buffer, must have at least 6 bytes of space. + * If %NULL, the length will be computed and returned + * and nothing will be written to @outbuf. + * + * Converts a single character to UTF-8. + * + * Return value: number of bytes written + **/ +size_t lucene_wctoutf8(char * outbuf, const wchar_t ch) +{ + gunichar c = ch; + guchar len = 0; + int first; + int i; + + if (c < 0x80) + { + first = 0; + len = 1; + } + else if (c < 0x800) + { + first = 0xc0; + len = 2; + } + else if (c < 0x10000) + { + first = 0xe0; + len = 3; + } + else if (c < 0x200000) + { + first = 0xf0; + len = 4; + } + else if (c < 0x4000000) + { + first = 0xf8; + len = 5; + } + else + { + first = 0xfc; + len = 6; + } + + if (outbuf) + { + for (i = len - 1; i > 0; --i) + { + outbuf[i] = (char)((c & 0x3f) | 0x80); + c >>= 6; + } + outbuf[0] = c | first; + } + + return len; +} + + +/** + * lucene_utf8towc: + * @p: a pointer to Unicode character encoded as UTF-8 + * + * Converts a sequence of bytes encoded as UTF-8 to a Unicode character. + * If @p does not point to a valid UTF-8 encoded character, results are + * undefined. If you are not sure that the bytes are complete + * valid Unicode characters, you should use lucene_utf8towc_validated() + * instead. + * + * Return value: the resulting character + **/ +size_t lucene_utf8towc(wchar_t *pwc, const char *p, size_t n) +{ + int i, mask = 0; + int result; + unsigned char c = (unsigned char) *p; + int len=0; + + UTF8_COMPUTE (c, mask, len); + if (len == -1) + return 0; + UTF8_GET (result, p, i, mask, len); + + *pwc = result; + return len; +} + + +//this function was not taken from gnome +size_t lucene_wcstoutf8(char * result, const wchar_t * str, size_t result_length){ + char *p=result; + int i = 0; + + while (p < result + result_length-1 && str[i] != 0) + p += lucene_wctoutf8(p,str[i++]); + + *p = '\0'; + + return p-result; +} +//this function was not taken from gnome +size_t lucene_utf8towcs(wchar_t * result, const char * str, size_t result_length){ + char *sp = (char*)str; + wchar_t *rp = result; + int i = 0; + + while (rp < result + result_length && *sp!=0){ + size_t r = lucene_utf8towc(rp,sp,6); + if ( r == -1 ) + return 0; + sp += r; + rp++; + } + + if ( sp-str < result_length ) + *rp = '\0'; + + size_t ret = sp-str; + return ret; +} +//get the number of bytes that make up the utf8 character. +//this function was not taken from gnome +size_t lucene_utf8charlen(const char *p) +{ + int mask = 0; + int len=0; + unsigned char c = (unsigned char) *p; + + UTF8_COMPUTE (c, mask, len); + return len; +} diff --git a/src/assistant/3rdparty/clucene/src/CLucene/debug/condition.cpp b/src/assistant/3rdparty/clucene/src/CLucene/debug/condition.cpp new file mode 100644 index 000000000..855419451 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/debug/condition.cpp @@ -0,0 +1,80 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#include "CLucene/StdHeader.h" +#include "condition.h" +#include "CLucene/util/Misc.h" +#ifdef _CL__CND_DEBUG + +#define __CND_STR_PRECONDITION "PRECONDITION" +#define __CND_STR_CONDITION "CONDITION" +#define __CND_STR_WARNING "WARNING" +#define __CND_STR_MESSAGE "MESSAGE" +#define __CND_STR_DEBUGMESSAGE "DEBUG MESSAGE" +#define __CND_STR_EXIT "EXIT" + +#ifndef _CND_DEBUG_DONTIMPLEMENT_OUTDEBUG +void _Cnd_OutDebug( const char* FormattedMsg, const char* StrTitle, const char* File, int32_t Line, int32_t Title, const char* Mes2, int32_t fatal ){ + #ifdef __WINDOWS_H + /*Display a standard messagebox*/ + MessageBox(NULL, FormattedMsg, StrTitle, (fatal==1 ? MB_ICONSTOP:MB_ICONEXCLAMATION) | MB_OK | MB_TASKMODAL); + #else + printf("%s\n",FormattedMsg); + #endif + + #if defined(_CND_DEBUG_WARN_DEBUGGER) /*attempt to signal windows debugger*/ + OutputDebugString(FormattedMsg); + DebugBreak(); /*Position debugger just before exit program*/ + #endif + + if ( fatal ) + debugFatalExit(1); +} +#endif + +void __cnd_FormatDebug( const char* File, int32_t Line, int32_t Title, const char* Mes2, int32_t fatal ) { + char M[512]; + char* StrTitle = NULL; + + if( Mes2 ) + _snprintf(M,512,"file:%s line:%d\n%s",File,Line,Mes2); + else + _snprintf(M,512,"file:%s line:%d",File,Line); + + /*Determine which title to use*/ + switch( Title ) { + case CND_STR_PRECONDITION: { + StrTitle = __CND_STR_PRECONDITION; + break; + } + case CND_STR_CONDITION: { + StrTitle = __CND_STR_CONDITION; + break; + } + case CND_STR_WARNING: { + StrTitle = __CND_STR_WARNING; + break; + } + case CND_STR_MESSAGE: { + StrTitle = __CND_STR_MESSAGE; + break; + } + case CND_STR_DEBUGMESSAGE: { + StrTitle = __CND_STR_DEBUGMESSAGE; + break; + } + case CND_STR_EXIT: { + StrTitle = __CND_STR_EXIT; + break; + } + default: + break; + }/*switch*/ + + _Cnd_OutDebug(M, StrTitle, File, Line, Title, Mes2, fatal); +} +#endif + diff --git a/src/assistant/3rdparty/clucene/src/CLucene/debug/condition.h b/src/assistant/3rdparty/clucene/src/CLucene/debug/condition.h new file mode 100644 index 000000000..ab227e508 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/debug/condition.h @@ -0,0 +1,64 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef __CONDITION_H +#define __CONDITION_H + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +/* +To enable condition debugging uncomment _CND_DEBUG in CLConfig.h +*/ + +#ifdef _CL__CND_DEBUG /* Don't include the debug code */ + #ifndef CND_STR_DEFINES + #define CND_STR_DEFINES + #define CND_STR_PRECONDITION 1 + #define CND_STR_CONDITION 2 + #define CND_STR_WARNING 3 + #define CND_STR_MESSAGE 4 + #define CND_STR_DEBUGMESSAGE 5 + #define CND_STR_EXIT 6 + #endif + + /* _CL__CND_DEBUG defined, include debug code */ + + #ifdef _CND_NODEBUGTEXT + #define CND_PRECONDITION(cond,usermessage) CND__EXITCONDITION(cond,__FILE__,__LINE__,CND_STR_PRECONDITION,NULL) + #define CND_CONDITION(cond,usermessage) CND__EXITCONDITION(cond,__FILE__,__LINE__,CND_STR_CONDITION,NULL) + #define CND_WARNING(cond,usermessage) CND__CONDITION(cond,__FILE__,__LINE__,CND_STR_WARNING,NULL) + #define CND_MESSAGE(cond,usermessage) CND__CONDITION(cond,__FILE__,__LINE__,CND_STR_MESSAGE,NULL) + #define CND_DEBUGMESSAGE(usermessage) CND__MESSAGE(__FILE__,__LINE__,CND_STR_DEBUGMESSAGE,NULL) + #else + #define CND_PRECONDITION(cond,usermessage) CND__EXITCONDITION(cond,__FILE__,__LINE__,CND_STR_PRECONDITION,usermessage) + #define CND_CONDITION(cond,usermessage) CND__EXITCONDITION(cond,__FILE__,__LINE__,CND_STR_CONDITION,usermessage) + #define CND_WARNING(cond,usermessage) CND__CONDITION(cond,__FILE__,__LINE__,CND_STR_WARNING,usermessage) + #define CND_MESSAGE(cond,usermessage) CND__CONDITION(cond,__FILE__,__LINE__,CND_STR_MESSAGE,usermessage) + #define CND_DEBUGMESSAGE(usermessage) CND__MESSAGE(__FILE__,__LINE__,CND_STR_DEBUGMESSAGE,usermessage) + #endif + + //if _CND_DEBUG_DONTIMPLEMENT_OUTDEBUG is defined, then you must implement + //this routine in the client application. The debug callback can then + //be better customised to the host application. + //Here is the default implementation: + void _Cnd_OutDebug( const char* FormattedMsg, const char* StrTitle, const char* File, int32_t Line, int32_t Title, const char* Mes2, int32_t fatal ); + + void __cnd_FormatDebug( const char* File, int32_t Line, int32_t Title, const char* Mes2, int32_t fatal ); + #define CND__EXIT(file,line,title,mes2) {__cnd_FormatDebug(file,line,title,mes2,1);} + #define CND__EXITCONDITION(cond,file,line,title,mes2) {if(!(cond)){__cnd_FormatDebug(file,line,title,mes2,1);}} + #define CND__CONDITION(cond,file,line,title,mes2) {if(!(cond)){__cnd_FormatDebug(file,line,title,mes2,0);}} + #define CND__MESSAGE(file,line,title,mes2) {__cnd_FormatDebug(file,line,title,mes2,0);} +#else + #define CND_PRECONDITION(cond, usermessage) + #define CND_CONDITION(cond, usermessage) + #define CND_WARNING(cond,usermessage) + #define CND_MESSAGE(cond,usermessage) + #define CND_DEBUGMESSAGE(usermessage) +#endif + +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/debug/error.cpp b/src/assistant/3rdparty/clucene/src/CLucene/debug/error.cpp new file mode 100644 index 000000000..53ea0e93b --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/debug/error.cpp @@ -0,0 +1,73 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#include "CLucene/StdHeader.h" + +CL_NS_USE(util) + + +#ifdef _LUCENE_DISABLE_EXCEPTIONS + #ifdef _LUCENE_PRAGMA_WARNINGS + #pragma message ("==================Lucene exceptions are disabled==================") + #else + #warning "==================Lucene exceptions are disabled==================" + #endif +#else + CLuceneError::CLuceneError(int num, const char* str, bool ownstr) + { + error_number = num; + _awhat=STRDUP_AtoA(str); + _twhat=NULL; + if ( ownstr ) + _CLDELETE_CaARRAY(str); + } + + CLuceneError::CLuceneError(const CLuceneError& clone) + { + this->error_number = clone.error_number; + this->_awhat = NULL; + this->_twhat = NULL; + + if ( clone._awhat != NULL ) + this->_awhat = STRDUP_AtoA(clone._awhat); + if ( clone._twhat != NULL ) + this->_twhat = STRDUP_TtoT(clone._twhat); + } + CLuceneError::~CLuceneError() throw(){ + _CLDELETE_CARRAY(_twhat); + _CLDELETE_CaARRAY(_awhat); + } + char* CLuceneError::what(){ +#ifdef _ASCII + if ( _twhat != NULL ) + return _twhat; +#endif + if ( _awhat == NULL ) + _awhat = STRDUP_TtoA(_twhat); + return _awhat; + } + TCHAR* CLuceneError::twhat(){ +#ifdef _ASCII + if ( _awhat != NULL ) + return _awhat; +#endif + if ( _twhat == NULL ) + _twhat = STRDUP_AtoT(_awhat); + return _twhat; + } + +#ifndef _ASCII + CLuceneError::CLuceneError(int num, const TCHAR* str, bool ownstr) + { + error_number = 0; + _awhat=NULL; + _twhat=STRDUP_TtoT(str); + if ( ownstr ) + _CLDELETE_CARRAY(str); + } +#endif + +#endif //_LUCENE_DISABLE_EXCEPTIONS diff --git a/src/assistant/3rdparty/clucene/src/CLucene/debug/error.h b/src/assistant/3rdparty/clucene/src/CLucene/debug/error.h new file mode 100644 index 000000000..5abcc802c --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/debug/error.h @@ -0,0 +1,74 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _lucene_debug_error_ +#define _lucene_debug_error_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#define CL_ERR_UNKNOWN -1 +#define CL_ERR_IO 1 +#define CL_ERR_NullPointer 2 +#define CL_ERR_Runtime 3 +#define CL_ERR_IllegalArgument 4 +#define CL_ERR_Parse 5 +#define CL_ERR_TokenMgr 6 +#define CL_ERR_UnsupportedOperation 7 +#define CL_ERR_InvalidState 8 +#define CL_ERR_IndexOutOfBounds 9 +#define CL_ERR_TooManyClauses 10 +#define CL_ERR_RAMTransaction 11 +#define CL_ERR_InvalidCast 12 +#define CL_ERR_IllegalState 13 + + + +//////////////////////////////////////////////////////// +//error try/throw/catch definitions +//////////////////////////////////////////////////////// +#ifdef _CL_DISABLE_NATIVE_EXCEPTIONS + /*#define try _jpr_Try + #define _CLCATCH _jpr_Catch + #define _CLFINALLY(x) xxxx + #define _CLTHROWA(y) _jpr_Throw + #define _THROWA_DEL(y) _jpr_Throw + #define _RETHROW(x) _jpr_Throw + #define _CLTHROWT(y) _jpr_Throw + + #define _THROWS ,_jpr_Throws*/ +#else + class CLuceneError + { + int error_number; + char* _awhat; + TCHAR* _twhat; + public: + CLuceneError(const CLuceneError& clone); + CLuceneError(int num, const char* str, bool ownstr); +#ifdef _UCS2 + CLuceneError(int num, const TCHAR* str, bool ownstr); +#endif + int number(){return error_number;} + char* what(); + TCHAR* twhat(); + ~CLuceneError() throw(); + }; + + //#define _THROWS //does nothing + #define _CLFINALLY(x) catch(...){ x; throw; } x //note: code x is not run if return is called + #define _CLTHROWA(number, str) throw CLuceneError(number, str,false) + #define _CLTHROWT(number, str) throw CLuceneError(number, str,false) + #define _CLTHROWA_DEL(number, str) throw CLuceneError(number, str,true) //throw a string ensures the value is deleted + #define _CLTHROWT_DEL(number, str) throw CLuceneError(number, str,true) //throw a string ensures the value is deleted + + +#endif //_LUCENE_DISABLE_EXCEPTIONS +// +//////////////////////////////////////////////////////// + +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/debug/lucenebase.h b/src/assistant/3rdparty/clucene/src/CLucene/debug/lucenebase.h new file mode 100644 index 000000000..86cdae1c5 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/debug/lucenebase.h @@ -0,0 +1,75 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _lucene_debug_lucenebase_ +#define _lucene_debug_lucenebase_ + +#ifdef _LUCENE_PRAGMA_ONCE +# pragma once +#endif + +CL_NS_DEF(debug) + +//Lucenebase is the superclass of all clucene objects. It provides +//memory debugging tracking and/or reference counting +class LuceneBase{ +public: +#ifdef LUCENE_ENABLE_MEMLEAKTRACKING + static void* operator new (size_t size); + static void operator delete (void *p); + int32_t __cl_initnum; ///< The order that the object was created at. This is then used to do a lookup in the objects list + + static void* operator new (size_t size, char const * file, int32_t line); + static void operator delete (void *p, char const * file, int32_t line); + + static void* __cl_voidpadd(void* data, const char* file, int line, size_t size); ///<add arbitary data to the lucenbase_list and returns the same data + static void __cl_voidpremove(const void* data, const char* file, int line);///<remove arbitary data to the lucenbase_list + static void __cl_unregister(const void* obj); ///<un register object from the mem leak and ref count system + + static int32_t __cl_GetUnclosedObjectsCount(); ///< gets the number of unclosed objects + static const char* __cl_GetUnclosedObject(int32_t item); ///< get the name of the nth unclosed object + static char* __cl_GetUnclosedObjects(); ///< get a string with the names of the unclosed objects + static void __cl_PrintUnclosedObjects(); ///< print unclosed objects to the stdout + + ///This will clear memory relating to refcounting + ///other tools can be used to more accurately identify + ///memory leaks. This should only be called just + ///before closing, and after retrieving the + ///unclosed object list + static void __cl_ClearMemory(); + +#endif //LUCENE_ENABLE_MEMLEAKTRACKING + + int __cl_refcount; + LuceneBase(){ + __cl_refcount=1; + } + inline int __cl_getref(){ + return __cl_refcount; + } + inline int __cl_addref(){ + __cl_refcount++; + return __cl_refcount; + } + inline int __cl_decref(){ + __cl_refcount--; + return __cl_refcount; + } + virtual ~LuceneBase(){}; +}; + +class LuceneVoidBase{ + public: + #ifdef _DEBUG + //a compile time check to make sure that _CLDELETE and _CLDECDELETE is being + //used correctly. + int dummy__see_mem_h_for_details; + #endif + virtual ~LuceneVoidBase(){}; +}; + +CL_NS_END +#endif //_lucene_debug_lucenebase_ diff --git a/src/assistant/3rdparty/clucene/src/CLucene/debug/mem.h b/src/assistant/3rdparty/clucene/src/CLucene/debug/mem.h new file mode 100644 index 000000000..e15c3de8b --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/debug/mem.h @@ -0,0 +1,130 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _lucene_debug_mem_h +#define _lucene_debug_mem_h + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include "lucenebase.h" + +//Macro for creating new objects +#if defined(LUCENE_ENABLE_MEMLEAKTRACKING) + #define _CLNEW new(__FILE__, __LINE__) + #define LUCENE_BASE public CL_NS(debug)::LuceneBase +#elif defined(LUCENE_ENABLE_REFCOUNT) + #define _CLNEW new + #define LUCENE_BASE public CL_NS(debug)::LuceneBase +#else + #define _CLNEW new + #define LUCENE_BASE public CL_NS(debug)::LuceneVoidBase + #define LUCENE_BASE_CHECK(obj) (obj)->dummy__see_mem_h_for_details +#endif +#define _CL_POINTER(x) (x==NULL?NULL:(x->__cl_addref()>=0?x:x)) //return a add-ref'd object +#define LUCENE_REFBASE public CL_NS(debug)::LuceneBase //this is the base of classes who *always* need refcounting + +#if defined(_DEBUG) + #if !defined(LUCENE_BASE_CHECK) + #define LUCENE_BASE_CHECK(x) + #endif +#else + #undef LUCENE_BASE_CHECK + #define LUCENE_BASE_CHECK(x) +#endif + +//Macro for creating new arrays +#ifdef LUCENE_ENABLE_MEMLEAKTRACKING + #define _CL_NEWARRAY(type,size) (type*)CL_NS(debug)::LuceneBase::__cl_voidpadd(new type[(size_t)size],__FILE__,__LINE__,(size_t)size); + #define _CLDELETE_ARRAY(x) if (x!=NULL){CL_NS(debug)::LuceneBase::__cl_voidpremove((const void*)x,__FILE__,__LINE__); delete [] x; x=NULL;} + #define _CLDELETE_LARRAY(x) if (x!=NULL){CL_NS(debug)::LuceneBase::__cl_voidpremove((const void*)x,__FILE__,__LINE__);delete [] x;} + #ifndef _CLDELETE_CARRAY + #define _CLDELETE_CARRAY(x) if (x!=NULL){CL_NS(debug)::LuceneBase::__cl_voidpremove((const void*)x,__FILE__,__LINE__);delete [] x; x=NULL;} + #define _CLDELETE_LCARRAY(x) if (x!=NULL){CL_NS(debug)::LuceneBase::__cl_voidpremove((const void*)x,__FILE__,__LINE__);delete [] x;} + #endif +#else + #define _CL_NEWARRAY(type,size) new type[size] + #define _CLDELETE_ARRAY(x) if (x!=NULL){delete [] x; x=NULL;} + #define _CLDELETE_LARRAY(x) if (x!=NULL){delete [] x;} + #ifndef _CLDELETE_CARRAY + #define _CLDELETE_CARRAY(x) if (x!=NULL){delete [] x; x=NULL;} + #define _CLDELETE_LCARRAY(x) if (x!=NULL){delete [] x;} + #endif +#endif +//a shortcut for deleting a carray and all its contents +#define _CLDELETE_CARRAY_ALL(x) {if ( x!=NULL ){ for(int xcda=0;x[xcda]!=NULL;xcda++)_CLDELETE_CARRAY(x[xcda]);}_CLDELETE_ARRAY(x)}; +#define _CLDELETE_ARRAY_ALL(x) {if ( x!=NULL ){ for(int xcda=0;x[xcda]!=NULL;xcda++)_CLDELETE(x[xcda]);}_CLDELETE_ARRAY(x)}; +#ifndef _CLDELETE_CaARRAY + #define _CLDELETE_CaARRAY _CLDELETE_CARRAY + #define _CLDELETE_LCaARRAY _CLDELETE_LCARRAY +#endif + +//Macro for deleting +#ifdef LUCENE_ENABLE_REFCOUNT + #define _CLDELETE(x) if (x!=NULL){ CND_PRECONDITION((x)->__cl_refcount>=0,"__cl_refcount was < 0"); if ((x)->__cl_decref() <= 0)delete x; x=NULL; } + #define _CLLDELETE(x) if (x!=NULL){ CND_PRECONDITION((x)->__cl_refcount>=0,"__cl_refcount was < 0"); if ((x)->__cl_decref() <= 0)delete x; } +#else + #define _CLDELETE(x) if (x!=NULL){ LUCENE_BASE_CHECK(x); delete x; x=NULL; } + #define _CLLDELETE(x) if (x!=NULL){ LUCENE_BASE_CHECK(x); delete x; } +#endif + +//_CLDECDELETE deletes objects which are *always* refcounted +#define _CLDECDELETE(x) if (x!=NULL){ CND_PRECONDITION((x)->__cl_refcount>=0,"__cl_refcount was < 0"); if ((x)->__cl_decref() <= 0)delete x; x=NULL; } +#define _CLLDECDELETE(x) if (x!=NULL){ CND_PRECONDITION((x)->__cl_refcount>=0,"__cl_refcount was < 0"); if ((x)->__cl_decref() <= 0)delete x; } + +//_VDelete should be used for deleting non-clucene objects. +//when using reference counting, _CLDELETE casts the object +//into a LuceneBase*. +#define _CLVDELETE(x) if(x!=NULL){delete x; x=NULL;} + +template<typename T> +class Array: LUCENE_BASE{ +public: + T* values; + size_t length; + + void deleteAll(){ + for (size_t i=0;i<length;i++) + _CLDELETE(values[i]); + _CLDELETE_ARRAY(values); + } + void deleteArray(){ + _CLDELETE_ARRAY(values); + } + + Array(){ + values = NULL; + length = 0; + } + Array(T* values, size_t length){ + this->values = values; + this->length = length; + } + Array(size_t length){ + this->values = _CL_NEWARRAY(T,length); + this->length = length; + } + ~Array(){} + + const T operator[](size_t _Pos) const + { + if (length <= _Pos){ + _CLTHROWA(CL_ERR_IllegalArgument,"vector subscript out of range"); + } + return (*(values + _Pos)); + } + T operator[](size_t _Pos) + { + if (length <= _Pos){ + _CLTHROWA(CL_ERR_IllegalArgument,"vector subscript out of range"); + } + return (*(values + _Pos)); + } + +}; + +#endif //_lucene_debug_lucenebase_ diff --git a/src/assistant/3rdparty/clucene/src/CLucene/debug/memtracking.cpp b/src/assistant/3rdparty/clucene/src/CLucene/debug/memtracking.cpp new file mode 100644 index 000000000..544a125a5 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/debug/memtracking.cpp @@ -0,0 +1,371 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#include "CLucene/StdHeader.h" +#include "CLucene/util/Misc.h" + +bool _lucene_disable_debuglogging = true; //if LUCENE_ENABLE_CONSTRUCTOR_LOG is on, dont do log if this is true +bool _lucene_run_objectcheck = false; //run a memory check before deleting objects +int _lucene_counter_break = -1; //to break at this item, change this + //and put break points at points below + +CL_NS_USE(util) +CL_NS_DEF(debug) + +#ifdef LUCENE_ENABLE_MEMLEAKTRACKING +int32_t _instance_counter = 0; //counter for initnumber + +struct _file{ + int32_t refcount; ///times this has been used + char* value; ///reference to the the basefile +}; //structure for name counting +struct _pointers{ + _file* file; + int32_t initline; + int32_t initnumber; +};//structure for pointer-filename references + +typedef CL_NS(util)::CLSet<const char*,_file*,Compare::Char,Deletor::Dummy,Deletor::Void<_file> > defFile; +typedef CL_NS(util)::CLSet<LuceneBase*,_pointers*,Compare::Void<LuceneBase>,Deletor::Dummy,Deletor::Void<_pointers> > defPointer; +typedef CL_NS(util)::CLSet<const void*,_pointers*,Compare::Void<const void>,Deletor::Dummy,Deletor::Void<_pointers> > defVoid; + +DEFINE_MUTEX(memleak_lock) +defFile LuceneBase_Files(false,true); //list of filenames used +defPointer LuceneBase_Pointers(false,true); //list of pointers counted +defVoid LuceneBase_Voids(false,true); //list of arbitary data added + +//variables to trim filenames to just the base names +char _files_trim_string[CL_MAX_DIR]; +int32_t _files_trim_start=-1; + +//trim the filename and return the refcounted _file* structure +_file* get_file(const char* file){ + if ( _files_trim_start == -1 ){ + //this trims the start of the name file name so + //that the whole of the filename is not stored - more asthetic :) + //need to find the base + _files_trim_start = strlen(__FILE__) - 21; //(length of debug/memtracking.cpp) + strcpy(_files_trim_string,__FILE__); + _files_trim_string[_files_trim_start] = 0; + } + if ( strncmp(file,_files_trim_string,_files_trim_start) == 0 ){ + //this file should be within the same directory area as we found lucenebase.cpp + //to be, lets trim the start + file+=_files_trim_start; + } + + //now return an existing files structure (with refcount++) or create a new one + defFile::iterator itr = LuceneBase_Files.find((const char*)file); + if ( itr != LuceneBase_Files.end() ){ + _file* bf = itr->second; + bf->refcount++; + return bf; + }else{ + _file* ref = new _file; + ref->value = new char[strlen(file)+1]; //cannot use _CL_NEWARRAY otherwise recursion + strcpy(ref->value,file); + + ref->refcount = 1; + LuceneBase_Files.insert(pair<const char*,_file*>(ref->value,ref)); + return ref; + } +} + +void remove_file(_file* bf){ + bf->refcount--; + if ( bf->refcount <= 0 ){ + defFile::iterator fi = LuceneBase_Files.find(bf->value); + CND_PRECONDITION(fi!=LuceneBase_Files.end(),"fi==NULL"); + delete[] bf->value; + LuceneBase_Files.removeitr(fi); + } +} + +#ifdef LUCENE_ENABLE_CONSTRUCTOR_LOG + void constructor_log(const char* type,const char* file,const int line, const int size){ + if ( _lucene_disable_debuglogging ){ + FILE* f = fopen("clucene.log","a"); + char buf[CL_MAX_DIR+5]; + sprintf(buf,"%s,%s,%d,%d\n",type,file,line,size); + fwrite(buf,sizeof(char),strlen(buf),f); + fclose(f); + } + } + #define CONSTRUCTOR_LOG(type,file,line,size) constructor_log(type,file,line,size) +#else + #define CONSTRUCTOR_LOG(type,file,line,size) +#endif + +//////////////////////////////////////////////////////////////////////////////// +// the _CLNEW&_CLDELETE new/delete operators +//////////////////////////////////////////////////////////////////////////////// +void* LuceneBase::operator new (size_t size, const char * file, int32_t line) +{ + SCOPED_LOCK_MUTEX(memleak_lock) + + void* p = malloc (size); + LuceneBase* lb = (LuceneBase*)p; + + //create the pointer struct + _file* br = get_file(file); + _pointers* bp = new _pointers; + bp->file = br; + bp->initnumber = _instance_counter++; + bp->initline = line; + + //associate this object with the pointer + lb->__cl_initnum = bp->initnumber; + + //break if necessary + if ( _lucene_counter_break == lb->__cl_initnum ) + CLDebugBreak(); //put break point here + + //add the pointer object + LuceneBase_Pointers.insert(pair<LuceneBase*,_pointers*>(lb, bp)); + + CONSTRUCTOR_LOG("newobj",file,line,size); + return p; +} +void LuceneBase::operator delete (void *p, char const * file, int32_t line) +{ + SCOPED_LOCK_MUTEX(memleak_lock) + + LuceneBase* lb=(LuceneBase*)p; + + defPointer::iterator itr = LuceneBase_Pointers.find(lb); + if ( itr != LuceneBase_Pointers.end() ){ + _pointers* bp = itr->second; + remove_file(bp->file); + + LuceneBase_Pointers.removeitr(itr); + }else{ + //break + } + free(p); +} + +/////////////////////////////////////////////////////////////////////////// +// the generic new/delete operators +/////////////////////////////////////////////////////////////////////////// +void* LuceneBase::operator new (size_t size) +{ + SCOPED_LOCK_MUTEX(memleak_lock) + + void* p = malloc (size); + LuceneBase* lb = (LuceneBase*)p; + + //create the pointer struct + _file* br = get_file("undefined"); + _pointers* bp = new _pointers; + bp->file = br; + bp->initnumber = _instance_counter++; + bp->initline = -1; + + //associate this object with the pointer + lb->__cl_initnum = bp->initnumber; + + //break if necessary + if ( _lucene_counter_break == lb->__cl_initnum ) + CLDebugBreak(); + + //add the pointer object + LuceneBase_Pointers.insert(pair<LuceneBase*,_pointers*>(lb,bp)); + + CONSTRUCTOR_LOG("newobj","unknown",-1,size); + return p; +} +void LuceneBase::operator delete (void *p) +{ + SCOPED_LOCK_MUTEX(memleak_lock) + + LuceneBase* lb=(LuceneBase*)p; + + defPointer::iterator itr = LuceneBase_Pointers.find(lb); + if ( itr != LuceneBase_Pointers.end() ){ + _pointers* bp = itr->second; + remove_file(bp->file); + LuceneBase_Pointers.removeitr(itr); + }else{ + CLDebugBreak(); + } + free(p); +} + +/////////////////////////////////////////////////////////////////////////// +// other memtracking functions +/////////////////////////////////////////////////////////////////////////// +void LuceneBase::__cl_unregister(const void* obj){ + SCOPED_LOCK_MUTEX(memleak_lock) + + LuceneBase* lb=(LuceneBase*)obj; + defPointer::iterator itr = LuceneBase_Pointers.find(lb); + CND_PRECONDITION(itr != LuceneBase_Pointers.end(),"__cl_unregister object not found"); + _pointers* bp = itr->second; + LuceneBase_Pointers.removeitr(itr); +} + +void* LuceneBase::__cl_voidpadd(void* data, const char* file, int line,size_t size){ + SCOPED_LOCK_MUTEX(memleak_lock) + + _file* br = get_file(file); + _pointers* bp = new _pointers; + bp->file = br; + bp->initnumber = _instance_counter++; + bp->initline = line; + + LuceneBase_Voids.insert(pair<void*,_pointers*>(data,bp)); + CONSTRUCTOR_LOG("newarr",file,line,size); + return data; +} +void LuceneBase::__cl_voidpremove(const void* data, const char* file, int line){ + SCOPED_LOCK_MUTEX(memleak_lock) + defVoid::iterator itr = LuceneBase_Voids.find(data); + if ( itr != LuceneBase_Voids.end() ){ + _pointers* bp = itr->second; + remove_file(bp->file); + LuceneBase_Voids.removeitr(itr); + }else{ + printf("Data deleted when not added with _CL_NEWARRAY in %s at %d\n",file,line); + } +} + + +//////////////////////////////////////////////////////////// + + +//////////////////////////////////////////////////////////// +//The lucene base memory leak printout functions +//////////////////////////////////////////////////////////// +//static +void __internalcl_PrintUnclosedObject(bool isObject, string& sb,_pointers* bp,_file* bf, bool print){ + TCHAR ttmp[100]; + char atmp[100]; + + sb.append(" "); + { + _i64tot(bp->initnumber,ttmp,10); + STRCPY_TtoA(atmp,ttmp,100); + sb.append(atmp); + } + if ( isObject ){ + sb.append("(obj). "); + }else{ + sb.append(". "); + } + sb.append(bf->value); + sb.append(", line "); + { + _i64tot(bp->initline,ttmp,10); + STRCPY_TtoA(atmp,ttmp,100); + sb.append(atmp); + } + sb.append("\n"); + + if ( print && sb.length() > 0 ){ + printf("%s\n", sb.c_str()); + sb = ""; + } +} +char* __internalcl_GetUnclosedObjects(bool print){ + TCHAR ttmp[100]; + char atmp[100]; + SCOPED_LOCK_MUTEX(memleak_lock) + + string sb; + bool unknowns = false; + if ( LuceneBase_Pointers.size() > 0 ){ + { + _i64tot(LuceneBase_Pointers.size(),ttmp,10); + STRCPY_TtoA(atmp,ttmp,100); + sb.append(atmp); + } + sb.append(" clucene objects are still open\n"); + + defPointer::iterator itr = LuceneBase_Pointers.begin(); + while ( itr != LuceneBase_Pointers.end() ){ + _pointers* bp = itr->second; + _file* bf = bp->file; + + if ( bp->initline == -1 ) + unknowns = true; + __internalcl_PrintUnclosedObject(true, sb,bp,bf,print); + + ++itr; + } + + defVoid::iterator itr2 = LuceneBase_Voids.begin(); + while ( itr2 != LuceneBase_Voids.end() ){ + _pointers* bp = itr2->second; + _file* bf = bp->file; + + if ( bp->initline == -1 ) + unknowns = true; + __internalcl_PrintUnclosedObject(false, sb,bp,bf,print); + + itr2++; + } + } + + if ( unknowns == true ){ + sb.append("*** Some memory was not created with _CLNEW and was not tracked... ***\n"); + sb.append("*** Use _CLNEW instead of new when creating CLucene objects ***\n"); + sb.append("*** Memory may also have not been freed in the current context ***\n"); + } + + if ( print ){ + if ( sb.length() > 0 ){ + printf("%s\n", sb.c_str()); + sb = ""; + } + return NULL; + }else{ + if ( sb.length() > 0 ) + return STRDUP_AtoA(sb.c_str()); + else + return NULL; + } +} + +void LuceneBase::__cl_ClearMemory(){ + SCOPED_LOCK_MUTEX(memleak_lock) + + while ( LuceneBase_Files.size() > 0 ){ + defFile::iterator fi = LuceneBase_Files.begin(); + _file* f = fi->second; + delete[] f->value; + LuceneBase_Files.removeitr (fi); + } + LuceneBase_Pointers.clear(); + LuceneBase_Voids.clear(); +} +char* LuceneBase::__cl_GetUnclosedObjects(){ + return __internalcl_GetUnclosedObjects(false); +} +//static +int32_t LuceneBase::__cl_GetUnclosedObjectsCount(){ + return LuceneBase_Pointers.size(); +} + +const char* LuceneBase::__cl_GetUnclosedObject(int32_t item){ + SCOPED_LOCK_MUTEX(memleak_lock) + + defPointer::iterator itr=LuceneBase_Pointers.begin(); + int32_t i=0; + for ( ;itr!=LuceneBase_Pointers.end() && i<item ;itr++ ){ + ++i; + } + if ( itr != LuceneBase_Pointers.end() ) + return itr->second->file->value; + else + return NULL; +} +void LuceneBase::__cl_PrintUnclosedObjects(){ + __internalcl_GetUnclosedObjects(true); +} +//////////////////////////////////////////////////////////// + +#endif //LUCENE_ENABLE_MEMLEAKTRACKING +CL_NS_END diff --git a/src/assistant/3rdparty/clucene/src/CLucene/document/DateField.cpp b/src/assistant/3rdparty/clucene/src/CLucene/document/DateField.cpp new file mode 100644 index 000000000..ff72b12bb --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/document/DateField.cpp @@ -0,0 +1,60 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#include "CLucene/StdHeader.h" + +#include "DateField.h" +#include "CLucene/util/Misc.h" +CL_NS_USE(util) +CL_NS_DEF(document) + +DateField::~DateField(){ +} + +TCHAR* DateField::timeToString(const int64_t time) { + TCHAR* buf = _CL_NEWARRAY(TCHAR,DATEFIELD_DATE_LEN + 1); + timeToString(time,buf); + return buf; +} +void DateField::timeToString(const int64_t time, TCHAR* buf) { + CND_PRECONDITION (buf, "buf == NULL"); + *buf = '\0'; + if (time < 0) + _CLTHROWA (CL_ERR_IllegalArgument,"time too early"); //todo: make richer error + + if (time > DATEFIELD_DATE_MAX) + _CLTHROWA (CL_ERR_IllegalArgument, "time too late (past DATEFIELD_DATE_MAX"); //todo: make richer error + + _i64tot(time, buf, 36); + int32_t bufLen = _tcslen(buf); + + CND_PRECONDITION (bufLen <= DATEFIELD_DATE_LEN, "timeToString length is greater than 9"); + + /* Supply leading zeroes if necessary. */ + if (bufLen < DATEFIELD_DATE_LEN) { + const int32_t nMissingZeroes = DATEFIELD_DATE_LEN - bufLen; + /* Move buffer contents forward to make room for leading zeroes. */ + for (int32_t i = DATEFIELD_DATE_LEN - 1; i >= nMissingZeroes; i--) + buf[i] = buf[i - nMissingZeroes]; + + /* Insert leading zeroes. */ + {// MSVC6 scoping fix + for (int32_t i = 0; i < nMissingZeroes; i++) + buf[i] = '0'; + } + + buf[DATEFIELD_DATE_LEN] = 0; + } + + CND_PRECONDITION (_tcslen(buf) == DATEFIELD_DATE_LEN, "timeToString return is not equal to DATEFIELD_DATE_LEN"); +} + +int64_t DateField::stringToTime(const TCHAR* time) { + TCHAR* end; + return _tcstoi64(time, &end, 36); +} + +CL_NS_END diff --git a/src/assistant/3rdparty/clucene/src/CLucene/document/DateField.h b/src/assistant/3rdparty/clucene/src/CLucene/document/DateField.h new file mode 100644 index 000000000..712fe9b62 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/document/DateField.h @@ -0,0 +1,64 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _lucene_document_DateField_ +#define _lucene_document_DateField_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +CL_NS_DEF(document) + +//here are some constants used throughout clucene +//make date strings long enough to last a millenium +#define DATEFIELD_DATE_MAX _ILONGLONG(31536000000000) //1000L*365*24*60*60*1000 + +#define DATEFIELD_DATE_LEN 9 ////Long.toString(DATEFIELD_DATE_MAX, Character.MAX_RADIX).length() + +/** +* Provides support for converting dates to strings and vice-versa. +* The strings are structured so that lexicographic sorting orders by date, +* which makes them suitable for use as field values and search terms. +* +* <P>Note that this class saves dates with millisecond granularity, +* which is bad for {@link RangeQuery} and {@link PrefixQuery}, as those +* queries are expanded to a BooleanQuery with a potentially large number +* of terms when searching. Thus you might want to use +* {@link DateTools} instead. +* +* <P> +* Note: dates before 1970 cannot be used, and therefore cannot be +* indexed when using this class. See {@link DateTools} for an +* alternative without such a limitation. +* +* @deprecated If you build a new index, use {@link DateTools} instead. This class is included for use with existing +* indices and will be removed in a future release. +*/ +class DateField :LUCENE_BASE { +public: + ~DateField(); + + /** + * Converts a millisecond time to a string suitable for indexing. + * @throws RuntimeException if the time specified in the + * method argument is negative, that is, before 1970 + */ + static TCHAR* timeToString(const int64_t time); + + /** + * Converts a millisecond time to a string suitable for indexing. + * @throws CL_ERR_IllegalArgument if the time specified in the + * method argument is negative, that is, before 1970 + * @param str must be a character array DATEFIELD_DATE_LEN+1 or longer + */ + static void timeToString(const int64_t time, TCHAR* str); + + /** Converts a string-encoded date into a millisecond time. */ + static int64_t stringToTime(const TCHAR* s); +}; +CL_NS_END +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/document/Document.cpp b/src/assistant/3rdparty/clucene/src/CLucene/document/Document.cpp new file mode 100644 index 000000000..a0ce03942 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/document/Document.cpp @@ -0,0 +1,237 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#include "CLucene/StdHeader.h" +#include "Document.h" +#include "Field.h" +#include "CLucene/util/StringBuffer.h" + +CL_NS_USE(util) +CL_NS_DEF(document) + + DocumentFieldEnumeration::DocumentFieldList::DocumentFieldList(Field* f, DocumentFieldList* n ) { + //Func - Constructor + //Pre - f != NULL + // n may be NULL + //Post - Instance has been created + CND_PRECONDITION(f != NULL, "f is NULL"); + + field = f; + next = n; + } + DocumentFieldEnumeration::DocumentFieldList::~DocumentFieldList(){ + //Func - Destructor + //Pre - true + //Post - Instance has been destroyed + + // Instead of recursively deleting the field list we do + // it iteratively to avoid stack overflows when + // dealing with several thousands of fields. + + if (!field) { + return; // nothing to do; deleted by different invocation of dtor + } + + DocumentFieldList* cur = next; + while (cur != NULL) + { + DocumentFieldList* temp = cur->next; + cur->next = NULL; + + _CLDELETE(cur); + cur = temp; + } + _CLDELETE(field); + } + + + DocumentFieldEnumeration::DocumentFieldEnumeration(const DocumentFieldList* fl){ + //Func - Constructor + //Pre - fl may be NULL + //Post - Instance has been created + + fields = fl; + } + + DocumentFieldEnumeration::~DocumentFieldEnumeration(){ + //Func - Destructor + //Pre - true + //Post - Instance has been destroyed + } + + bool DocumentFieldEnumeration::hasMoreElements() const { + return fields == NULL ? false : true; + } + + Field* DocumentFieldEnumeration::nextElement() { + //Func - Return the next element in the enumeration + //Pre - true + //Post - The next element is returned or NULL + + + Field* result = NULL; + //Check if fields is still valid + if (fields){ + result = fields->field; + fields = fields->next; + } + return result; + } + + /** Constructs a new document with no fields. */ + Document::Document(){ + //Func - Constructor + //Pre - true + //Post - Instance has been created + boost = 1.0f; + fieldList = NULL; + } + + Document::~Document(){ + //Func - Destructor + //Pre - true + //Post - Instance has been destroyed + boost = 1.0f; + _CLDELETE(fieldList); + } + + void Document::clear(){ + _CLDELETE(fieldList); + } + + void Document::add(Field& field) { + fieldList = _CLNEW DocumentFieldEnumeration::DocumentFieldList(&field, fieldList); + } + + void Document::setBoost(qreal boost) { + this->boost = boost; + } + + qreal Document::getBoost() const { + return boost; + } + + + Field* Document::getField(const TCHAR* name) const{ + CND_PRECONDITION(name != NULL, "name is NULL"); + + for (DocumentFieldEnumeration::DocumentFieldList* list = fieldList; list != NULL; list = list->next) + //cannot use interning here, because name is probably not interned + if ( _tcscmp(list->field->name(), name) == 0 ){ + return list->field; + } + + return NULL; + } + + const TCHAR* Document::get(const TCHAR* field) const { + CND_PRECONDITION(field != NULL, "field is NULL"); + Field *f = getField(field); + if (f!=NULL) + return f->stringValue(); //this returns null it is a binary(reader) + else + return NULL; + } + + DocumentFieldEnumeration* Document::fields() const { + return _CLNEW DocumentFieldEnumeration(fieldList); + } + + + TCHAR* Document::toString() const { + StringBuffer ret(_T("Document<")); + for (DocumentFieldEnumeration::DocumentFieldList* list = fieldList; list != NULL; list = list->next) { + TCHAR* tmp = list->field->toString(); + ret.append( tmp ); + if (list->next != NULL) + ret.append(_T(" ")); + _CLDELETE_ARRAY( tmp ); + } + ret.append(_T(">")); + return ret.toString(); + } + + + + void Document::removeField(const TCHAR* name) { + CND_PRECONDITION(name != NULL, "name is NULL"); + + DocumentFieldEnumeration::DocumentFieldList* previous = NULL; + DocumentFieldEnumeration::DocumentFieldList* current = fieldList; + while (current != NULL) { + //cannot use interning here, because name is probably not interned + if ( _tcscmp(current->field->name(),name) == 0 ){ + if (previous){ + previous->next = current->next; + }else + fieldList = current->next; + current->next=NULL; //ensure fieldlist destructor doesnt delete it + _CLDELETE(current); + return; + } + previous = current; + current = current->next; + } + } + + void Document::removeFields(const TCHAR* name) { + CND_PRECONDITION(name != NULL, "name is NULL"); + + DocumentFieldEnumeration::DocumentFieldList* previous = NULL; + DocumentFieldEnumeration::DocumentFieldList* current = fieldList; + while (current != NULL) { + //cannot use interning here, because name is probably not interned + if ( _tcscmp(current->field->name(),name) == 0 ){ + if (previous){ + previous->next = current->next; + }else + fieldList = current->next; + + current->next=NULL; //ensure fieldlist destructor doesnt delete it + _CLDELETE(current); + + if ( previous ) + current = previous->next; + else + current = fieldList; + }else{ + previous = current; + current = current->next; + } + } + } + + TCHAR** Document::getValues(const TCHAR* name) { + DocumentFieldEnumeration* it = fields(); + int32_t count = 0; + while ( it->hasMoreElements() ){ + Field* f = it->nextElement(); + //cannot use interning here, because name is probably not interned + if ( _tcscmp(f->name(),name) == 0 && f->stringValue() != NULL ) + count++; + } + _CLDELETE(it); + it = fields(); + + //todo: there must be a better way of doing this, we are doing two iterations of the fields + TCHAR** ret = NULL; + if ( count > 0 ){ + //start again + ret = _CL_NEWARRAY(TCHAR*,count+1); + int32_t i=0; + while ( it->hasMoreElements() ){ + Field* fld=it->nextElement(); + if ( _tcscmp(fld->name(),name)== 0 && fld->stringValue() != NULL ){ + ret[i] = stringDuplicate(fld->stringValue()); + i++; + } + } + ret[count]=NULL; + } + _CLDELETE(it); + return ret; + } +CL_NS_END diff --git a/src/assistant/3rdparty/clucene/src/CLucene/document/Document.h b/src/assistant/3rdparty/clucene/src/CLucene/document/Document.h new file mode 100644 index 000000000..ba7a283f7 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/document/Document.h @@ -0,0 +1,158 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _lucene_document_Document_ +#define _lucene_document_Document_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include "Field.h" + +///todo: jlucene has change from using DocumentFieldList/Enumeration +///to using a java List... do we want to do this too? +CL_NS_DEF(document) + +class Document; //predefine +class DocumentFieldEnumeration :LUCENE_BASE{ + class DocumentFieldList :LUCENE_BASE{ + public: + DocumentFieldList(Field* f, DocumentFieldList* n); + ~DocumentFieldList(); + Field* field; + DocumentFieldList* next; + }; + friend class Document; +private: + const DocumentFieldList* fields; +public: + DocumentFieldEnumeration(const DocumentFieldList* fl); + ~DocumentFieldEnumeration(); + bool hasMoreElements() const; + Field* nextElement(); +}; + +/** Documents are the unit of indexing and search. +* +* A Document is a set of fields. Each field has a name and a textual value. +* A field may be {@link Field#isStored() stored} with the document, in which +* case it is returned with search hits on the document. Thus each document +* should typically contain one or more stored fields which uniquely identify +* it. +* +* <p>Note that fields which are <i>not</i> {@link Field#isStored() stored} are +* <i>not</i> available in documents retrieved from the index, e.g. with {@link +* Hits#doc(int32_t, Document*)}, {@link Searcher#doc(int32_t, Document*)} or {@link +* IndexReader#document(int32_t, Document*)}. +*/ +class Document:LUCENE_BASE { +private: + DocumentFieldEnumeration::DocumentFieldList* fieldList; + qreal boost; +public: + Document(); + ~Document(); + + /** + * <p>Adds a field to a document. Several fields may be added with + * the same name. In this case, if the fields are indexed, their text is + * treated as though appended for the purposes of search.</p> + * <p> Note that add like the removeField(s) methods only makes sense + * prior to adding a document to an index. These methods cannot + * be used to change the content of an existing index! In order to achieve this, + * a document has to be deleted from an index and a new changed version of that + * document has to be added.</p> + * + */ + void add(Field& field); + /** Returns a field with the given name if any exist in this document, or + * null. If multiple fields exists with this name, this method returns the + * first value added. + * Note: name is case sensitive + */ + Field* getField(const TCHAR* name) const; + + /** Returns the string value of the field with the given name if any exist in + * this document, or null. If multiple fields exist with this name, this + * method returns the first value added. If only binary fields with this name + * exist, returns null. + * Note: name is case sensitive + */ + const TCHAR* get(const TCHAR* field) const; + + /** Returns an Enumeration of all the fields in a document. */ + DocumentFieldEnumeration* fields() const; + /** Prints the fields of a document for human consumption. */ + TCHAR* toString() const; + + /** Sets a boost factor for hits on any field of this document. This value + * will be multiplied into the score of all hits on this document. + * + * <p>Values are multiplied into the value of {@link Field#getBoost()} of + * each field in this document. Thus, this method in effect sets a default + * boost for the fields of this document. + * + * @see Field#setBoost(qreal) + */ + void setBoost(qreal boost); + + /** Returns the boost factor for hits on any field of this document. + * + * <p>The default value is 1.0. + * + * <p>Note: This value is not stored directly with the document in the index. + * Documents returned from {@link IndexReader#document(int32_t, Document*)} and + * {@link Hits#doc(int32_t, Document*)} may thus not have the same value present as when + * this document was indexed. + * + * @see #setBoost(qreal) + */ + qreal getBoost() const; + + + /** + * <p>Removes field with the specified name from the document. + * If multiple fields exist with this name, this method removes the first field that has been added. + * If there is no field with the specified name, the document remains unchanged.</p> + * <p> Note that the removeField(s) methods like the add method only make sense + * prior to adding a document to an index. These methods cannot + * be used to change the content of an existing index! In order to achieve this, + * a document has to be deleted from an index and a new changed version of that + * document has to be added.</p> + * Note: name is case sensitive + */ + void removeField(const TCHAR* name); + + /** + * <p>Removes all fields with the given name from the document. + * If there is no field with the specified name, the document remains unchanged.</p> + * <p> Note that the removeField(s) methods like the add method only make sense + * prior to adding a document to an index. These methods cannot + * be used to change the content of an existing index! In order to achieve this, + * a document has to be deleted from an index and a new changed version of that + * document has to be added.</p> + * Note: name is case sensitive + */ + void removeFields(const TCHAR* name); + + /** + * Returns an array of values of the field specified as the method parameter. + * This method can return <code>null</code>. + * Note: name is case sensitive + * + * @param name the name of the field + * @return a <code>String[]</code> of field values + */ + TCHAR** getValues(const TCHAR* name); + + /** + * Empties out the document so that it can be reused + */ + void clear(); +}; +CL_NS_END +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/document/Field.cpp b/src/assistant/3rdparty/clucene/src/CLucene/document/Field.cpp new file mode 100644 index 000000000..8cd88a36b --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/document/Field.cpp @@ -0,0 +1,315 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#include "CLucene/StdHeader.h" +#include "CLucene/util/Reader.h" +#include "Field.h" +#include "CLucene/util/Misc.h" +#include "CLucene/util/StringIntern.h" +#include "CLucene/util/StringBuffer.h" + +CL_NS_USE(util) +CL_NS_DEF(document) + +Field::Field(const TCHAR* Name, const TCHAR* String, bool store, bool index, bool token, const bool storeTermVector) +{ +//Func - Constructor +//Pre - Name != NULL and contains the name of the field +// String != NULL and contains the value of the field +// store indicates if the field must be stored +// index indicates if the field must be indexed +// token indicates if the field must be tokenized +//Post - The instance has been created + + CND_PRECONDITION(Name != NULL, "Name is NULL"); + CND_PRECONDITION(String != NULL,"String is NULL"); + CND_PRECONDITION(!(!index && storeTermVector),"cannot store a term vector for fields that are not indexed."); + + _name = CLStringIntern::intern( Name CL_FILELINE); + _stringValue = stringDuplicate( String ); + _readerValue = NULL; + _streamValue = NULL; + boost=1.0f; + omitNorms = false; + + int cfg = 0; + if ( store ) + cfg |= STORE_YES; + if ( index && token ) + cfg |= INDEX_TOKENIZED; + else if ( index && !token ) + cfg |= INDEX_UNTOKENIZED; + + if ( storeTermVector ) + _CLTHROWA(CL_ERR_IllegalArgument,"Stored term vector is deprecated with using this constructor"); + + setConfig(cfg); +} + +Field::Field(const TCHAR* Name, Reader* reader, bool store, bool index, bool token, const bool storeTermVector) +{ +//Func - Constructor +//Pre - Name != NULL and contains the name of the field +// reader != NULL and contains a Reader +// store indicates if the field must be stored +// index indicates if the field must be indexed +// token indicates if the field must be tokenized +//Post - The instance has been created + + CND_PRECONDITION(Name != NULL, "Name is NULL"); + CND_PRECONDITION(reader != NULL, "reader is NULL"); + + _name = CLStringIntern::intern( Name CL_FILELINE); + _stringValue = NULL; + _readerValue = reader; + _streamValue = NULL; + boost=1.0f; + omitNorms = false; + + int cfg = 0; + if ( store ) + cfg |= STORE_YES; + if ( index && token ) + cfg |= INDEX_TOKENIZED; + else if ( index && !token ) + cfg |= INDEX_UNTOKENIZED; + + if ( storeTermVector ) + _CLTHROWA(CL_ERR_IllegalArgument,"Stored term vector is deprecated with using this constructor"); + + setConfig(cfg); +} + +Field::Field(const TCHAR* Name, Reader* reader, int config) +{ + CND_PRECONDITION(Name != NULL, "Name is NULL"); + CND_PRECONDITION(reader != NULL, "reader is NULL"); + + _name = CLStringIntern::intern( Name CL_FILELINE); + _stringValue = NULL; + _readerValue = reader; + _streamValue = NULL; + boost=1.0f; + omitNorms = false; + + setConfig(config); +} + + +Field::Field(const TCHAR* Name, const TCHAR* Value, int config) +{ + CND_PRECONDITION(Name != NULL, "Name is NULL"); + CND_PRECONDITION(Value != NULL, "value is NULL"); + + _name = CLStringIntern::intern( Name CL_FILELINE); + _stringValue = stringDuplicate( Value ); + _readerValue = NULL; + _streamValue = NULL; + boost=1.0f; + omitNorms = false; + + setConfig(config); +} + +Field::Field(const TCHAR* Name, jstreams::StreamBase<char>* Value, int config) +{ + CND_PRECONDITION(Name != NULL, "Name is NULL"); + CND_PRECONDITION(Value != NULL, "value is NULL"); + + _name = CLStringIntern::intern( Name CL_FILELINE); + _stringValue = NULL; + _readerValue = NULL; + _streamValue = Value; + boost=1.0f; + omitNorms = false; + + setConfig(config); +} + +Field::~Field(){ +//Func - Destructor +//Pre - true +//Post - Instance has been destroyed + + CLStringIntern::unintern(_name); + _CLDELETE_CARRAY(_stringValue); + _CLDELETE(_readerValue); + _CLVDELETE( _streamValue ); +} + + +/*===============FIELDS=======================*/ +const TCHAR* Field::name() { return _name; } ///<returns reference +TCHAR* Field::stringValue() { return _stringValue; } ///<returns reference +Reader* Field::readerValue() { return _readerValue; } ///<returns reference +jstreams::StreamBase<char>* Field::streamValue() { return _streamValue; } ///<returns reference + +bool Field::isStored() { return (config & STORE_YES) != 0; } +bool Field::isIndexed() { return (config & INDEX_TOKENIZED)!=0 || (config & INDEX_UNTOKENIZED)!=0; } +bool Field::isTokenized() { return (config & INDEX_TOKENIZED) != 0; } +bool Field::isCompressed() { return (config & STORE_COMPRESS) != 0; } +bool Field::isBinary() { return _streamValue!=NULL; } + +bool Field::isTermVectorStored() { return (config & TERMVECTOR_YES) != 0; } +bool Field::isStoreOffsetWithTermVector() { return (config & TERMVECTOR_YES) != 0 && (config & TERMVECTOR_WITH_OFFSETS) != 0; } +bool Field::isStorePositionWithTermVector() { return (config & TERMVECTOR_YES) != 0 && (config & TERMVECTOR_WITH_POSITIONS) != 0; } + +bool Field::getOmitNorms() { return omitNorms; } +void Field::setOmitNorms(bool omitNorms) { this->omitNorms=omitNorms; } + +void Field::setBoost(qreal boost) { this->boost = boost; } +qreal Field::getBoost() { return boost; } + +void Field::setConfig(int x){ + int newConfig=0; + + //set storage settings + if ( (x & STORE_YES) || (x & STORE_COMPRESS) ){ + newConfig |= STORE_YES; + if ( x & STORE_COMPRESS ) + newConfig |= STORE_COMPRESS; + }else + newConfig |= STORE_NO; + + if ( (x & INDEX_NO)==0 ){ + bool index=false; + + if ( x & INDEX_NONORMS ){ + newConfig |= INDEX_NONORMS; + index = true; + } + + if ( x & INDEX_TOKENIZED && x & INDEX_UNTOKENIZED ) + _CLTHROWA(CL_ERR_IllegalArgument,"it doesn't make sense to have an untokenised and tokenised field"); + if ( x & INDEX_TOKENIZED ){ + newConfig |= INDEX_TOKENIZED; + index = true; + } + if ( x & INDEX_UNTOKENIZED ){ + newConfig |= INDEX_UNTOKENIZED; + index = true; + } + if ( !index ) + newConfig |= INDEX_NO; + }else + newConfig |= INDEX_NO; + + if ( newConfig & INDEX_NO && newConfig & STORE_NO ) + _CLTHROWA(CL_ERR_IllegalArgument,"it doesn't make sense to have a field that is neither indexed nor stored"); + + //set termvector settings + if ( (x & TERMVECTOR_NO) == 0 ){ + bool termVector=false; + if ( x & TERMVECTOR_YES ){ + termVector=true; + } + if ( x & TERMVECTOR_WITH_OFFSETS ){ + newConfig |= TERMVECTOR_WITH_OFFSETS; + termVector=true; + } + if ( x & TERMVECTOR_WITH_POSITIONS ){ + newConfig |= TERMVECTOR_WITH_POSITIONS; + termVector=true; + } + if ( termVector ){ + if ( newConfig & INDEX_NO ) + _CLTHROWA(CL_ERR_IllegalArgument,"cannot store a term vector for fields that are not indexed."); + + newConfig |= TERMVECTOR_YES; + }else + newConfig |= TERMVECTOR_NO; + }else + newConfig |= TERMVECTOR_NO; + + config = newConfig; +} + +TCHAR* Field::toString() { + CL_NS(util)::StringBuffer result; + if (isStored()) { + result.append( _T("stored") ); + } + if (isIndexed()) { + if (result.length() > 0) + result.append( _T(",") ); + result.append( _T("indexed") ); + } + if (isTokenized()) { + if (result.length() > 0) + result.append( _T(",") ); + result.append( _T("tokenized") ); + } + if (isTermVectorStored()) { + if (result.length() > 0) + result.append( _T(",") ); + result.append( _T("termVector") ); + } + if (isStoreOffsetWithTermVector()) { + if (result.length() > 0) + result.appendChar( ',' ); + result.append( _T("termVectorOffsets") ); + } + if (isStorePositionWithTermVector()) { + if (result.length() > 0) + result.appendChar( ',' ); + result.append( _T("termVectorPosition") ); + } + if (isBinary()) { + if (result.length() > 0) + result.appendChar( ',' ); + result.append( _T("binary") ); + } + if (getOmitNorms()) { + result.append( _T(",omitNorms") ); + } + result.appendChar('<'); + result.append(name()); + result.appendChar(':'); + + if (_stringValue != NULL) + result.append(_stringValue); + else if ( _readerValue != NULL ) + result.append( _T("Reader") ); + else if ( _streamValue != NULL ) + result.append( _T("Stream") ); + else + result.append( _T("NULL") ); + + result.appendChar('>'); + return result.toString(); +} + + +Field* Field::Keyword(const TCHAR* Name, const TCHAR* Value) { + return _CLNEW Field(Name,Value,Field::STORE_YES | Field::INDEX_UNTOKENIZED); +} + +Field* Field::UnIndexed(const TCHAR* Name, const TCHAR* Value) { + return _CLNEW Field(Name,Value,Field::STORE_YES | Field::INDEX_NO); +} + +Field* Field::Text(const TCHAR* Name, const TCHAR* Value, const bool storeTermVector) { + if ( storeTermVector ) + return _CLNEW Field(Name,Value,Field::STORE_YES | Field::INDEX_TOKENIZED | Field::TERMVECTOR_YES); + else + return _CLNEW Field(Name,Value,Field::STORE_YES | Field::INDEX_TOKENIZED); +} + +Field* Field::UnStored(const TCHAR* Name, const TCHAR* Value, const bool storeTermVector) { + if ( storeTermVector ) + return _CLNEW Field(Name,Value,Field::STORE_NO | Field::INDEX_TOKENIZED | Field::TERMVECTOR_YES); + else + return _CLNEW Field(Name,Value,Field::STORE_NO | Field::INDEX_TOKENIZED); +} + +Field* Field::Text(const TCHAR* Name, Reader* Value, const bool storeTermVector) { + if ( storeTermVector ) + return _CLNEW Field(Name,Value,Field::INDEX_TOKENIZED | Field::TERMVECTOR_YES); + else + return _CLNEW Field(Name,Value,Field::INDEX_TOKENIZED); +} + +CL_NS_END diff --git a/src/assistant/3rdparty/clucene/src/CLucene/document/Field.h b/src/assistant/3rdparty/clucene/src/CLucene/document/Field.h new file mode 100644 index 000000000..771a1382b --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/document/Field.h @@ -0,0 +1,261 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _lucene_document_Field_ +#define _lucene_document_Field_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include "CLucene/util/Reader.h" +#include "CLucene/util/streambase.h" + +CL_NS_DEF(document) +/** +A field is a section of a Document. Each field has two parts, a name and a +value. Values may be free text, provided as a String or as a Reader, or they +may be atomic keywords, which are not further processed. Such keywords may +be used to represent dates, urls, etc. Fields are optionally stored in the +index, so that they may be returned with hits on the document. + +PORTING: CLucene doesn't directly support compressed fields. However, it is easy +to reproduce this functionality by using the GZip streams in the contrib package. +Also note that binary fields are not read immediately in CLucene, a substream +is pointed directly to the field's data, in affect creating a lazy load ability. +This means that large fields are best saved in binary format (even if they are +text), so that they can be loaded lazily. +*/ +class Field :LUCENE_BASE{ +private: + const TCHAR* _name; + TCHAR* _stringValue; + CL_NS(util)::Reader* _readerValue; + jstreams::StreamBase<char>* _streamValue; + + int config; + qreal boost; + bool omitNorms; +public: + enum Store{ + /** Store the original field value in the index. This is useful for short texts + * like a document's title which should be displayed with the results. The + * value is stored in its original form, i.e. no analyzer is used before it is + * stored. + */ + STORE_YES=1, + /** Do not store the field value in the index. */ + STORE_NO=2, + + /** Store the original field value in the index in a compressed form. This is + * useful for long documents and for binary valued fields. + * NOTE: CLucene does not directly support compressed fields, to store a + * compressed field. + * //TODO: need better documentation on how to add a compressed field + * //because actually we still need to write a GZipOutputStream... + */ + STORE_COMPRESS=4 + }; + + enum Index{ + /** Do not index the field value. This field can thus not be searched, + * but one can still access its contents provided it is + * {@link Field::Store stored}. */ + INDEX_NO=16, + /** Index the field's value so it can be searched. An Analyzer will be used + * to tokenize and possibly further normalize the text before its + * terms will be stored in the index. This is useful for common text. + */ + INDEX_TOKENIZED=32, + /** Index the field's value without using an Analyzer, so it can be searched. + * As no analyzer is used the value will be stored as a single term. This is + * useful for unique Ids like product numbers. + */ + INDEX_UNTOKENIZED=64, + /** Index the field's value without an Analyzer, and disable + * the storing of norms. No norms means that index-time boosting + * and field length normalization will be disabled. The benefit is + * less memory usage as norms take up one byte per indexed field + * for every document in the index. + */ + INDEX_NONORMS=128 + }; + + enum TermVector{ + /** Do not store term vectors. */ + TERMVECTOR_NO=256, + /** Store the term vectors of each document. A term vector is a list + * of the document's terms and their number of occurences in that document. */ + TERMVECTOR_YES=512, + /** + * Store the term vector + token position information + * + * @see #YES + */ + TERMVECTOR_WITH_POSITIONS=1024, + /** + * Store the term vector + Token offset information + * + * @see #YES + */ + TERMVECTOR_WITH_OFFSETS=2048 + }; + + _CL_DEPRECATED( another overload ) Field(const TCHAR* name, const TCHAR* value, bool store, bool index, bool token, const bool storeTermVector=false); + _CL_DEPRECATED( another overload ) Field(const TCHAR* name, CL_NS(util)::Reader* reader, bool store, bool index, bool token, const bool storeTermVector=false); + + Field(const TCHAR* name, const TCHAR* value, int configs); + Field(const TCHAR* name, CL_NS(util)::Reader* reader, int configs); + Field(const TCHAR* name, jstreams::StreamBase<char>* stream, int configs); + ~Field(); + + /** Constructs a String-valued Field that is not tokenized, but is indexed + * and stored. Useful for non-text fields, e.g. date or url. + * @deprecated Use new Field(name,value,Field::STORE_YES | Field::INDEX_UNTOKENIZED) + */ + _CL_DEPRECATED( new Field(*) ) static Field* Keyword(const TCHAR* name, const TCHAR* value); + + /** Constructs a String-valued Field that is not tokenized nor indexed, + * but is stored in the index, for return with hits. + * @deprecated Use new Field(name,value,Field::STORE_YES | Field::INDEX_NO) + */ + _CL_DEPRECATED( new Field(*) ) static Field* UnIndexed(const TCHAR* name, const TCHAR* value); + + /** Constructs a String-valued Field that is tokenized and indexed, + * and is stored in the index, for return with hits. Useful for short text + * fields, like "title" or "subject". + * @deprecated Use new Field(name,value,Field::STORE_YES | Field::INDEX_TOKENIZED) + */ + _CL_DEPRECATED( new Field(*) ) static Field* Text(const TCHAR* name, const TCHAR* value, const bool storeTermVector=false); + + /** Constructs a String-valued Field that is tokenized and indexed, + * but that is not stored in the index. + * @deprecated Use new Field(name,value,Field::STORE_NO | Field::INDEX_TOKENIZED) + */ + _CL_DEPRECATED( new Field(*) ) static Field* UnStored(const TCHAR* name, const TCHAR* value, const bool storeTermVector=false); + + /** Constructs a Reader-valued Field that is tokenized and indexed, but is + * *not* stored in the index verbatim. Useful for longer text fields, like + * "body". + * @deprecated Use new Field(name,value, Field::INDEX_TOKENIZED) + */ + _CL_DEPRECATED( new Field(*) ) static Field* Text(const TCHAR* name, CL_NS(util)::Reader* value, const bool storeTermVector=false); + + /** The name of the field (e.g., "date", "subject", "title", "body", etc.) + * as an interned string. */ + const TCHAR* name(); ///<returns reference + + /** The value of the field as a String, or null. If null, the Reader value + * or binary value is used. Exactly one of stringValue(), readerValue() and + * streamValue() must be set. */ + TCHAR* stringValue(); ///<returns reference + + /** The value of the field as a reader, or null. If null, the String value + * or stream value is used. Exactly one of stringValue(), readerValue() and + * streamValue() must be set. */ + CL_NS(util)::Reader* readerValue(); + + /** The value of the field as a String, or null. If null, the String value + * or Reader value is used. Exactly one of stringValue(), readerValue() and + * streamValue() must be set. */ + jstreams::StreamBase<char>* streamValue(); + + // True iff the value of the field is to be stored in the index for return + // with search hits. It is an error for this to be true if a field is + // Reader-valued. + bool isStored(); + + // True iff the value of the field is to be indexed, so that it may be + // searched on. + bool isIndexed(); + + // True iff the value of the field should be tokenized as text prior to + // indexing. Un-tokenized fields are indexed as a single word and may not be + // Reader-valued. + bool isTokenized(); + + /** True if the value of the field is stored and compressed within the index + * NOTE: CLucene does not actually support compressed fields, Instead, a reader + * will be returned with a pointer to a SubIndexInputStream. A GZipInputStream + * and a UTF8 reader must be used to actually read the content. This flag + * will only be set if the index was created by another lucene implementation. + */ + bool isCompressed(); + + //Set configs using XOR. This resets all the settings + //For example, to use term vectors with positions and offsets do: + //object->setConfig(TERMVECTOR_WITH_POSITIONS | TERMVECTOR_WITH_OFFSETS); + void setConfig(int termVector); + + /** True iff the term or terms used to index this field are stored as a term + * vector, available from {@link IndexReader#getTermFreqVector(int32_t,TCHAR*)}. + * These methods do not provide access to the original content of the field, + * only to terms used to index it. If the original content must be + * preserved, use the <code>stored</code> attribute instead. + * + * @see IndexReader#getTermFreqVector(int32_t, String) + */ + bool isTermVectorStored(); + + /** + * True iff terms are stored as term vector together with their offsets + * (start and end positon in source text). + */ + bool isStoreOffsetWithTermVector(); + + /** + * True iff terms are stored as term vector together with their token positions. + */ + bool isStorePositionWithTermVector(); + + /** Returns the boost factor for hits for this field. + * + * <p>The default value is 1.0. + * + * <p>Note: this value is not stored directly with the document in the index. + * Documents returned from {@link IndexReader#document(int)} and + * {@link Hits#doc(int)} may thus not have the same value present as when + * this field was indexed. + * + * @see #setBoost(float) + */ + qreal getBoost(); + + /** Sets the boost factor hits on this field. This value will be + * multiplied into the score of all hits on this field of this document. + * + * <p>The boost is multiplied by {@link Document#getBoost()} of the document + * containing this field. If a document has multiple fields with the same + * name, all such values are multiplied together. This product is then + * multipled by the value {@link Similarity#lengthNorm(String,int)}, and + * rounded by {@link Similarity#encodeNorm(float)} before it is stored in the + * index. One should attempt to ensure that this product does not overflow + * the range of that encoding. + * + * @see Document#setBoost(float) + * @see Similarity#lengthNorm(String, int) + * @see Similarity#encodeNorm(float) + */ + void setBoost(qreal value); + + /** True iff the value of the filed is stored as binary */ + bool isBinary(); + + /** True if norms are omitted for this indexed field */ + bool getOmitNorms(); + + /** Expert: + * + * If set, omit normalization factors associated with this indexed field. + * This effectively disables indexing boosts and length normalization for this field. + */ + void setOmitNorms(bool omitNorms); + + // Prints a Field for human consumption. + TCHAR* toString(); +}; +CL_NS_END +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/index/CompoundFile.cpp b/src/assistant/3rdparty/clucene/src/CLucene/index/CompoundFile.cpp new file mode 100644 index 000000000..efa0e2563 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/index/CompoundFile.cpp @@ -0,0 +1,380 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +* +* Changes are Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies). +------------------------------------------------------------------------------*/ +#include "CLucene/StdHeader.h" +#include "CompoundFile.h" +#include "CLucene/util/Misc.h" + +CL_NS_USE(store) +CL_NS_USE(util) +CL_NS_DEF(index) + +CompoundFileReader::CSIndexInput::CSIndexInput(CL_NS(store)::IndexInput* base, + const int64_t fileOffset, const int64_t length) +{ + this->base = base; + this->fileOffset = fileOffset; + this->_length = length; +} + +void CompoundFileReader::CSIndexInput::readInternal(uint8_t* b, const int32_t len) +{ + SCOPED_LOCK_MUTEX(base->THIS_LOCK) + + int64_t start = getFilePointer(); + if(start + len > _length) + _CLTHROWA(CL_ERR_IO, "read past EOF"); + base->seek(fileOffset + start); + base->readBytes(b, len); +} + +CompoundFileReader::CSIndexInput::~CSIndexInput() +{ +} + +IndexInput* CompoundFileReader::CSIndexInput::clone() const +{ + return _CLNEW CSIndexInput(*this); +} + +CompoundFileReader::CSIndexInput::CSIndexInput(const CSIndexInput& clone) + : BufferedIndexInput(clone) +{ + this->base = clone.base; //no need to clone this.. + this->fileOffset = clone.fileOffset; + this->_length = clone._length; +} + +void CompoundFileReader::CSIndexInput::close() +{ +} + +CompoundFileReader::CompoundFileReader(Directory* dir, const QString& name) + : entries(false, true) +{ + directory = dir; + fileName = name; + + bool success = false; + try { + stream = dir->openInput(name); + + // read the directory and init files + int32_t count = stream->readVInt(); + FileEntry* entry = NULL; + TCHAR tid[CL_MAX_PATH]; + for (int32_t i = 0; i < count; i++) { + int64_t offset = stream->readLong(); + int32_t read = stream->readString(tid, CL_MAX_PATH); + QString aid(QString::fromWCharArray(tid, read)); + + // set length of the previous entry + if (entry != NULL) + entry->length = offset - entry->offset; + + entry = _CLNEW FileEntry(offset); + entries.put(aid, entry); + } + + // set the length of the final entry + if (entry != NULL) + entry->length = stream->length() - entry->offset; + success = true; + } _CLFINALLY ( + if (!success && (stream != NULL)) { + try { + stream->close(); + _CLDELETE(stream); + } catch (CLuceneError& err) { + if (err.number() != CL_ERR_IO) + throw err; + } + } + ) +} + +CompoundFileReader::~CompoundFileReader() +{ + close(); +} + +Directory* CompoundFileReader::getDirectory() +{ + return directory; +} + +QString CompoundFileReader::getName() const +{ + return fileName; +} + +void CompoundFileReader::close() +{ + SCOPED_LOCK_MUTEX(THIS_LOCK) + + if (stream != NULL) { + entries.clear(); + stream->close(); + _CLDELETE(stream); + } +} + +IndexInput* CompoundFileReader::openInput(const QString& id) +{ + SCOPED_LOCK_MUTEX(THIS_LOCK) + + if (stream == NULL) + _CLTHROWA(CL_ERR_IO, "Stream closed"); + + const FileEntry* entry = entries.get(id); + if (entry == NULL) { + char buf[CL_MAX_PATH + 30]; + strcpy(buf,"No sub-file with id "); + strncat(buf, id.toLocal8Bit().constData(), CL_MAX_PATH); + strcat(buf, " found"); + _CLTHROWA(CL_ERR_IO,buf); + } + return _CLNEW CSIndexInput(stream, entry->offset, entry->length); +} + +QStringList CompoundFileReader::list() const +{ + // for (EntriesType::const_iterator i=entries.begin();i!=entries.end();i++){ + // names->push_back(i->first); + // ++i; + // } + + QStringList names; + EntriesType::const_iterator itr; + // TODO: verify this, see old code above ??? + for (itr = entries.begin(); itr != entries.end(); ++itr) + names.push_back(itr->first); + + return names; +} + +bool CompoundFileReader::fileExists(const QString& name) const +{ + return entries.exists(name); +} + +int64_t CompoundFileReader::fileModified(const QString& name) const +{ + return directory->fileModified(fileName); +} + +void CompoundFileReader::touchFile(const QString& name) +{ + directory->touchFile(fileName); +} + +bool CompoundFileReader::doDeleteFile(const QString& name) +{ + _CLTHROWA(CL_ERR_UnsupportedOperation, + "UnsupportedOperationException: CompoundFileReader::doDeleteFile"); +} + +void CompoundFileReader::renameFile(const QString& from, const QString& to) +{ + _CLTHROWA(CL_ERR_UnsupportedOperation, + "UnsupportedOperationException: CompoundFileReader::renameFile"); +} + +int64_t CompoundFileReader::fileLength(const QString& name) const +{ + FileEntry* e = entries.get(name); + if (e == NULL) { + char buf[CL_MAX_PATH + 30]; + strcpy(buf,"File "); + strncat(buf, name.toLocal8Bit().constData(), CL_MAX_PATH); + strcat(buf," does not exist"); + _CLTHROWA(CL_ERR_IO,buf); + } + return e->length; +} + +IndexOutput* CompoundFileReader::createOutput(const QString& name) +{ + _CLTHROWA(CL_ERR_UnsupportedOperation, + "UnsupportedOperationException: CompoundFileReader::createOutput"); +} + +LuceneLock* CompoundFileReader::makeLock(const QString& name) +{ + _CLTHROWA(CL_ERR_UnsupportedOperation, + "UnsupportedOperationException: CompoundFileReader::makeLock"); +} + +QString CompoundFileReader::toString() const +{ + QString ret(QLatin1String("CompoundFileReader@")); + return ret.append(fileName); +} + +CompoundFileWriter::CompoundFileWriter(Directory* dir, const QString& name) + : ids(false) + , entries(true) +{ + if (dir == NULL) + _CLTHROWA(CL_ERR_NullPointer, "directory cannot be null"); + + if (name.isEmpty()) + _CLTHROWA(CL_ERR_NullPointer, "name cannot be null"); + + merged = false; + directory = dir; + fileName = name; +} + +CompoundFileWriter::~CompoundFileWriter() +{ +} + +Directory* CompoundFileWriter::getDirectory() +{ + return directory; +} + +/** Returns the name of the compound file. */ +QString CompoundFileWriter::getName() const +{ + return fileName; +} + +void CompoundFileWriter::addFile(const QString& file) +{ + if (merged) + _CLTHROWA(CL_ERR_IO, "Can't add extensions after merge has been called"); + + if (file.isEmpty()) + _CLTHROWA(CL_ERR_NullPointer, "file cannot be null"); + + if (ids.find(file) != ids.end()) { + char buf[CL_MAX_PATH + 30]; + strcpy(buf, "File "); + strncat(buf, file.toLocal8Bit().constData(), CL_MAX_PATH); + strcat(buf," already added"); + _CLTHROWA(CL_ERR_IO,buf); + } + ids.insert(file); + entries.push_back(_CLNEW WriterFileEntry(file)); +} + +void CompoundFileWriter::close() +{ + if (merged) + _CLTHROWA(CL_ERR_IO, "Merge already performed"); + + if (entries.size() == 0) // isEmpty() + _CLTHROWA(CL_ERR_IO, "No entries to merge have been defined"); + + merged = true; + + // open the compound stream + IndexOutput* os = NULL; + try { + os = directory->createOutput(fileName); + + // Write the number of entries + os->writeVInt(entries.size()); + + // Write the directory with all offsets at 0. + // Remember the positions of directory entries so that we can + // adjust the offsets later + { //msvc6 for scope fix + TCHAR tfile[CL_MAX_PATH]; + for (CLLinkedList<WriterFileEntry*>::iterator i = entries.begin(); + i != entries.end(); i++) { + WriterFileEntry* fe = *i; + fe->directoryOffset = os->getFilePointer(); + os->writeLong(0); // for now + tfile[fe->file.toWCharArray(tfile)] = '\0'; + os->writeString(tfile, _tcslen(tfile)); + } + } + + // Open the files and copy their data into the stream. + // Remember the locations of each file's data section. + { //msvc6 for scope fix + int32_t bufferLength = 1024; + uint8_t buffer[1024]; + for (CLLinkedList<WriterFileEntry*>::iterator i = entries.begin(); + i != entries.end(); i++) { + WriterFileEntry* fe = *i; + fe->dataOffset = os->getFilePointer(); + copyFile(fe, os, buffer, bufferLength); + } + } + + { //msvc6 for scope fix + // Write the data offsets into the directory of the compound stream + for (CLLinkedList<WriterFileEntry*>::iterator i = entries.begin(); + i != entries.end(); i++) { + WriterFileEntry* fe = *i; + os->seek(fe->directoryOffset); + os->writeLong(fe->dataOffset); + } + } + + + } _CLFINALLY ( + if (os != NULL) { + try { + os->close(); + _CLDELETE(os); + } catch (...) { } + } + ); +} + +void CompoundFileWriter::copyFile(WriterFileEntry* source, IndexOutput* os, + uint8_t* buffer, int32_t bufferLength) +{ + IndexInput* is = NULL; + try { + int64_t startPtr = os->getFilePointer(); + + is = directory->openInput(source->file); + int64_t length = is->length(); + int64_t remainder = length; + int32_t chunk = bufferLength; + + while(remainder > 0) { + int32_t len = (int32_t)min((int64_t)chunk, remainder); + is->readBytes(buffer, len); + os->writeBytes(buffer, len); + remainder -= len; + } + + // Verify that remainder is 0 + if (remainder != 0) { + TCHAR buf[CL_MAX_PATH+100]; + _sntprintf(buf, CL_MAX_PATH + 100, _T("Non-zero remainder length ") + _T("after copying: %d (id: %s, length: %d, buffer size: %d)"), + remainder, source->file.toLocal8Bit().constData(), length, chunk); + _CLTHROWT(CL_ERR_IO, buf); + } + + // Verify that the output length diff is equal to original file + int64_t endPtr = os->getFilePointer(); + int64_t diff = endPtr - startPtr; + if (diff != length) { + TCHAR buf[100]; + _sntprintf(buf, 100, _T("Difference in the output file offsets %d ") + _T("does not match the original file length %d"), diff, length); + _CLTHROWT(CL_ERR_IO,buf); + } + } _CLFINALLY ( + if (is != NULL) { + is->close(); + _CLDELETE(is); + } + ); +} + +CL_NS_END diff --git a/src/assistant/3rdparty/clucene/src/CLucene/index/CompoundFile.h b/src/assistant/3rdparty/clucene/src/CLucene/index/CompoundFile.h new file mode 100644 index 000000000..84799a62f --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/index/CompoundFile.h @@ -0,0 +1,219 @@ +/* + * Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team + * + * Distributable under the terms of either the Apache License (Version 2.0) or + * the GNU Lesser General Public License, as specified in the COPYING file. + * + * Changes are Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies). +*/ +#ifndef _lucene_index_compoundfile_h +#define _lucene_index_compoundfile_h + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include <QtCore/QString> +#include <QtCore/QStringList> + +#include "CLucene/store/IndexInput.h" +#include "CLucene/store/IndexOutput.h" +#include "CLucene/store/Directory.h" +#include "CLucene/store/Lock.h" +#include "CLucene/util/VoidList.h" +#include "CLucene/util/VoidMap.h" + +CL_NS_DEF(index) + +// Class for accessing a compound stream. +// This class implements a directory, but is limited to only read operations. +// Directory methods that would normally modify data throw an exception. +class CompoundFileReader : public CL_NS(store)::Directory +{ +private: + /** Implementation of an IndexInput that reads from a portion of the + * compound file. The visibility is left as "package" *only* because + * this helps with testing since JUnit test cases in a different class + * can then access package fields of this class. + */ + class CSIndexInput : public CL_NS(store)::BufferedIndexInput + { + private: + CL_NS(store)::IndexInput* base; + int64_t fileOffset; + int64_t _length; + protected: + /** Expert: implements buffer refill. Reads uint8_ts from the current + * position in the input. + * @param b the array to read uint8_ts into + * @param length the number of uint8_ts to read + */ + void readInternal(uint8_t* b, const int32_t len); + void seekInternal(const int64_t pos) {} + + public: + CSIndexInput(CL_NS(store)::IndexInput* base, const int64_t fileOffset, + const int64_t length); + CSIndexInput(const CSIndexInput& clone); + ~CSIndexInput(); + + /** Closes the stream to futher operations. */ + void close(); + CL_NS(store)::IndexInput* clone() const; + + int64_t length() { return _length; } + + QString getDirectoryType() const { + return CompoundFileReader::DirectoryType(); } + }; + + class FileEntry : LUCENE_BASE + { + public: + FileEntry() + : offset(0), length(0) {} + FileEntry(int64_t _offset) + : offset(_offset), length(0) {} + ~FileEntry() {} + + int64_t offset; + int64_t length; + }; + + // Base info + CL_NS(store)::Directory* directory; + QString fileName; + + CL_NS(store)::IndexInput* stream; + + typedef CL_NS(util)::CLHashMap<QString, FileEntry*, + CL_NS(util)::Compare::Qstring, + CL_NS(util)::Equals::Qstring, + CL_NS(util)::Deletor::DummyQString, + CL_NS(util)::Deletor::Object<FileEntry> > EntriesType; + EntriesType entries; +protected: + // Removes an existing file in the directory-> + bool doDeleteFile(const QString& name); + +public: + CompoundFileReader(CL_NS(store)::Directory* dir, const QString& name); + ~CompoundFileReader(); + CL_NS(store)::Directory* getDirectory(); + QString getName() const; + + void close(); + CL_NS(store)::IndexInput* openInput(const QString& id); + + /** Returns an array of strings, one for each file in the directory-> */ + QStringList list() const; + /** Returns true iff a file with the given name exists. */ + bool fileExists(const QString& name) const; + /** Returns the time the named file was last modified. */ + int64_t fileModified(const QString& name) const; + /** Set the modified time of an existing file to now. */ + void touchFile(const QString& name); + /** Renames an existing file in the directory-> + If a file already exists with the new name, then it is replaced. + This replacement should be atomic. */ + void renameFile(const QString& from, const QString& to); + /** Returns the length of a file in the directory. + * @throws IOException if the file does not exist */ + int64_t fileLength(const QString& name) const; + /** Not implemented + * @throws UnsupportedOperationException */ + CL_NS(store)::IndexOutput* createOutput(const QString& name); + /** Not implemented + * @throws UnsupportedOperationException */ + CL_NS(store)::LuceneLock* makeLock(const QString& name); + + QString toString() const; + + static QString DirectoryType() { return QLatin1String("CFS"); } + QString getDirectoryType() const { return DirectoryType(); } +}; + + + +// Combines multiple files into a single compound file. +// The file format:<br> +// <ul> +// <li>VInt fileCount</li> +// <li>{Directory} +// fileCount entries with the following structure:</li> +// <ul> +// <li>int64_t dataOffset</li> +// <li>UTFString extension</li> +// </ul> +// <li>{File Data} +// fileCount entries with the raw data of the corresponding file</li> +// </ul> +// The fileCount integer indicates how many files are contained in this compound +// file. The {directory} that follows has that many entries. Each directory entry +// contains an encoding identifier, an int64_t pointer to the start of this file's +// data section, and a UTF String with that file's extension. +class CompoundFileWriter : LUCENE_BASE +{ + class WriterFileEntry : LUCENE_BASE { + public: + WriterFileEntry() + : dataOffset(0), directoryOffset(0) {} + WriterFileEntry(const QString& _file) + : file(_file), dataOffset(0), directoryOffset(0) {} + ~WriterFileEntry() {} + + QString file; + // temporary holder for the start of this file's data section + int64_t dataOffset; + // temporary holder for the start of directory entry for this file + int64_t directoryOffset; + }; + + bool merged; + QString fileName; + CL_NS(store)::Directory* directory; + + CL_NS(util)::CLHashSet<QString, CL_NS(util)::Compare::Qstring, + CL_NS(util)::Deletor::DummyQString> ids; + + CL_NS(util)::CLLinkedList<WriterFileEntry*, + CL_NS(util)::Deletor::Object<WriterFileEntry> > entries; + + // Copy the contents of the file with specified extension into the + // provided output stream. Use the provided buffer for moving data + // to reduce memory allocation. + void copyFile(WriterFileEntry* source, CL_NS(store)::IndexOutput* os, + uint8_t* buffer, int32_t bufferLength); + +public: + // Create the compound stream in the specified file. The file name is the + // entire name (no extensions are added). + CompoundFileWriter(CL_NS(store)::Directory* dir, const QString& name); + ~CompoundFileWriter(); + + // Returns the directory of the compound file. + CL_NS(store)::Directory* getDirectory(); + + QString getName() const ; + /* Add a source stream. <code>file</code> is the string by which the + * sub-stream will be known in the compound stream. + * + * @throws IllegalStateException if this writer is closed + * @throws NullPointerException if <code>file</code> is null + * @throws IllegalArgumentException if a file with the same name + * has been added already + */ + void addFile(const QString& file); + /* Merge files with the extensions added up to now. + * All files with these extensions are combined sequentially into the + * compound stream. After successful merge, the source files + * @throws IllegalStateException if close() had been called before or + * if no file has been added to this object + * are deleted. + */ + void close(); +}; + +CL_NS_END + +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/index/DocumentWriter.cpp b/src/assistant/3rdparty/clucene/src/CLucene/index/DocumentWriter.cpp new file mode 100644 index 000000000..dcbc31591 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/index/DocumentWriter.cpp @@ -0,0 +1,571 @@ +/* + * Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team + * + * Distributable under the terms of either the Apache License (Version 2.0) or + * the GNU Lesser General Public License, as specified in the COPYING file. + * + * Changes are Copyright(C) 2007, 2008 by Nokia Corporation and/or its subsidiary(-ies), all rights reserved. +*/ +#include "CLucene/StdHeader.h" + +#include "DocumentWriter.h" +#include "FieldInfos.h" +#include "IndexWriter.h" +#include "FieldsWriter.h" +#include "Term.h" +#include "TermInfo.h" +#include "TermInfosWriter.h" + +#include "CLucene/analysis/AnalysisHeader.h" + +#include "CLucene/search/Similarity.h" +#include "TermInfosWriter.h" +#include "FieldsWriter.h" + +CL_NS_USE(util) +CL_NS_USE(store) +CL_NS_USE(analysis) +CL_NS_USE(document) +CL_NS_DEF(index) + +/*Posting*/ + +DocumentWriter::Posting::Posting(Term* t, const int32_t position, + TermVectorOffsetInfo* offset) +{ + //Func - Constructor + //Pre - t contains a valid reference to a Term + //Post - Instance has been created + freq = 1; + + term = _CL_POINTER(t); + positions.values = (int32_t*)malloc(sizeof(int32_t)); + positions.values[0] = position; + positions.length = 1; + + if ( offset != NULL ){ + this->offsets.values =(TermVectorOffsetInfo*)malloc(sizeof(TermVectorOffsetInfo)); + this->offsets.values[0] = *offset; + this->offsets.length = 1; + } +} + +DocumentWriter::Posting::~Posting() +{ + //Func - Destructor + //Pre - true + //Post - The instance has been destroyed + + free(positions.values); + if ( this->offsets.values != NULL ) + free(this->offsets.values); + _CLDECDELETE(term); +} + +DocumentWriter::DocumentWriter(Directory* d, Analyzer* a, + CL_NS(search)::Similarity* sim, const int32_t mfl) + : analyzer(a) + , directory(d) + , maxFieldLength(mfl) + , fieldInfos(NULL) + , fieldLengths(NULL) + , similarity(sim) + , termIndexInterval(IndexWriter::DEFAULT_TERM_INDEX_INTERVAL) + , fieldPositions(NULL) + , fieldBoosts(NULL) + , termBuffer(_CLNEW Term) +{ + //Pre - d contains a valid reference to a Directory + // d contains a valid reference to a Analyzer + // mfl > 0 and contains the maximum field length + //Post - Instance has been created + + CND_PRECONDITION(((mfl > 0) || (mfl == IndexWriter::FIELD_TRUNC_POLICY__WARN)), + "mfl is 0 or smaller than IndexWriter::FIELD_TRUNC_POLICY__WARN") + + fieldInfos = NULL; + fieldLengths = NULL; +} + +DocumentWriter::DocumentWriter(CL_NS(store)::Directory* d, + CL_NS(analysis)::Analyzer* a, IndexWriter* writer) + : analyzer(a) + , directory(d) + , maxFieldLength(writer->getMaxFieldLength()) + , fieldInfos(NULL) + , fieldLengths(NULL) + , similarity(writer->getSimilarity()) + , termIndexInterval(writer->getTermIndexInterval()) + , fieldPositions(NULL) + , fieldBoosts(NULL) + , termBuffer(_CLNEW Term) +{ + //Pre - d contains a valid reference to a Directory + // d contains a valid reference to a Analyzer + // mfl > 0 and contains the maximum field length + //Post - Instance has been created + + CND_PRECONDITION(((maxFieldLength > 0) + || (maxFieldLength == IndexWriter::FIELD_TRUNC_POLICY__WARN)), + "mfl is 0 or smaller than IndexWriter::FIELD_TRUNC_POLICY__WARN") + + fieldInfos = NULL; + fieldLengths = NULL; + +} + +DocumentWriter::~DocumentWriter() +{ + //Func - Destructor + //Pre - true + //Post - The instance has been destroyed + clearPostingTable(); + _CLDELETE( fieldInfos ); + _CLDELETE_ARRAY(fieldLengths); + _CLDELETE_ARRAY(fieldPositions); + _CLDELETE_ARRAY(fieldBoosts); + _CLDELETE_ARRAY(fieldOffsets); + + _CLDECDELETE(termBuffer); +} + +void DocumentWriter::clearPostingTable() +{ + PostingTableType::iterator itr = postingTable.begin(); + while (itr != postingTable.end()){ + _CLDELETE(itr->second); + _CLLDECDELETE(itr->first); + ++itr; + } + postingTable.clear(); +} + +void DocumentWriter::addDocument(const QString& segment, Document* doc) +{ + CND_PRECONDITION(fieldInfos == NULL, "fieldInfos!=NULL") + + // write field names + fieldInfos = _CLNEW FieldInfos(); + fieldInfos->add(doc); + + QString buf = Misc::segmentname(segment, QLatin1String(".fnm")); + fieldInfos->write(directory, buf); + + // write field values + FieldsWriter fieldsWriter(directory, segment, fieldInfos); + try { + fieldsWriter.addDocument(doc); + } _CLFINALLY ( + fieldsWriter.close() + ); + + // clear postingTable + clearPostingTable(); + + int32_t fieldInfoSize = fieldInfos->size(); + fieldLengths = _CL_NEWARRAY(int32_t, fieldInfoSize); // init fieldLengths + fieldPositions = _CL_NEWARRAY(int32_t, fieldInfoSize); // init fieldPositions + fieldOffsets = _CL_NEWARRAY(int32_t, fieldInfoSize); // init fieldOffsets + fieldBoosts = _CL_NEWARRAY(qreal, fieldInfoSize); // init fieldBoosts + + qreal fbd = doc->getBoost(); + for (int32_t i = 0; i < fieldInfoSize; ++i) { + fieldLengths[i] = 0; + fieldPositions[i] = 0; + fieldOffsets[i] = 0; + //initialise fieldBoost array with default boost + fieldBoosts[i] = fbd; + } + + // invert doc into postingTable + invertDocument(doc); + + // sort postingTable into an array + Posting** postings = NULL; + int32_t postingsLength = 0; + sortPostingTable(postings, postingsLength); + + //DEBUG: + /*for (int32_t i = 0; i < postingsLength; i++) { + Posting* posting = postings[i]; + + TCHAR* b = posting->term->toString(); + _cout << b << " freq=" << posting->freq; + _CLDELETE(b); + + _cout << " pos=" << posting->positions[0]; + for (int32_t j = 1; j < posting->freq; j++) + _cout <<"," << posting->positions[j]; + + _cout << endl; + }*/ + + + // write postings + writePostings(postings, postingsLength, segment); + + // write norms of indexed fields + writeNorms(segment); + _CLDELETE_ARRAY(postings); +} + +void DocumentWriter::sortPostingTable(Posting**& array, int32_t& arraySize) +{ + // copy postingTable into an array + arraySize = postingTable.size(); + array = _CL_NEWARRAY(Posting*,arraySize); + PostingTableType::iterator postings = postingTable.begin(); + int32_t i=0; + while ( postings != postingTable.end() ){ + array[i] = (Posting*)postings->second; + postings++; + i++; + } + // sort the array + quickSort(array, 0, i - 1); +} + + +void DocumentWriter::invertDocument(const Document* doc) +{ + DocumentFieldEnumeration* fields = doc->fields(); + try { + while (fields->hasMoreElements()) { + Field* field = (Field*)fields->nextElement(); + const TCHAR* fieldName = field->name(); + const int32_t fieldNumber = fieldInfos->fieldNumber(fieldName); + + int32_t length = fieldLengths[fieldNumber]; // length of field + int32_t position = fieldPositions[fieldNumber]; // position in field + if (length>0) + position+=analyzer->getPositionIncrementGap(fieldName); + int32_t offset = fieldOffsets[fieldNumber]; // offset field + + if (field->isIndexed()) { + if (!field->isTokenized()) { // un-tokenized field + //FEATURE: this is bug in java: if using a Reader, then + //field value will not be added. With CLucene, an untokenized + //field with a reader will still be added (if it isn't stored, + //because if it's stored, then the reader has already been read. + const TCHAR* charBuf = NULL; + int64_t dataLen = 0; + + if (field->stringValue() == NULL && !field->isStored() ) { + CL_NS(util)::Reader* r = field->readerValue(); + // this call tries to read the entire stream + // this may invalidate the string for the further calls + // it may be better to do this via a FilterReader + // TODO make a better implementation of this + dataLen = r->read(charBuf, LUCENE_INT32_MAX_SHOULDBE); + if (dataLen == -1) + dataLen = 0; + //todo: would be better to pass the string length, in case + //a null char is passed, but then would need to test the output too. + } else { + charBuf = field->stringValue(); + dataLen = _tcslen(charBuf); + } + + if(field->isStoreOffsetWithTermVector()){ + TermVectorOffsetInfo tio; + tio.setStartOffset(offset); + tio.setEndOffset(offset + dataLen); + addPosition(fieldName, charBuf, position++, &tio ); + }else + addPosition(fieldName, charBuf, position++, NULL); + offset += dataLen; + length++; + } else { // field must be tokenized + CL_NS(util)::Reader* reader; // find or make Reader + bool delReader = false; + if (field->readerValue() != NULL) { + reader = field->readerValue(); + } else if (field->stringValue() != NULL) { + reader = _CLNEW CL_NS(util)::StringReader( + field->stringValue(),_tcslen(field->stringValue()), + false); + delReader = true; + } else { + _CLTHROWA(CL_ERR_IO,"field must have either String or Reader value"); + } + + try { + // Tokenize field and add to postingTable. + CL_NS(analysis)::TokenStream* stream = + analyzer->tokenStream(fieldName, reader); + + try { + CL_NS(analysis)::Token t; + int32_t lastTokenEndOffset = -1; + while (stream->next(&t)) { + position += (t.getPositionIncrement() - 1); + + if(field->isStoreOffsetWithTermVector()){ + TermVectorOffsetInfo tio; + tio.setStartOffset(offset + t.startOffset()); + tio.setEndOffset(offset + t.endOffset()); + addPosition(fieldName, t.termText(), position++, &tio); + } else + addPosition(fieldName, t.termText(), position++, NULL); + + lastTokenEndOffset = t.endOffset(); + length++; + // Apply field truncation policy. + if (maxFieldLength != IndexWriter::FIELD_TRUNC_POLICY__WARN) { + // The client programmer has explicitly authorized us to + // truncate the token stream after maxFieldLength tokens. + if ( length > maxFieldLength) + break; + } else if (length > IndexWriter::DEFAULT_MAX_FIELD_LENGTH) { + const TCHAR* errMsgBase = + _T("Indexing a huge number of tokens from a single") + _T(" field (\"%s\", in this case) can cause CLucene") + _T(" to use memory excessively.") + _T(" By default, CLucene will accept only %s tokens") + _T(" tokens from a single field before forcing the") + _T(" client programmer to specify a threshold at") + _T(" which to truncate the token stream.") + _T(" You should set this threshold via") + _T(" IndexReader::maxFieldLength (set to LUCENE_INT32_MAX") + _T(" to disable truncation, or a value to specify maximum number of fields)."); + + TCHAR defaultMaxAsChar[34]; + _i64tot(IndexWriter::DEFAULT_MAX_FIELD_LENGTH, + defaultMaxAsChar, 10 + ); + int32_t errMsgLen = _tcslen(errMsgBase) + + _tcslen(fieldName) + + _tcslen(defaultMaxAsChar); + TCHAR* errMsg = _CL_NEWARRAY(TCHAR,errMsgLen+1); + + _sntprintf(errMsg, errMsgLen,errMsgBase, fieldName, defaultMaxAsChar); + + _CLTHROWT_DEL(CL_ERR_Runtime,errMsg); + } + } // while token->next + + if(lastTokenEndOffset != -1 ) + offset += lastTokenEndOffset + 1; + } _CLFINALLY ( + stream->close(); + _CLDELETE(stream); + ); + } _CLFINALLY ( + if (delReader) { + _CLDELETE(reader); + } + ); + } // if/else field is to be tokenized + fieldLengths[fieldNumber] = length; // save field length + fieldPositions[fieldNumber] = position; // save field position + fieldBoosts[fieldNumber] *= field->getBoost(); + fieldOffsets[fieldNumber] = offset; + } // if field is to beindexed + } // while more fields available + } _CLFINALLY ( + _CLDELETE(fields); + ); +} + +void DocumentWriter::addPosition(const TCHAR* field, const TCHAR* text, + const int32_t position, TermVectorOffsetInfo* offset) +{ + termBuffer->set(field,text,false); + + Posting* ti = postingTable.get(termBuffer); + if (ti != NULL) { // word seen before + int32_t freq = ti->freq; + if (ti->positions.length == freq) { + // positions array is full, realloc its size + ti->positions.length = freq*2; + ti->positions.values = (int32_t*)realloc(ti->positions.values, ti->positions.length * sizeof(int32_t)); + } + ti->positions.values[freq] = position; // add new position + + if (offset != NULL) { + if (ti->offsets.length == freq){ + ti->offsets.length = freq*2; + ti->offsets.values = (TermVectorOffsetInfo*)realloc(ti->offsets.values, ti->offsets.length * sizeof(TermVectorOffsetInfo)); + } + ti->offsets[freq] = *offset; + } + + ti->freq = freq + 1; // update frequency + } else { // word not seen before + Term* term = _CLNEW Term( field, text, false); + postingTable.put(term, _CLNEW Posting(term, position, offset)); + } +} + +//static +void DocumentWriter::quickSort(Posting**& postings, const int32_t lo, const int32_t hi) +{ + if(lo >= hi) + return; + + int32_t mid = (lo + hi) / 2; + + if(postings[lo]->term->compareTo(postings[mid]->term) > 0) { + Posting* tmp = postings[lo]; + postings[lo] = postings[mid]; + postings[mid] = tmp; + } + + if(postings[mid]->term->compareTo(postings[hi]->term) > 0) { + Posting* tmp = postings[mid]; + postings[mid] = postings[hi]; + postings[hi] = tmp; + + if(postings[lo]->term->compareTo(postings[mid]->term) > 0) { + Posting* tmp2 = postings[lo]; + postings[lo] = postings[mid]; + postings[mid] = tmp2; + } + } + + int32_t left = lo + 1; + int32_t right = hi - 1; + + if (left >= right) + return; + + const Term* partition = postings[mid]->term; //not kept, so no need to finalize + + for( ;; ) { + while(postings[right]->term->compareTo(partition) > 0) + --right; + + while(left < right && postings[left]->term->compareTo(partition) <= 0) + ++left; + + if(left < right) { + Posting* tmp = postings[left]; + postings[left] = postings[right]; + postings[right] = tmp; + --right; + } else { + break; + } + } + + quickSort(postings, lo, left); + quickSort(postings, left + 1, hi); +} + +void DocumentWriter::writePostings(Posting** postings, + const int32_t postingsLength, const QString& segment) +{ + #define __DOCLOSE(obj) \ + if (obj!=NULL) { \ + try { \ + obj->close(); \ + _CLDELETE(obj); \ + } catch(CLuceneError &e) { \ + ierr = e.number(); \ + err = e.what(); \ + } catch(...) { \ + err = "Unknown error while closing posting tables"; \ + } \ + } + + IndexOutput* freq = NULL; + IndexOutput* prox = NULL; + TermInfosWriter* tis = NULL; + TermVectorsWriter* termVectorWriter = NULL; + try { + //open files for inverse index storage + QString buf = Misc::segmentname(segment, QLatin1String(".frq")); + freq = directory->createOutput(buf); + + buf = Misc::segmentname(segment, QLatin1String(".prx")); + prox = directory->createOutput(buf); + + tis = _CLNEW TermInfosWriter(directory, segment, fieldInfos, + termIndexInterval); + TermInfo* ti = _CLNEW TermInfo(); + const TCHAR* currentField = NULL; + for (int32_t i = 0; i < postingsLength; i++) { + Posting* posting = postings[i]; + + // add an entry to the dictionary with pointers to prox and freq files + ti->set(1, freq->getFilePointer(), prox->getFilePointer(), -1); + tis->add(posting->term, ti); + + // add an entry to the freq file + int32_t postingFreq = posting->freq; + if (postingFreq == 1) // optimize freq=1 + freq->writeVInt(1); // set low bit of doc num. + else { + freq->writeVInt(0); // the document number + freq->writeVInt(postingFreq); // frequency in doc + } + + int32_t lastPosition = 0; // write positions + for (int32_t j = 0; j < postingFreq; ++j) { // use delta-encoding + prox->writeVInt(posting->positions.values[j] - lastPosition); + lastPosition = posting->positions.values[j]; + } + + // check to see if we switched to a new field + const TCHAR* termField = posting->term->field(); + if ( currentField == NULL || _tcscmp(currentField,termField) != 0 ) { + //todo, can we do an intern'd check? + // changing field - see if there is something to save + currentField = termField; + FieldInfo* fi = fieldInfos->fieldInfo(currentField); + + if (fi->storeTermVector) { + if (termVectorWriter == NULL) { + termVectorWriter = _CLNEW TermVectorsWriter(directory, + segment, fieldInfos); + termVectorWriter->openDocument(); + } + termVectorWriter->openField(currentField); + } else if (termVectorWriter != NULL) { + termVectorWriter->closeField(); + } + } + if (termVectorWriter != NULL && termVectorWriter->isFieldOpen()) { + termVectorWriter->addTerm(posting->term->text(), postingFreq, + &posting->positions, &posting->offsets); + } + } + if (termVectorWriter != NULL) + termVectorWriter->closeDocument(); + _CLDELETE(ti); + } _CLFINALLY ( + const char* err = NULL; + int32_t ierr = 0; + + // make an effort to close all streams we can but remember and re-throw + // the first exception encountered in this process + __DOCLOSE(freq); + __DOCLOSE(prox); + __DOCLOSE(tis); + __DOCLOSE(termVectorWriter); + if (err != NULL) + _CLTHROWA(ierr,err); + ); +} + +void DocumentWriter::writeNorms(const QString& segment) +{ + for(int32_t n = 0; n < fieldInfos->size(); n++){ + FieldInfo* fi = fieldInfos->fieldInfo(n); + if(fi->isIndexed && !fi->omitNorms) { + qreal norm = fieldBoosts[n] * similarity->lengthNorm( + fi->name, fieldLengths[n]); + + QString fn(segment + QLatin1String(".f%1")); + IndexOutput* norms = directory->createOutput(fn.arg(n)); + try { + norms->writeByte(CL_NS(search)::Similarity::encodeNorm(norm)); + }_CLFINALLY ( + norms->close(); + _CLDELETE(norms); + ) + } + } +} + +CL_NS_END diff --git a/src/assistant/3rdparty/clucene/src/CLucene/index/DocumentWriter.h b/src/assistant/3rdparty/clucene/src/CLucene/index/DocumentWriter.h new file mode 100644 index 000000000..7096ba3ee --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/index/DocumentWriter.h @@ -0,0 +1,107 @@ +/* + * Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team + * + * Distributable under the terms of either the Apache License (Version 2.0) or + * the GNU Lesser General Public License, as specified in the COPYING file. + * + * Changes are Copyright(C) 2007, 2008 by Nokia Corporation and/or its subsidiary(-ies), all rights reserved. +*/ +#ifndef _lucene_index_DocumentWriter_ +#define _lucene_index_DocumentWriter_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include <QtCore/QString> + +#include "CLucene/analysis/AnalysisHeader.h" +#include "CLucene/document/Document.h" +#include "CLucene/store/Directory.h" +#include "FieldInfos.h" +#include "IndexWriter.h" +#include "CLucene/util/VoidMap.h" +#include "CLucene/document/Field.h" +#include "TermInfo.h" +#include "CLucene/search/Similarity.h" +#include "TermInfosWriter.h" +#include "FieldsWriter.h" +#include "Term.h" + +CL_NS_DEF(index) + +class DocumentWriter : LUCENE_BASE +{ +public: + // info about a Term in a doc + class Posting : LUCENE_BASE + { + public: + Term* term; // the Term + int32_t freq; // its frequency in doc + Array<int32_t> positions; // positions it occurs at + Array<TermVectorOffsetInfo> offsets; + + Posting(Term* t, const int32_t position, TermVectorOffsetInfo* offset); + ~Posting(); + }; + +private: + CL_NS(analysis)::Analyzer* analyzer; + CL_NS(store)::Directory* directory; + FieldInfos* fieldInfos; //array + const int32_t maxFieldLength; + CL_NS(search)::Similarity* similarity; + int32_t termIndexInterval; + + // Keys are Terms, values are Postings. + // Used to buffer a document before it is written to the index. + typedef CL_NS(util)::CLHashtable<Term*, Posting*, Term::Compare, + Term::Equals> PostingTableType; + PostingTableType postingTable; + int32_t* fieldLengths; //array + int32_t* fieldPositions; //array + int32_t* fieldOffsets; //array + qreal* fieldBoosts; //array + + Term* termBuffer; +public: + /** This ctor used by test code only. + * + * @param directory The directory to write the document information to + * @param analyzer The analyzer to use for the document + * @param similarity The Similarity function + * @param maxFieldLength The maximum number of tokens a field may have + */ + DocumentWriter(CL_NS(store)::Directory* d, CL_NS(analysis)::Analyzer* a, + CL_NS(search)::Similarity* similarity, const int32_t maxFieldLength); + + DocumentWriter(CL_NS(store)::Directory* directory, + CL_NS(analysis)::Analyzer* analyzer, IndexWriter* writer); + ~DocumentWriter(); + + void addDocument(const QString& segment, CL_NS(document)::Document* doc); + + +private: + // Tokenizes the fields of a document into Postings. + void invertDocument(const CL_NS(document)::Document* doc); + + void addPosition(const TCHAR* field, const TCHAR* text, + const int32_t position, TermVectorOffsetInfo* offset); + + void sortPostingTable(Posting**& array, int32_t& arraySize); + + static void quickSort(Posting**& postings, const int32_t lo, const int32_t hi); + + void writePostings(Posting** postings, const int32_t postingsLength, + const QString& segment); + + void writeNorms(const QString& segment); + + void clearPostingTable(); +}; + +CL_NS_END + +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/index/FieldInfo.h b/src/assistant/3rdparty/clucene/src/CLucene/index/FieldInfo.h new file mode 100644 index 000000000..387c4a6ac --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/index/FieldInfo.h @@ -0,0 +1,16 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _lucene_index_FieldInfo_ +#define _lucene_index_FieldInfo_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#error "This header is deprecated, use FieldInfos.h instead" + +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/index/FieldInfos.cpp b/src/assistant/3rdparty/clucene/src/CLucene/index/FieldInfos.cpp new file mode 100644 index 000000000..0c66882f4 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/index/FieldInfos.cpp @@ -0,0 +1,236 @@ +/* + * Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team + * + * Distributable under the terms of either the Apache License (Version 2.0) or + * the GNU Lesser General Public License, as specified in the COPYING file. + * + * Changes are Copyright(C) 2007, 2008 by Nokia Corporation and/or its subsidiary(-ies), all rights reserved. +*/ +#include "CLucene/StdHeader.h" +#include "FieldInfos.h" + +#include "CLucene/store/Directory.h" +#include "CLucene/document/Document.h" +#include "CLucene/document/Field.h" +#include "CLucene/util/VoidMap.h" +#include "CLucene/util/Misc.h" +#include "CLucene/util/StringIntern.h" + +CL_NS_USE(store) +CL_NS_USE(document) +CL_NS_USE(util) +CL_NS_DEF(index) + +FieldInfo::FieldInfo(const TCHAR* _fieldName, bool _isIndexed, + int32_t _fieldNumber, bool _storeTermVector, bool _storeOffsetWithTermVector, + bool _storePositionWithTermVector, bool _omitNorms) + : name(CLStringIntern::intern(_fieldName CL_FILELINE)) + , isIndexed(_isIndexed) + , number(_fieldNumber) + , storeTermVector(_storeTermVector) + , storeOffsetWithTermVector(_storeOffsetWithTermVector) + , storePositionWithTermVector(_storeTermVector) + , omitNorms(_omitNorms) +{ +} + +FieldInfo::~FieldInfo() +{ + CL_NS(util)::CLStringIntern::unintern(name); +} + +// #pragma mark -- FieldInfos + +FieldInfos::FieldInfos() + : byName(false, false) + , byNumber(true) +{ +} + +FieldInfos::~FieldInfos() +{ + byName.clear(); + byNumber.clear(); +} + +FieldInfos::FieldInfos(Directory* d, const QString& name) + : byName(false, false) + , byNumber(true) +{ + IndexInput* input = d->openInput(name); + try { + read(input); + } _CLFINALLY ( + input->close(); + _CLDELETE(input); + ); +} + +void FieldInfos::add(const Document* doc) +{ + DocumentFieldEnumeration* fields = doc->fields(); + Field* field; + while (fields->hasMoreElements()) { + field = fields->nextElement(); + add(field->name(), field->isIndexed(), field->isTermVectorStored()); + } + _CLDELETE(fields); +} + +void FieldInfos::add(const TCHAR* name, bool isIndexed, bool storeTermVector, + bool storePositionWithTermVector, bool storeOffsetWithTermVector, bool omitNorms) +{ + FieldInfo* fi = fieldInfo(name); + if (fi == NULL) { + addInternal(name, isIndexed, storeTermVector, + storePositionWithTermVector, + storeOffsetWithTermVector, omitNorms); + } else { + if (fi->isIndexed != isIndexed) { + // once indexed, always index + fi->isIndexed = true; + } + + if (fi->storeTermVector != storeTermVector) { + // once vector, always vector + fi->storeTermVector = true; + } + + if (fi->storePositionWithTermVector != storePositionWithTermVector) { + // once vector, always vector + fi->storePositionWithTermVector = true; + } + + if (fi->storeOffsetWithTermVector != storeOffsetWithTermVector) { + // once vector, always vector + fi->storeOffsetWithTermVector = true; + } + + if (fi->omitNorms != omitNorms) { + // once norms are stored, always store + fi->omitNorms = false; + } + } +} + +void FieldInfos::add(const TCHAR** names, bool isIndexed, bool storeTermVectors, + bool storePositionWithTermVector, bool storeOffsetWithTermVector, bool omitNorms) +{ + int32_t i=0; + while (names[i] != NULL) { + add(names[i], isIndexed, storeTermVectors, storePositionWithTermVector, + storeOffsetWithTermVector, omitNorms); + ++i; + } +} + +int32_t FieldInfos::fieldNumber(const TCHAR* fieldName) const +{ + FieldInfo* fi = fieldInfo(fieldName); + return (fi != NULL) ? fi->number : -1; +} + +FieldInfo* FieldInfos::fieldInfo(const TCHAR* fieldName) const +{ + return byName.get(fieldName); +} + +const TCHAR* FieldInfos::fieldName(const int32_t fieldNumber) const +{ + FieldInfo* fi = fieldInfo(fieldNumber); + return (fi == NULL) ? LUCENE_BLANK_STRING : fi->name; +} + +FieldInfo* FieldInfos::fieldInfo(const int32_t fieldNumber) const +{ + if (fieldNumber < 0 || (size_t)fieldNumber >= byNumber.size()) + return NULL; + return byNumber[fieldNumber]; +} + +int32_t FieldInfos::size() const +{ + return byNumber.size(); +} + +void FieldInfos::write(Directory* d, const QString& name) const +{ + IndexOutput* output = d->createOutput(name); + try { + write(output); + } _CLFINALLY ( + output->close(); + _CLDELETE(output); + ); +} + +void FieldInfos::write(IndexOutput* output) const +{ + output->writeVInt(size()); + FieldInfo* fi; + uint8_t bits; + for (int32_t i = 0; i < size(); ++i) { + fi = fieldInfo(i); + bits = 0x0; + if (fi->isIndexed) + bits |= IS_INDEXED; + + if (fi->storeTermVector) + bits |= STORE_TERMVECTOR; + + if (fi->storePositionWithTermVector) + bits |= STORE_POSITIONS_WITH_TERMVECTOR; + + if (fi->storeOffsetWithTermVector) + bits |= STORE_OFFSET_WITH_TERMVECTOR; + + if (fi->omitNorms) + bits |= OMIT_NORMS; + + output->writeString(fi->name, _tcslen(fi->name)); + output->writeByte(bits); + } +} + +void FieldInfos::read(IndexInput* input) +{ + int32_t size = input->readVInt(); + for (int32_t i = 0; i < size; ++i) { + // we could read name into a string buffer, but we can't be sure what + // the maximum field length will be. + TCHAR* name = input->readString(); + uint8_t bits = input->readByte(); + bool isIndexed = (bits & IS_INDEXED) != 0; + bool storeTermVector = (bits & STORE_TERMVECTOR) != 0; + bool storePositionsWithTermVector = + (bits & STORE_POSITIONS_WITH_TERMVECTOR) != 0; + bool storeOffsetWithTermVector = (bits & STORE_OFFSET_WITH_TERMVECTOR) != 0; + bool omitNorms = (bits & OMIT_NORMS) != 0; + + addInternal(name, isIndexed, storeTermVector, + storePositionsWithTermVector, storeOffsetWithTermVector, omitNorms); + _CLDELETE_CARRAY(name); + } +} + +void FieldInfos::addInternal(const TCHAR* name, bool isIndexed, + bool storeTermVector, bool storePositionWithTermVector, + bool storeOffsetWithTermVector, bool omitNorms) +{ + FieldInfo* fi = _CLNEW FieldInfo(name, isIndexed, byNumber.size(), + storeTermVector, storePositionWithTermVector, storeOffsetWithTermVector, + omitNorms); + byNumber.push_back(fi); + byName.put(fi->name, fi); +} + +bool FieldInfos::hasVectors() const +{ + for (int32_t i = 0; i < size(); i++) { + if (fieldInfo(i)->storeTermVector) + return true; + } + return false; +} + +CL_NS_END diff --git a/src/assistant/3rdparty/clucene/src/CLucene/index/FieldInfos.h b/src/assistant/3rdparty/clucene/src/CLucene/index/FieldInfos.h new file mode 100644 index 000000000..7b6d0f56d --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/index/FieldInfos.h @@ -0,0 +1,171 @@ +/* + * Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team + * + * Distributable under the terms of either the Apache License (Version 2.0) or + * the GNU Lesser General Public License, as specified in the COPYING file. + * + * Changes are Copyright(C) 2007, 2008 by Nokia Corporation and/or its subsidiary(-ies), all rights reserved. +*/ +#ifndef _lucene_index_FieldInfos_ +#define _lucene_index_FieldInfos_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include <QtCore/QString> + +#include "CLucene/store/Directory.h" +#include "CLucene/document/Document.h" +#include "CLucene/util/VoidMap.h" +#include "CLucene/util/VoidList.h" + +CL_NS_DEF(index) + +class FieldInfo : LUCENE_BASE +{ + public: + //name of the field + const TCHAR* name; + + //Is field indexed? true = yes false = no + bool isIndexed; + + //field number + const int32_t number; + + // true if term vector for this field should be stored + bool storeTermVector; + bool storeOffsetWithTermVector; + bool storePositionWithTermVector; + + bool omitNorms; // omit norms associated with indexed fields + + //Func - Constructor + // Initialises FieldInfo. + // na holds the name of the field + // tk indicates whether this field is indexed or not + // nu indicates its number + //Pre - na != NULL and holds the name of the field + // tk is true or false + // number >= 0 + //Post - The FieldInfo instance has been created and initialized. + // name holds the duplicated string of na + // isIndexed = tk + // number = nu + FieldInfo(const TCHAR* fieldName, bool isIndexed, int32_t fieldNumber, + bool storeTermVector, bool storeOffsetWithTermVector, + bool storePositionWithTermVector, bool omitNorms); + + //Func - Destructor + //Pre - true + //Post - The instance has been destroyed + ~FieldInfo(); +}; + +/* Access to the Field Info file that describes document fields and whether or + * not they are indexed. Each segment has a separate Field Info file. Objects + * of this class are thread-safe for multiple readers, but only one thread can + * be adding documents at a time, with no other reader or writer threads + * accessing this object. +*/ +class FieldInfos : LUCENE_BASE +{ +private: + // we now use internd field names, so we can use the voidCompare to + // directly compare the strings + typedef CL_NS(util)::CLHashMap<const TCHAR*, FieldInfo*, + CL_NS(util)::Compare::TChar,CL_NS(util)::Equals::TChar > defByName; + defByName byName; + + CL_NS(util)::CLArrayList<FieldInfo*, + CL_NS(util)::Deletor::Object<FieldInfo> > byNumber; + +public: + enum { + IS_INDEXED = 0x1, + STORE_TERMVECTOR = 0x2, + STORE_POSITIONS_WITH_TERMVECTOR = 0x4, + STORE_OFFSET_WITH_TERMVECTOR = 0x8, + OMIT_NORMS = 0x10 + }; + + FieldInfos(); + ~FieldInfos(); + + // Construct a FieldInfos object using the directory and the name of the + // file IndexInput + // @param d The directory to open the IndexInput from + // @param name Name of the file to open the IndexInput from in the Directory + // @throws IOException + // @see #read + FieldInfos(CL_NS(store)::Directory* d, const QString& name); + + int32_t fieldNumber(const TCHAR* fieldName)const; + + // Return the fieldinfo object referenced by the fieldNumber. + // @param fieldNumber + // @return the FieldInfo object or null when the given fieldNumber + // doesn't exist. + FieldInfo* fieldInfo(const TCHAR* fieldName) const; + + // Return the fieldName identified by its number. + // @param fieldNumber + // @return the fieldName or an empty string when the field + // with the given number doesn't exist. + const TCHAR* fieldName(const int32_t fieldNumber) const; + + FieldInfo* fieldInfo(const int32_t fieldNumber) const; + + int32_t size()const; + + bool hasVectors() const; + + // Adds field info for a Document. + void add(const CL_NS(document)::Document* doc); + + // Merges in information from another FieldInfos. + void add(FieldInfos* other); + + + /** If the field is not yet known, adds it. If it is known, checks to make + * sure that the isIndexed flag is the same as was given previously for this + * field. If not - marks it as being indexed. Same goes for the TermVector + * parameters. + * + * @param name The name of the field + * @param isIndexed true if the field is indexed + * @param storeTermVector true if the term vector should be stored + * @param storePositionWithTermVector true if the term vector with positions should be stored + * @param storeOffsetWithTermVector true if the term vector with offsets should be stored + */ + void add(const TCHAR* name, bool isIndexed, bool storeTermVector = false, + bool storePositionWithTermVector = false, + bool storeOffsetWithTermVector = false, bool omitNorms = false); + + /** + * Assumes the fields are not storing term vectors + * @param names The names of the fields + * @param isIndexed true if the field is indexed + * @param storeTermVector true if the term vector should be stored + * + * @see #add(String, boolean) + */ + void add(const TCHAR** names, bool isIndexed, bool storeTermVector = false, + bool storePositionWithTermVector = false, + bool storeOffsetWithTermVector = false, bool omitNorms = false); + + void write(CL_NS(store)::Directory* d, const QString& name) const; + void write(CL_NS(store)::IndexOutput* output) const; + +private: + void read(CL_NS(store)::IndexInput* input); + void addInternal(const TCHAR* name, bool isIndexed, bool storeTermVector, + bool storePositionWithTermVector, bool storeOffsetWithTermVector, + bool omitNorms); + +}; + +CL_NS_END + +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/index/FieldsReader.cpp b/src/assistant/3rdparty/clucene/src/CLucene/index/FieldsReader.cpp new file mode 100644 index 000000000..e3f9d1cc2 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/index/FieldsReader.cpp @@ -0,0 +1,231 @@ +/* + * Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team + * + * Distributable under the terms of either the Apache License (Version 2.0) or + * the GNU Lesser General Public License, as specified in the COPYING file. + * + * Changes are Copyright(C) 2007, 2008 by Nokia Corporation and/or its subsidiary(-ies), all rights reserved. +*/ +#include "CLucene/StdHeader.h" +#include "FieldsReader.h" + +#include "CLucene/util/VoidMap.h" +#include "CLucene/util/Misc.h" +#include "CLucene/store/Directory.h" +#include "CLucene/document/Document.h" +#include "CLucene/document/Field.h" +#include "FieldInfos.h" +#include "FieldsWriter.h" + +CL_NS_USE(store) +CL_NS_USE(document) +CL_NS_USE(util) +CL_NS_DEF(index) + +FieldsReader::FieldsReader(Directory* d, const QString& segment, FieldInfos* fn) + : fieldInfos(fn) +{ + //Func - Constructor + //Pre - d contains a valid reference to a Directory + // segment != NULL + // fn contains a valid reference to a FieldInfos + //Post - The instance has been created + + CND_PRECONDITION(!segment.isEmpty(), "segment != NULL"); + + QString buf = Misc::segmentname(segment, QLatin1String(".fdt")); + fieldsStream = d->openInput(buf); + + buf = Misc::segmentname(segment, QLatin1String(".fdx")); + indexStream = d->openInput(buf); + + _size = (int32_t)indexStream->length() / 8; +} + +FieldsReader::~FieldsReader() +{ + //Func - Destructor + //Pre - true + //Post - The instance has been destroyed + close(); +} + +void FieldsReader::close() +{ + //Func - Closes the FieldsReader + //Pre - true + //Post - The FieldsReader has been closed + if (fieldsStream) { + fieldsStream->close(); + _CLDELETE(fieldsStream); + } + + if(indexStream) { + indexStream->close(); + _CLDELETE(indexStream); + } +} + +int32_t FieldsReader::size() const +{ + return _size; +} + +bool FieldsReader::doc(int32_t n, Document* doc) +{ + if ( n * 8L > indexStream->length() ) + return false; + + indexStream->seek(n * 8L); + int64_t position = indexStream->readLong(); + fieldsStream->seek(position); + + int32_t numFields = fieldsStream->readVInt(); + for (int32_t i = 0; i < numFields; i++) { + int32_t fieldNumber = fieldsStream->readVInt(); + FieldInfo* fi = fieldInfos->fieldInfo(fieldNumber); + + if ( fi == NULL ) + _CLTHROWA(CL_ERR_IO, "Field stream is invalid"); + + uint8_t bits = fieldsStream->readByte(); + if ((bits & FieldsWriter::FIELD_IS_BINARY) != 0) { + int32_t fieldLen = fieldsStream->readVInt(); + FieldsReader::FieldsStreamHolder* subStream = new + FieldsReader::FieldsStreamHolder(fieldsStream, fieldLen); + uint8_t bits = Field::STORE_YES; + Field* f = _CLNEW Field( + fi->name, // name + subStream, // read value + bits); + + doc->add(*f); + + //now skip over the rest of the field + if (fieldsStream->getFilePointer() + fieldLen + == fieldsStream->length()) { + // set to eof + fieldsStream->seek(fieldsStream->getFilePointer() + fieldLen - 1); + fieldsStream->readByte(); + } else { + fieldsStream->seek(fieldsStream->getFilePointer() + fieldLen); + } + } else { + uint8_t bits = Field::STORE_YES; + if (fi->isIndexed && (bits & FieldsWriter::FIELD_IS_TOKENIZED)!=0 ) + bits |= Field::INDEX_TOKENIZED; + else if (fi->isIndexed && (bits & FieldsWriter::FIELD_IS_TOKENIZED) == 0) + bits |= Field::INDEX_UNTOKENIZED; + else + bits |= Field::INDEX_NO; + + if (fi->storeTermVector) { + if (fi->storeOffsetWithTermVector) { + if (fi->storePositionWithTermVector) { + bits |= Field::TERMVECTOR_WITH_OFFSETS; + bits |= Field::TERMVECTOR_WITH_POSITIONS; + } else { + bits |= Field::TERMVECTOR_WITH_OFFSETS; + } + } else if (fi->storePositionWithTermVector) { + bits |= Field::TERMVECTOR_WITH_POSITIONS; + } else { + bits |= Field::TERMVECTOR_YES; + } + } else { + bits |= Field::TERMVECTOR_NO; + } + + if ( (bits & FieldsWriter::FIELD_IS_COMPRESSED) != 0 ) { + bits |= Field::STORE_COMPRESS; + int32_t fieldLen = fieldsStream->readVInt(); + FieldsStreamHolder* subStream = new + FieldsStreamHolder(fieldsStream, fieldLen); + + // TODO: we dont have gzip inputstream available, must alert + // user to somehow use a gzip inputstream + Field* f = _CLNEW Field( + fi->name, // name + subStream, // read value + bits); + + f->setOmitNorms(fi->omitNorms); + doc->add(*f); + + // now skip over the rest of the field + if (fieldsStream->getFilePointer() + fieldLen + == fieldsStream->length()) { + //set to eof + fieldsStream->seek(fieldsStream->getFilePointer() + fieldLen - 1); + fieldsStream->readByte(); + } else { + fieldsStream->seek(fieldsStream->getFilePointer() + fieldLen); + } + } else { + TCHAR* fvalue = fieldsStream->readString(true); + Field* f = _CLNEW Field( + fi->name, // name + fvalue, // read value + bits); + // TODO: could optimise this + _CLDELETE_CARRAY(fvalue); + f->setOmitNorms(fi->omitNorms); + doc->add(*f); + } + } + } + return true; +} + +FieldsReader::FieldsStreamHolder::FieldsStreamHolder(IndexInput* indexInput, + int32_t subLength) +{ + this->indexInput = indexInput->clone(); + this->indexInputStream = new IndexInputStream(this->indexInput); + this->subStream = new jstreams::SubInputStream<char>(indexInputStream, + subLength); + + this->size = subStream->getSize(); + this->position = subStream->getPosition(); + this->error = subStream->getError(); + this->status = subStream->getStatus(); +} + +FieldsReader::FieldsStreamHolder::~FieldsStreamHolder() +{ + delete subStream; + delete indexInputStream; + + indexInput->close(); + _CLDELETE(indexInput); +} + +int32_t FieldsReader::FieldsStreamHolder::read(const char*& start, int32_t _min, + int32_t _max) +{ + int32_t ret = subStream->read(start,_min,_max); + this->position = subStream->getPosition(); + this->error = subStream->getError(); + this->status = subStream->getStatus(); + return ret; +} + +int64_t FieldsReader::FieldsStreamHolder::skip(int64_t ntoskip) +{ + int64_t ret = subStream->skip(ntoskip); + this->position = subStream->getPosition(); + this->error = subStream->getError(); + this->status = subStream->getStatus(); + return ret; +} + +int64_t FieldsReader::FieldsStreamHolder::reset(int64_t pos) +{ + int64_t ret = subStream->reset(pos); + this->position = subStream->getPosition(); + this->error = subStream->getError(); + this->status = subStream->getStatus(); + return ret; +} + +CL_NS_END diff --git a/src/assistant/3rdparty/clucene/src/CLucene/index/FieldsReader.h b/src/assistant/3rdparty/clucene/src/CLucene/index/FieldsReader.h new file mode 100644 index 000000000..53589a5cc --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/index/FieldsReader.h @@ -0,0 +1,60 @@ +/* + * Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team + * + * Distributable under the terms of either the Apache License (Version 2.0) or + * the GNU Lesser General Public License, as specified in the COPYING file. + * + * Changes are Copyright(C) 2007, 2008 by Nokia Corporation and/or its subsidiary(-ies), all rights reserved. +*/ +#ifndef _lucene_index_FieldsReader_ +#define _lucene_index_FieldsReader_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include <QtCore/QString> + +#include "CLucene/store/Directory.h" +#include "CLucene/document/Document.h" +#include "CLucene/util/subinputstream.h" +#include "CLucene/store/IndexInput.h" + +CL_NS_DEF(index) + +class FieldInfos; + +class FieldsReader : LUCENE_BASE +{ +private: + const FieldInfos* fieldInfos; + CL_NS(store)::IndexInput* fieldsStream; + CL_NS(store)::IndexInput* indexStream; + int32_t _size; + + class FieldsStreamHolder : public jstreams::StreamBase<char> + { + CL_NS(store)::IndexInput* indexInput; + CL_NS(store)::IndexInputStream* indexInputStream; + jstreams::SubInputStream<char>* subStream; + + public: + FieldsStreamHolder(CL_NS(store)::IndexInput* indexInput, int32_t subLength); + ~FieldsStreamHolder(); + int32_t read(const char*& start, int32_t _min, int32_t _max); + int64_t skip(int64_t ntoskip); + int64_t reset(int64_t pos); + }; + +public: + FieldsReader(CL_NS(store)::Directory* d, const QString& segment, FieldInfos* fn); + ~FieldsReader(); + void close(); + int32_t size() const; + // loads the fields from n'th document into doc. returns true on success. + bool doc(int32_t n, CL_NS(document)::Document* doc); +}; + +CL_NS_END + +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/index/FieldsWriter.cpp b/src/assistant/3rdparty/clucene/src/CLucene/index/FieldsWriter.cpp new file mode 100644 index 000000000..ceb6735cb --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/index/FieldsWriter.cpp @@ -0,0 +1,186 @@ +/* + * Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team + * + * Distributable under the terms of either the Apache License (Version 2.0) or + * the GNU Lesser General Public License, as specified in the COPYING file. + * + * Changes are Copyright(C) 2007, 2008 by Nokia Corporation and/or its subsidiary(-ies), all rights reserved. +*/ +#include "CLucene/StdHeader.h" +#include "FieldsWriter.h" + +#include "CLucene/util/VoidMap.h" +#include "CLucene/util/Reader.h" +#include "CLucene/util/Misc.h" +#include "CLucene/store/Directory.h" +#include "CLucene/store/IndexOutput.h" +#include "CLucene/document/Document.h" +#include "CLucene/document/Field.h" +#include "FieldInfos.h" + +CL_NS_USE(store) +CL_NS_USE(util) +CL_NS_USE(document) +CL_NS_DEF(index) + +FieldsWriter::FieldsWriter(Directory* d, const QString& segment, FieldInfos* fn) + : fieldInfos(fn) +{ + //Func - Constructor + //Pre - d contains a valid reference to a directory + // segment != NULL and contains the name of the segment + //Post - fn contains a valid reference toa a FieldInfos + + CND_PRECONDITION(!segment.isEmpty(), "segment is NULL"); + + QString buf = Misc::segmentname(segment, QLatin1String(".fdt")); + fieldsStream = d->createOutput(buf); + + buf = Misc::segmentname(segment, QLatin1String(".fdx")); + indexStream = d->createOutput(buf); + + CND_CONDITION(indexStream != NULL, "indexStream is NULL"); +} + +FieldsWriter::~FieldsWriter() +{ + //Func - Destructor + //Pre - true + //Post - Instance has been destroyed + + close(); +} + +void FieldsWriter::close() +{ + //Func - Closes all streams and frees all resources + //Pre - true + //Post - All streams have been closed all resources have been freed + + //Check if fieldsStream is valid + if (fieldsStream) { + //Close fieldsStream + fieldsStream->close(); + _CLDELETE(fieldsStream); + } + + //Check if indexStream is valid + if (indexStream) { + //Close indexStream + indexStream->close(); + _CLDELETE(indexStream); + } +} + +void FieldsWriter::addDocument(Document* doc) +{ + //Func - Adds a document + //Pre - doc contains a valid reference to a Document + // indexStream != NULL + // fieldsStream != NULL + //Post - The document doc has been added + + CND_PRECONDITION(indexStream != NULL, "indexStream is NULL"); + CND_PRECONDITION(fieldsStream != NULL, "fieldsStream is NULL"); + + indexStream->writeLong(fieldsStream->getFilePointer()); + + int32_t storedCount = 0; + DocumentFieldEnumeration* fields = doc->fields(); + while (fields->hasMoreElements()) { + Field* field = fields->nextElement(); + if (field->isStored()) + storedCount++; + } + _CLDELETE(fields); + fieldsStream->writeVInt(storedCount); + + fields = doc->fields(); + while (fields->hasMoreElements()) { + Field* field = fields->nextElement(); + if (field->isStored()) { + fieldsStream->writeVInt(fieldInfos->fieldNumber(field->name())); + + uint8_t bits = 0; + if (field->isTokenized()) + bits |= FieldsWriter::FIELD_IS_TOKENIZED; + if (field->isBinary()) + bits |= FieldsWriter::FIELD_IS_BINARY; + if (field->isCompressed()) + bits |= FieldsWriter::FIELD_IS_COMPRESSED; + + fieldsStream->writeByte(bits); + + if ( field->isCompressed()) { + _CLTHROWA(CL_ERR_Runtime, + "CLucene does not directly support compressed fields. " + "Write a compressed byte array instead"); + } else { + // FEATURE: this problem in Java Lucene too, if using Reader, + // data is not stored. + // + // TODO: this is a logic bug... + // if the field is stored, and indexed, and is using a reader + // the field wont get indexed + // + // if we could write zero prefixed vints (therefore static + // length), then we could write a reader directly to the field + // indexoutput and then go back and write the data length. + // however this is not supported in lucene yet... + // if this is ever implemented, then it would make sense to + // also be able to combine the FieldsWriter and + // DocumentWriter::invertDocument process, and use a + // streamfilter to write the field data while the documentwrite + // analyses the document! how cool would that be! it would cut + // out all these buffers!!! + + // compression is disabled for the current field + if (field->isBinary()) { + // TODO: since we currently don't support static length vints, + // we have to read the entire stream into memory first.... ugly! + jstreams::StreamBase<char>* stream = field->streamValue(); + const char* sd; + // how do we make sure we read the entire index in now??? + // TODO: we need to have a max amount, and guarantee its all + // in or throw an error... + int32_t rl = stream->read(sd,10000000,0); + + if ( rl < 0 ) { + // TODO: could we detect this earlier and not actually + // write the field?? + fieldsStream->writeVInt(0); + } else { + // TODO: if this int could be written with a constant + // length, then the stream could be read and written a + // bit at a time then the length is re-written at the end. + fieldsStream->writeVInt(rl); + fieldsStream->writeBytes((uint8_t*)sd, rl); + } + } else if (field->stringValue() == NULL ) { + // we must be using readerValue + CND_PRECONDITION(!field->isIndexed(), + "Cannot store reader if it is indexed too") + Reader* r = field->readerValue(); + + //read the entire string + const TCHAR* rv; + int64_t rl = r->read(rv, LUCENE_INT32_MAX_SHOULDBE); + if ( rl > LUCENE_INT32_MAX_SHOULDBE ) + _CLTHROWA(CL_ERR_Runtime, "Field length too long"); + else if ( rl < 0 ) + rl = 0; + + fieldsStream->writeString( rv, (int32_t)rl); + } else if (field->stringValue() != NULL ) { + fieldsStream->writeString(field->stringValue(), + _tcslen(field->stringValue())); + } else { + _CLTHROWA(CL_ERR_Runtime, "No values are set for the field"); + } + } + } + } + _CLDELETE(fields); +} + +CL_NS_END diff --git a/src/assistant/3rdparty/clucene/src/CLucene/index/FieldsWriter.h b/src/assistant/3rdparty/clucene/src/CLucene/index/FieldsWriter.h new file mode 100644 index 000000000..7dde5f225 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/index/FieldsWriter.h @@ -0,0 +1,49 @@ +/* + * Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team + * + * Distributable under the terms of either the Apache License (Version 2.0) or + * the GNU Lesser General Public License, as specified in the COPYING file. + * + * Changes are Copyright(C) 2007, 2008 by Nokia Corporation and/or its subsidiary(-ies), all rights reserved. +*/ +#ifndef _lucene_index_FieldsWriter_ +#define _lucene_index_FieldsWriter_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include <QtCore/QString> + +#include "CLucene/document/Document.h" +#include "CLucene/store/Directory.h" +#include "CLucene/store/IndexOutput.h" + +CL_NS_DEF(index) + +class FieldInfos; + +class FieldsWriter : LUCENE_BASE +{ +private: + FieldInfos* fieldInfos; + + CL_NS(store)::IndexOutput* fieldsStream; + CL_NS(store)::IndexOutput* indexStream; + +public: + LUCENE_STATIC_CONSTANT(uint8_t, FIELD_IS_TOKENIZED = 0x1); + LUCENE_STATIC_CONSTANT(uint8_t, FIELD_IS_BINARY = 0x2); + LUCENE_STATIC_CONSTANT(uint8_t, FIELD_IS_COMPRESSED = 0x4); + + FieldsWriter(CL_NS(store)::Directory* d, const QString& segment, FieldInfos* fn); + ~FieldsWriter(); + + void close(); + + void addDocument(CL_NS(document)::Document* doc); +}; + +CL_NS_END + +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/index/IndexModifier.cpp b/src/assistant/3rdparty/clucene/src/CLucene/index/IndexModifier.cpp new file mode 100644 index 000000000..1423cc7a8 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/index/IndexModifier.cpp @@ -0,0 +1,254 @@ +/* + * Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team + * + * Distributable under the terms of either the Apache License (Version 2.0) or + * the GNU Lesser General Public License, as specified in the COPYING file. + * + * Changes are Copyright(C) 2007, 2008 by Nokia Corporation and/or its subsidiary(-ies), all rights reserved. +*/ +#include "CLucene/StdHeader.h" +#include "IndexModifier.h" + +#include "Term.h" +#include "IndexWriter.h" +#include "IndexReader.h" + +CL_NS_DEF(index) +CL_NS_USE(util) +CL_NS_USE(store) +CL_NS_USE(analysis) +CL_NS_USE(document) + +IndexModifier::IndexModifier(Directory* directory, Analyzer* analyzer, bool create) { + init(directory, analyzer, create); +} + +IndexModifier::IndexModifier(const QString& dirName, Analyzer* analyzer, bool create) { + Directory* dir = FSDirectory::getDirectory(dirName, create); + init(dir, analyzer, create); +} + +void IndexModifier::init(Directory* directory, Analyzer* analyzer, bool create) { + indexWriter = NULL; + indexReader = NULL; + this->analyzer = analyzer; + open = false; + + useCompoundFile = true; + int32_t maxBufferedDocs = IndexWriter::DEFAULT_MAX_BUFFERED_DOCS; + this->maxFieldLength = IndexWriter::DEFAULT_MAX_FIELD_LENGTH; + int32_t mergeFactor = IndexWriter::DEFAULT_MERGE_FACTOR; + + this->directory = _CL_POINTER(directory); + createIndexReader(); + open = true; +} + +IndexModifier::~IndexModifier(){ + close(); +} + +void IndexModifier::assureOpen() const{ + if (!open) { + _CLTHROWA(CL_ERR_IllegalState,"Index is closed"); + } +} + +void IndexModifier::createIndexWriter() { + if (indexWriter == NULL) { + if (indexReader != NULL) { + indexReader->close(); + _CLDELETE(indexReader); + } + indexWriter = _CLNEW IndexWriter(directory, analyzer, false); + indexWriter->setUseCompoundFile(useCompoundFile); + //indexWriter->setMaxBufferedDocs(maxBufferedDocs); + indexWriter->setMaxFieldLength(maxFieldLength); + //indexWriter->setMergeFactor(mergeFactor); + } +} + +void IndexModifier::createIndexReader() { + if (indexReader == NULL) { + if (indexWriter != NULL) { + indexWriter->close(); + _CLDELETE(indexWriter); + } + indexReader = IndexReader::open(directory); + } +} + +void IndexModifier::flush() { + SCOPED_LOCK_MUTEX(directory->THIS_LOCK) + assureOpen(); + if (indexWriter != NULL) { + indexWriter->close(); + _CLDELETE(indexWriter); + createIndexWriter(); + } else { + indexReader->close(); + _CLDELETE(indexReader); + createIndexReader(); + } +} + +void IndexModifier::addDocument(Document* doc, Analyzer* docAnalyzer) { + SCOPED_LOCK_MUTEX(directory->THIS_LOCK) + assureOpen(); + createIndexWriter(); + if (docAnalyzer != NULL) + indexWriter->addDocument(doc, docAnalyzer); + else + indexWriter->addDocument(doc); +} + +int32_t IndexModifier::deleteDocuments(Term* term) { + SCOPED_LOCK_MUTEX(directory->THIS_LOCK) + assureOpen(); + createIndexReader(); + return indexReader->deleteDocuments(term); +} + +void IndexModifier::deleteDocument(int32_t docNum) { + SCOPED_LOCK_MUTEX(directory->THIS_LOCK) + assureOpen(); + createIndexReader(); + indexReader->deleteDocument(docNum); +} + +int32_t IndexModifier::docCount() { + SCOPED_LOCK_MUTEX(directory->THIS_LOCK) + assureOpen(); + if (indexWriter != NULL) + return indexWriter->docCount(); + else + return indexReader->numDocs(); +} + +void IndexModifier::optimize() { + SCOPED_LOCK_MUTEX(directory->THIS_LOCK) + assureOpen(); + createIndexWriter(); + indexWriter->optimize(); +} + +void IndexModifier::setUseCompoundFile(bool useCompoundFile) { + SCOPED_LOCK_MUTEX(directory->THIS_LOCK) + assureOpen(); + if (indexWriter != NULL) + indexWriter->setUseCompoundFile(useCompoundFile); + this->useCompoundFile = useCompoundFile; +} + +bool IndexModifier::getUseCompoundFile() { + SCOPED_LOCK_MUTEX(directory->THIS_LOCK) + assureOpen(); + createIndexWriter(); + return indexWriter->getUseCompoundFile(); +} + +void IndexModifier::setMaxFieldLength(int32_t maxFieldLength) { + SCOPED_LOCK_MUTEX(directory->THIS_LOCK) + assureOpen(); + if (indexWriter != NULL) + indexWriter->setMaxFieldLength(maxFieldLength); + this->maxFieldLength = maxFieldLength; +} + +int32_t IndexModifier::getMaxFieldLength() { + SCOPED_LOCK_MUTEX(directory->THIS_LOCK) + assureOpen(); + createIndexWriter(); + return indexWriter->getMaxFieldLength(); +} + +void IndexModifier::setMaxBufferedDocs(int32_t maxBufferedDocs) { + SCOPED_LOCK_MUTEX(directory->THIS_LOCK) + assureOpen(); + if (indexWriter != NULL) + indexWriter->setMaxBufferedDocs(maxBufferedDocs); + this->maxBufferedDocs = maxBufferedDocs; +} + +int32_t IndexModifier::getMaxBufferedDocs() { + SCOPED_LOCK_MUTEX(directory->THIS_LOCK) + assureOpen(); + createIndexWriter(); + return indexWriter->getMaxBufferedDocs(); +} +void IndexModifier::setMergeFactor(int32_t mergeFactor) { + SCOPED_LOCK_MUTEX(directory->THIS_LOCK) + assureOpen(); + if (indexWriter != NULL) + indexWriter->setMergeFactor(mergeFactor); + this->mergeFactor = mergeFactor; +} + +int32_t IndexModifier::getMergeFactor() { + SCOPED_LOCK_MUTEX(directory->THIS_LOCK) + assureOpen(); + createIndexWriter(); + return indexWriter->getMergeFactor(); +} + +void IndexModifier::close() { + SCOPED_LOCK_MUTEX(directory->THIS_LOCK) + if (indexWriter != NULL) { + indexWriter->close(); + _CLDELETE(indexWriter); + } else { + indexReader->close(); + _CLDELETE(indexReader); + } + _CLDECDELETE(directory) + open = false; +} + +QString IndexModifier::toString() const +{ + QString ret(QLatin1String("Index@")); + return ret.append(directory->toString()); +} + + + +int64_t IndexModifier::getCurrentVersion() const{ + SCOPED_LOCK_MUTEX(directory->THIS_LOCK) + return IndexReader::getCurrentVersion(directory); +} + +TermDocs* IndexModifier::termDocs(Term* term){ + SCOPED_LOCK_MUTEX(directory->THIS_LOCK) + assureOpen(); + createIndexReader(); + return indexReader->termDocs(term); +} + +TermEnum* IndexModifier::terms(Term* term){ + SCOPED_LOCK_MUTEX(directory->THIS_LOCK) + assureOpen(); + createIndexReader(); + if ( term != NULL ) + return indexReader->terms(term); + else + return indexReader->terms(); +} + + + CL_NS(document)::Document* IndexModifier::document(const int32_t n){ + Document* ret = _CLNEW Document; + if (!document(n,ret) ) + _CLDELETE(ret); + return ret; + } +bool IndexModifier::document(int32_t n, CL_NS(document)::Document* doc){ + SCOPED_LOCK_MUTEX(directory->THIS_LOCK) + assureOpen(); + createIndexReader(); + return indexReader->document(n, doc); +} +CL_NS(store)::Directory* IndexModifier::getDirectory(){ + return directory; +} + +CL_NS_END diff --git a/src/assistant/3rdparty/clucene/src/CLucene/index/IndexModifier.h b/src/assistant/3rdparty/clucene/src/CLucene/index/IndexModifier.h new file mode 100644 index 000000000..4e9963f5a --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/index/IndexModifier.h @@ -0,0 +1,316 @@ +/* +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +* +* Changes are Copyright(C) 2007, 2008 by Nokia Corporation and/or its subsidiary(-ies), all rights reserved. +*/ +#ifndef _lucene_index_IndexModifier_ +#define _lucene_index_IndexModifier_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include <QtCore/QString> + +#include "CLucene/store/Directory.h" +#include "CLucene/document/Document.h" +#include "CLucene/analysis/AnalysisHeader.h" + +CL_NS_DEF(index) + +class IndexReader; +class IndexWriter; +class Term; +class TermDocs; +class TermEnum; + +/** +* A class to modify an index, i.e. to delete and add documents. This +* class hides {@link IndexReader} and {@link IndexWriter} so that you +* do not need to care about implementation details such as that adding +* documents is done via IndexWriter and deletion is done via IndexReader. +* +* <p>Note that you cannot create more than one <code>IndexModifier</code> object +* on the same directory at the same time. +* +* <p>Example usage: +* +* <div align="left" class="java"> +* <table border="0" cellpadding="3" cellspacing="0" bgcolor="#ffffff"> +* <tr> +* <td nowrap="nowrap" valign="top" align="left"> +* <code> +* //note this code will leak memory :) +* Analyzer* analyzer = <b>new</b> StandardAnalyzer();<br/> +* // create an index in /tmp/index, overwriting an existing one:<br/> +* IndexModifier* indexModifier = <b>new</b> IndexModifier("/tmp/index", analyzer, <b>true</b>);<br/> +* Document* doc = <b>new </b>Document*();<br/> +* doc->add(*<b>new </b>Field("id", "1", Field::STORE_YES| Field::INDEX_UNTOKENIZED));<br/> +* doc->add(*<b>new </b>Field("body", "a simple test", Field::STORE_YES, Field::INDEX_TOKENIZED));<br/> +* indexModifier->addDocument(doc);<br/> +* <b>int32_t </b>deleted = indexModifier->deleteDocuments(<b>new </b>Term("id", "1"));<br/> +* printf("Deleted %d document", deleted);<br/> +* indexModifier->flush();<br/> +* printf( "$d docs in index", indexModifier->docCount() );<br/> +* indexModifier->close(); +* </code></td> +* </tr> +* </table> +* </div> +* +* <p>Not all methods of IndexReader and IndexWriter are offered by this +* class. If you need access to additional methods, either use those classes +* directly or implement your own class that extends <code>IndexModifier</code>. +* +* <p>Although an instance of this class can be used from more than one +* thread, you will not get the best performance. You might want to use +* IndexReader and IndexWriter directly for that (but you will need to +* care about synchronization yourself then). +* +* <p>While you can freely mix calls to add() and delete() using this class, +* you should batch you calls for best performance. For example, if you +* want to update 20 documents, you should first delete all those documents, +* then add all the new documents. +* +*/ +class IndexModifier : LUCENE_BASE +{ +protected: + IndexWriter* indexWriter; + IndexReader* indexReader; + + CL_NS(store)::Directory* directory; + CL_NS(analysis)::Analyzer* analyzer; + bool open; + + // Lucene defaults: + bool useCompoundFile; + int32_t maxBufferedDocs; + int32_t maxFieldLength; + int32_t mergeFactor; + +public: + + /** + * Open an index with write access. + * + * @param directory the index directory + * @param analyzer the analyzer to use for adding new documents + * @param create <code>true</code> to create the index or overwrite + * the existing one; <code>false</code> to append to the existing index + */ + IndexModifier(CL_NS(store)::Directory* directory, + CL_NS(analysis)::Analyzer* analyzer, bool create); + + ~IndexModifier(); + + /** + * Open an index with write access. + * + * @param dirName the index directory + * @param analyzer the analyzer to use for adding new documents + * @param create <code>true</code> to create the index or overwrite + * the existing one; <code>false</code> to append to the existing index + */ + IndexModifier(const QString& dirName, CL_NS(analysis)::Analyzer* analyzer, + bool create); + +protected: + + // Initialize an IndexWriter. @throws IOException + void init(CL_NS(store)::Directory* directory, + CL_NS(analysis)::Analyzer* analyzer, bool create); + + // Throw an IllegalStateException if the index is closed. + // @throws IllegalStateException + void assureOpen() const; + + // Close the IndexReader and open an IndexWriter. @throws IOException + void createIndexWriter(); + + // Close the IndexWriter and open an IndexReader. @throws IOException + void createIndexReader(); + +public: + // Make sure all changes are written to disk. @throws IOException + void flush(); + + // Adds a document to this index, using the provided analyzer instead of + // the one specific in the constructor. If the document contains more than + // {@link #setMaxFieldLength(int32_t)} terms for a given field, the + // remainder are discarded. + // @see IndexWriter#addDocument(Document*, Analyzer*) + // @throws IllegalStateException if the index is closed + void addDocument(CL_NS(document)::Document* doc, CL_NS(analysis)::Analyzer* + docAnalyzer = NULL); + + + /** + * Deletes all documents containing <code>term</code>. + * This is useful if one uses a document field to hold a unique ID string for + * the document. Then to delete such a document, one merely constructs a + * term with the appropriate field and the unique ID string as its text and + * passes it to this method. Returns the number of documents deleted. + * @return the number of documents deleted + * @see IndexReader#deleteDocuments(Term*) + * @throws IllegalStateException if the index is closed + */ + int32_t deleteDocuments(Term* term); + + /** + * Deletes the document numbered <code>docNum</code>. + * @see IndexReader#deleteDocument(int32_t) + * @throws IllegalStateException if the index is closed + */ + void deleteDocument(int32_t docNum); + + /** + * Returns the number of documents currently in this index. + * @see IndexWriter#docCount() + * @see IndexReader#numDocs() + * @throws IllegalStateException if the index is closed + */ + int32_t docCount(); + + /** + * Merges all segments together into a single segment, optimizing an index + * for search. + * @see IndexWriter#optimize() + * @throws IllegalStateException if the index is closed + */ + void optimize(); + + /** + * Setting to turn on usage of a compound file. When on, multiple files + * for each segment are merged into a single file once the segment creation + * is finished. This is done regardless of what directory is in use. + * @see IndexWriter#setUseCompoundFile(bool) + * @throws IllegalStateException if the index is closed + */ + void setUseCompoundFile(bool useCompoundFile); + + /** + * @throws IOException + * @see IndexModifier#setUseCompoundFile(bool) + */ + bool getUseCompoundFile(); + + /** + * The maximum number of terms that will be indexed for a single field in a + * document. This limits the amount of memory required for indexing, so that + * collections with very large files will not crash the indexing process by + * running out of memory.<p/> + * Note that this effectively truncates large documents, excluding from the + * index terms that occur further in the document. If you know your source + * documents are large, be sure to set this value high enough to accomodate + * the expected size. If you set it to Integer.MAX_VALUE, then the only limit + * is your memory, but you should anticipate an OutOfMemoryError.<p/> + * By default, no more than 10,000 terms will be indexed for a field. + * @see IndexWriter#setMaxFieldLength(int32_t) + * @throws IllegalStateException if the index is closed + */ + void setMaxFieldLength(int32_t maxFieldLength); + + /** + * @throws IOException + * @see IndexModifier#setMaxFieldLength(int32_t) + */ + int32_t getMaxFieldLength(); + + /* + * The maximum number of terms that will be indexed for a single field in a + * document. This limits the amount of memory required for indexing, so that + * collections with very large files will not crash the indexing process by + * running out of memory.<p/> + * Note that this effectively truncates large documents, excluding from the + * index terms that occur further in the document. If you know your source + * documents are large, be sure to set this value high enough to accomodate + * the expected size. If you set it to Integer.MAX_VALUE, then the only limit + * is your memory, but you should anticipate an OutOfMemoryError.<p/> + * By default, no more than 10,000 terms will be indexed for a field. + * @see IndexWriter#setMaxBufferedDocs(int32_t) + * @throws IllegalStateException if the index is closed + */ + void setMaxBufferedDocs(int32_t maxBufferedDocs); + + // @see IndexModifier#setMaxBufferedDocs(int32_t) @throws IOException + int32_t getMaxBufferedDocs(); + + /* + * Determines how often segment indices are merged by addDocument(). With + * smaller values, less RAM is used while indexing, and searches on + * unoptimized indices are faster, but indexing speed is slower. With larger + * values, more RAM is used during indexing, and while searches on unoptimized + * indices are slower, indexing is faster. Thus larger values (> 10) are + * best for batch index creation, and smaller values (< 10) for indices + * that are interactively maintained. + * <p>This must never be less than 2. The default value is 10. + * + * @see IndexWriter#setMergeFactor(int32_t) + * @throws IllegalStateException if the index is closed + */ + void setMergeFactor(int32_t mergeFactor); + + /** + * @throws IOException + * @see IndexModifier#setMergeFactor(int32_t) + */ + int32_t getMergeFactor(); + + /** + * Close this index, writing all pending changes to disk. + * + * @throws IllegalStateException if the index has been closed before already + */ + void close(); + + QString toString() const; + + /** + * Gets the version number of the currently open index. + */ + int64_t getCurrentVersion() const; + + /** + * Returns an enumeration of all the documents which contain term. + * + * Warning: This is not threadsafe. Make sure you lock the modifier object + * while using the TermDocs. If the IndexReader that the modifier manages + * is closed, the TermDocs object will fail. + */ + TermDocs* termDocs(Term* term = NULL); + + /** + * Returns an enumeration of all terms after a given term. + * If no term is given, an enumeration of all the terms + * in the index is returned. + * The enumeration is ordered by Term.compareTo(). Each term + * is greater than all that precede it in the enumeration. + * + * Warning: This is not threadsafe. Make sure you lock the modifier object + * while using the TermDocs. If the IndexReader that the modifier manages + * is closed, the Document will be invalid + */ + TermEnum* terms(Term* term = NULL); + + /** + * Returns the stored fields of the n-th Document in this index. + * + * Warning: This is not threadsafe. Make sure you lock the modifier object + * while using the TermDocs. If the IndexReader that the modifier manages + * is closed, the Document will be invalid + */ + bool document(const int32_t n, CL_NS(document)::Document* doc); + _CL_DEPRECATED(document(i, document)) + CL_NS(document)::Document* document(const int32_t n); + + // Returns the directory used by this index. + CL_NS(store)::Directory* getDirectory(); +}; + +CL_NS_END + +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/index/IndexReader.cpp b/src/assistant/3rdparty/clucene/src/CLucene/index/IndexReader.cpp new file mode 100644 index 000000000..91c735632 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/index/IndexReader.cpp @@ -0,0 +1,668 @@ +/* + * Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team + * + * Distributable under the terms of either the Apache License (Version 2.0) or + * the GNU Lesser General Public License, as specified in the COPYING file. + * + * Changes are Copyright(C) 2007, 2008 by Nokia Corporation and/or its subsidiary(-ies), all rights reserved. +*/ +# include <QtCore/QFile> +# include <QtCore/QStringList> + +#include "CLucene/StdHeader.h" +#include "IndexReader.h" +#include "IndexWriter.h" + +#include "CLucene/store/Directory.h" +#include "CLucene/store/FSDirectory.h" +#include "CLucene/store/Lock.h" +#include "CLucene/document/Document.h" +#include "CLucene/search/Similarity.h" +#include "SegmentInfos.h" +#include "MultiReader.h" +#include "Terms.h" + +CL_NS_USE(util) +CL_NS_USE(store) +CL_NS_DEF(index) + +IndexReader::IndexReader(Directory* dir) + : stale(false) + , hasChanges(false) + , closeDirectory(false) + , directoryOwner(false) + , segmentInfos(NULL) + , directory(_CL_POINTER(dir)) + , writeLock(NULL) +{ +} + +IndexReader::IndexReader(Directory* dir, SegmentInfos* infos, bool close) + : stale(false) + , hasChanges(false) + , closeDirectory(close) + , directoryOwner(true) + , segmentInfos(infos) + , directory(_CL_POINTER(dir)) + , writeLock(NULL) +{ +} + +IndexReader::~IndexReader() +{ + if (writeLock != NULL) { + writeLock->release(); + _CLDELETE(writeLock); + } + _CLDELETE(segmentInfos); + _CLDECDELETE(directory); +} + +IndexReader* IndexReader::open(const QString& path) +{ + //Func - Static method. + // Returns an IndexReader reading the index in an FSDirectory in the named path. + //Pre - path != NULL and contains the path of the index for which an IndexReader must be + // instantiated + // closeDir indicates if the directory needs to be closed + //Post - An IndexReader has been returned that reads tnhe index located at path + + CND_PRECONDITION(!path.isEmpty(), "path is NULL"); + + Directory* dir = FSDirectory::getDirectory(path, false); + IndexReader* reader = open(dir, true); + //because fsdirectory will now have a refcount of 1 more than + //if the reader had been opened with a directory object, + //we need to do a refdec + _CLDECDELETE(dir); + return reader; +} + +IndexReader* IndexReader::open(Directory* directory, bool closeDirectory) +{ + //Func - Static method. + // Returns an IndexReader reading the index in an FSDirectory in the named path. + //Pre - directory represents a directory + // closeDir indicates if the directory needs to be closed + //Post - An IndexReader has been returned that reads the index located at directory + + // in- & inter-process sync + SCOPED_LOCK_MUTEX(directory->THIS_LOCK) + + //Instantiate an IndexReader::LockWith which can produce an IndexReader + LuceneLock* lock = directory->makeLock(QLatin1String("commit.lock")); + IndexReader::LockWith with(lock, directory); + + IndexReader* ret = NULL; + try { + //Create an IndexReader reading the index + ret = with.runAndReturn(); + } _CLFINALLY ( + _CLDELETE(lock); + ); + + CND_CONDITION(ret != NULL, "ret is NULL"); + ret->closeDirectory = closeDirectory; + + return ret; +} + +CL_NS(document)::Document* IndexReader::document(const int32_t n) +{ + CL_NS(document)::Document* ret = _CLNEW CL_NS(document)::Document; + if (!document(n, ret)) + _CLDELETE(ret); + return ret; +} + +IndexReader* IndexReader::LockWith::doBody() +{ + //Func - Reads the segmentinfo file and depending on the number of segments found + // it returns a SegmentsReader or a SegmentReader + //Pre - directory != NULL + //Post - Depending on the number of Segments present in directory this method + // returns an empty SegmentsReader when there are no segments, a SegmentReader when + // directory contains 1 segment and a nonempty SegmentsReader when directory + // contains multiple segements + + CND_PRECONDITION(directory != NULL, "directory is NULL"); + + //Instantiate SegmentInfos + SegmentInfos* infos = _CLNEW SegmentInfos; + try { + //Have SegmentInfos read the segments file in directory + infos->read(directory); + } catch(...) { + //make sure infos is cleaned up + _CLDELETE(infos); + throw; + } + + // If there is at least one segment (if infos.size() >= 1), the last + // SegmentReader object will close the directory when the SegmentReader + // object itself is closed (see SegmentReader::doClose). + // If there are no segments, there will be no "last SegmentReader object" + // to fulfill this responsibility, so we need to explicitly close the + // directory in the segmentsreader.close + + //Count the number segments in the directory + const uint32_t nSegs = infos->size(); + + if (nSegs == 1 ) { + // index is optimized + return _CLNEW SegmentReader(infos, infos->info(0)); + } else { + //Instantiate an array of pointers to SegmentReaders of size nSegs (The number of segments in the index) + IndexReader** readers = NULL; + + if (nSegs > 0){ + uint32_t infosize = infos->size(); + readers = _CL_NEWARRAY(IndexReader*,infosize+1); + for (uint32_t i = 0; i < infosize; ++i) { + //Instantiate a SegementReader responsible for reading the i-th segment and store it in + //the readers array + readers[i] = _CLNEW SegmentReader(infos->info(i)); + } + readers[infosize] = NULL; + } + + //return an instance of SegmentsReader which is a reader that manages all Segments + return _CLNEW MultiReader(directory, infos, readers); + }// end if +} + +uint64_t IndexReader::lastModified(const QString& directory) +{ + //Func - Static method + // Returns the time the index in the named directory was last modified. + //Pre - directory != NULL and contains the path name of the directory to check + //Post - The last modified time of the index has been returned + + CND_PRECONDITION(!directory.isEmpty(), "directory is NULL"); + + return FSDirectory::fileModified(directory, QLatin1String("segments")); +} + +int64_t IndexReader::getCurrentVersion(Directory* directory) +{ + // in- & inter-process sync + SCOPED_LOCK_MUTEX(directory->THIS_LOCK) + + int64_t ret = 0; + bool locked = false; + LuceneLock* commitLock = directory->makeLock(IndexWriter::COMMIT_LOCK_NAME); + try { + locked = commitLock->obtain(IndexWriter::COMMIT_LOCK_TIMEOUT); + ret = SegmentInfos::readCurrentVersion(directory); + } _CLFINALLY ( + if (locked) + commitLock->release(); + _CLDELETE(commitLock); + ) + return ret; +} + +int64_t IndexReader::getCurrentVersion(const QString& directory) +{ + Directory* dir = FSDirectory::getDirectory(directory, false); + int64_t version = getCurrentVersion(dir); + dir->close(); + _CLDECDELETE(dir); + return version; +} + +int64_t IndexReader::getVersion() +{ + return segmentInfos->getVersion(); +} + +bool IndexReader::isCurrent() +{ + // in- & inter-process sync + SCOPED_LOCK_MUTEX(directory->THIS_LOCK) + + bool ret = false; + bool locked = false; + LuceneLock* commitLock = directory->makeLock(IndexWriter::COMMIT_LOCK_NAME); + try { + locked = commitLock->obtain(IndexWriter::COMMIT_LOCK_TIMEOUT); + ret = SegmentInfos::readCurrentVersion(directory) + == segmentInfos->getVersion(); + } _CLFINALLY( + if (locked) + commitLock->release(); + _CLDELETE(commitLock); + ) + return ret; +} + +uint64_t IndexReader::lastModified(const Directory* directory) +{ + //Func - Static method + // Returns the time the index in this directory was last modified. + //Pre - directory contains a valid reference + //Post - The last modified time of the index has been returned + + return directory->fileModified(QLatin1String("segments")); +} + + +bool IndexReader::indexExists(const QString& directory) +{ + //Func - Static method + // Checks if an index exists in the named directory + //Pre - directory != NULL + //Post - Returns true if an index exists at the specified directory-> + // If the directory does not exist or if there is no index in it. + // false is returned. + + CND_PRECONDITION(!directory.isEmpty(), "directory is NULL"); + return QFile(directory + QLatin1String("/segments")).exists(); +} + + +void IndexReader::setNorm(int32_t doc, const TCHAR* field, uint8_t value) +{ + SCOPED_LOCK_MUTEX(THIS_LOCK) + if(directoryOwner) + aquireWriteLock(); + doSetNorm(doc, field, value); + hasChanges = true; +} + +void IndexReader::aquireWriteLock() +{ + if (stale) { + _CLTHROWA(CL_ERR_IO, + "IndexReader out of date and no longer valid for delete, " + "undelete, or setNorm operations"); + } + + if (writeLock == NULL) { + LuceneLock* writeLock = directory->makeLock(QLatin1String("write.lock")); + if (!writeLock->obtain(IndexWriter::WRITE_LOCK_TIMEOUT)) // obtain write lock + _CLTHROWA(CL_ERR_IO,"Index locked for write"); // + writeLock + this->writeLock = writeLock; + + // we have to check whether index has changed since this reader was opened. + // if so, this reader is no longer valid for deletion + if (SegmentInfos::readCurrentVersion(directory) > segmentInfos->getVersion()) { + stale = true; + this->writeLock->release(); + _CLDELETE(this->writeLock); + _CLTHROWA(CL_ERR_IO,"IndexReader out of date and no longer valid " + "for delete, undelete, or setNorm operations"); + } + } +} + + +void IndexReader::setNorm(int32_t doc, const TCHAR* field, qreal value) +{ + setNorm(doc, field, CL_NS(search)::Similarity::encodeNorm(value)); +} + +bool IndexReader::indexExists(const Directory* directory) +{ + //Func - Static method + // Checks if an index exists in the directory + //Pre - directory is a valid reference + //Post - Returns true if an index exists at the specified directory-> + // If the directory does not exist or if there is no index in it. + // false is returned. + + return directory->fileExists(QLatin1String("segments")); +} + +TermDocs* IndexReader::termDocs(Term* term) const +{ + //Func - Returns an enumeration of all the documents which contain + // term. For each document, the document number, the frequency of + // the term in that document is also provided, for use in search scoring. + // Thus, this method implements the mapping: + // + // Term => <docNum, freq>* + // The enumeration is ordered by document number. Each document number + // is greater than all that precede it in the enumeration. + //Pre - term != NULL + //Post - A reference to TermDocs containing an enumeration of all found documents + // has been returned + + CND_PRECONDITION(term != NULL, "term is NULL"); + + //Reference an instantiated TermDocs instance + TermDocs* _termDocs = termDocs(); + //Seek all documents containing term + _termDocs->seek(term); + //return the enumaration + return _termDocs; +} + +TermPositions* IndexReader::termPositions(Term* term) const +{ + //Func - Returns an enumeration of all the documents which contain term. For each + // document, in addition to the document number and frequency of the term in + // that document, a list of all of the ordinal positions of the term in the document + // is available. Thus, this method implements the mapping: + // + // Term => <docNum, freq,<pos 1, pos 2, ...pos freq-1>>* + // + // This positional information faciliates phrase and proximity searching. + // The enumeration is ordered by document number. Each document number is greater than + // all that precede it in the enumeration. + //Pre - term != NULL + //Post - A reference to TermPositions containing an enumeration of all found documents + // has been returned + + CND_PRECONDITION(term != NULL, "term is NULL"); + + //Reference an instantiated termPositions instance + TermPositions* _termPositions = termPositions(); + //Seek all documents containing term + _termPositions->seek(term); + //return the enumeration + return _termPositions; +} + +void IndexReader::deleteDocument(const int32_t docNum) +{ + //Func - Deletes the document numbered docNum. Once a document is deleted it will not appear + // in TermDocs or TermPostitions enumerations. Attempts to read its field with the document + // method will result in an error. The presence of this document may still be reflected in + // the docFreq statistic, though this will be corrected eventually as the index is further modified. + //Pre - docNum >= 0 + //Post - If successful the document identified by docNum has been deleted. If no writelock + // could be obtained an exception has been thrown stating that the index was locked or has no write access + + SCOPED_LOCK_MUTEX(THIS_LOCK) + + CND_PRECONDITION(docNum >= 0, "docNum is negative"); + + if (directoryOwner) + aquireWriteLock(); + + //Have the document identified by docNum deleted + doDelete(docNum); + hasChanges = true; +} + +/** +* Commit changes resulting from delete, undeleteAll, or setNorm operations +* +* @throws IOException +*/ +void IndexReader::commit() +{ + SCOPED_LOCK_MUTEX(THIS_LOCK) + if(hasChanges){ + if(directoryOwner){ + { + SCOPED_LOCK_MUTEX(directory->THIS_LOCK) // in- & inter-process sync + + LuceneLock* commitLock = directory->makeLock(QLatin1String("commit.lock")); + IndexReader::CommitLockWith cl(commitLock,this); + cl.run(); + _CLDELETE(commitLock); + + } + if (writeLock != NULL) { + writeLock->release(); // release write lock + _CLDELETE(writeLock); + } + }else + doCommit(); + } + hasChanges = false; +} + + +void IndexReader::undeleteAll() +{ + SCOPED_LOCK_MUTEX(THIS_LOCK) + if(directoryOwner) + aquireWriteLock(); + doUndeleteAll(); + hasChanges = true; +} + +int32_t IndexReader::deleteDocuments(Term* term) +{ + //Func - Deletes all documents containing term. This is useful if one uses a + // document field to hold a unique ID string for the document. Then to delete such + // a document, one merely constructs a term with the appropriate field and the unique + // ID string as its text and passes it to this method. + //Pre - term != NULL + //Post - All documents containing term have been deleted. The number of deleted documents + // has been returned + + CND_PRECONDITION(term != NULL, "term is NULL"); + + //Search for the documents contain term + TermDocs* docs = termDocs(term); + + //Check if documents have been found + if ( docs == NULL ){ + return 0; + } + + //initialize + int32_t Counter = 0; + try { + //iterate through the found documents + while (docs->next()) { + //Delete the document + deleteDocument(docs->doc()); + ++Counter; + } + }_CLFINALLY( + //Close the enumeration + docs->close(); + ); + + //Delete the enumeration of found documents + _CLDELETE( docs ); + + //Return the number of deleted documents + return Counter; +} + +TCHAR** IndexReader::getFieldNames() +{ + CL_NS(util)::StringArrayWithDeletor array; + getFieldNames(IndexReader::ALL, array); + + array.setDoDelete(false); + TCHAR** ret = _CL_NEWARRAY(TCHAR*,array.size()+1); + int j=0; + CL_NS(util)::StringArrayWithDeletor::iterator itr = array.begin(); + while ( itr != array.end() ){ + ret[j]=*itr; + ++j;++itr; + } + ret[j]=NULL; + return ret; +} + +TCHAR** IndexReader::getFieldNames(bool indexed) +{ + CL_NS(util)::StringArrayWithDeletor array; + getFieldNames(indexed?IndexReader::INDEXED:IndexReader::UNINDEXED, array); + + array.setDoDelete(false); + TCHAR** ret = _CL_NEWARRAY(TCHAR*,array.size()+1); + int j=0; + CL_NS(util)::StringArrayWithDeletor::iterator itr = array.begin(); + while ( itr != array.end() ){ + ret[j]=*itr; + ++j;++itr; + } + ret[j]=NULL; + return ret; +} + +void IndexReader::close() +{ + //Func - Closes files associated with this index and also saves any new deletions to disk. + // No other methods should be called after this has been called. + //Pre - true + //Post - All files associated with this index have been deleted and new deletions have been + // saved to disk + SCOPED_LOCK_MUTEX(THIS_LOCK) + + CloseCallbackMap::iterator iter; + for (iter = closeCallbacks.begin(); iter != closeCallbacks.end(); iter++) { + CloseCallback callback = *iter->first; + callback(this, iter->second); + } + + commit(); + doClose(); + + if(closeDirectory) { + directory->close(); + _CLDECDELETE(directory); + } +} + +bool IndexReader::isLocked(Directory* directory) +{ + //Func - Static method + // Checks if the index in the directory is currently locked. + //Pre - directory is a valid reference to a directory to check for a lock + //Post - Returns true if the index in the named directory is locked otherwise false + + //Check the existence of the file write.lock and return true when it does and false + //when it doesn't + LuceneLock* l1 = directory->makeLock(QLatin1String("write.lock")); + LuceneLock* l2 = directory->makeLock(QLatin1String("commit.lock")); + + bool ret = l1->isLocked() || l2->isLocked(); + + _CLDELETE(l1); + _CLDELETE(l2); + return ret; +} + +bool IndexReader::isLocked(const QString& directory) +{ + //Func - Static method + // Checks if the index in the named directory is currently locked. + //Pre - directory != NULL and contains the directory to check for a lock + //Post - Returns true if the index in the named directory is locked otherwise false + + CND_PRECONDITION(!directory.isEmpty(), "directory is NULL"); + + Directory* dir = FSDirectory::getDirectory(directory, false); + bool ret = isLocked(dir); + dir->close(); + _CLDECDELETE(dir); + + return ret; +} + +/** Returns true if there are norms stored for this field. */ +bool IndexReader::hasNorms(const TCHAR* field) +{ + // backward compatible implementation. + // SegmentReader has an efficient implementation. + return norms(field) != NULL; +} + +void IndexReader::unlock(const QString& path) +{ + FSDirectory* dir = FSDirectory::getDirectory(path, false); + unlock(dir); + dir->close(); + _CLDECDELETE(dir); +} + +void IndexReader::unlock(Directory* directory) +{ + //Func - Static method + // Forcibly unlocks the index in the named directory-> + // Caution: this should only be used by failure recovery code, + // when it is known that no other process nor thread is in fact + // currently accessing this index. + //Pre - directory is a valid reference to a directory + //Post - The directory has been forcibly unlocked + LuceneLock* lock; + + lock = directory->makeLock(QLatin1String("write.lock")); + lock->release(); + _CLDELETE(lock); + + lock = directory->makeLock(QLatin1String("commit.lock")); + lock->release(); + _CLDELETE(lock); +} + +bool IndexReader::isLuceneFile(const QString& filename) +{ + if (filename.isNull() || filename.isEmpty()) + return false; + + size_t len = filename.length(); + if (len < 6) //need at least x.frx + return false; + + if (filename == QLatin1String("segments")) + return true; + + if (filename == QLatin1String("segments.new")) + return true; + + if (filename == QLatin1String("deletable")) + return true; + + QStringList extList; + extList << QLatin1String(".cfs") + << QLatin1String(".fnm") << QLatin1String(".fdx") << QLatin1String(".fdt") + << QLatin1String(".tii") << QLatin1String(".tis") << QLatin1String(".frq") + << QLatin1String(".prx") << QLatin1String(".del") << QLatin1String(".tvx") + << QLatin1String(".tvd") << QLatin1String(".tvf") << QLatin1String(".tvp"); + + QString suffix = filename.right(4); + if (extList.contains(suffix, Qt::CaseInsensitive)) + return true; + + if (suffix.leftRef(2) == QLatin1String(".f")) { + suffix = suffix.remove(0, 2); + if (suffix.length() > 0) { + for (int i = 0; i < suffix.length(); ++i) { + if (!suffix.at(i).isDigit()) + return false; + } + return true; + } + } + return false; +} + +void IndexReader::addCloseCallback(CloseCallback callback, void* parameter) +{ + closeCallbacks.put(callback, parameter); +} + +// #pragma mark -- IndexReader::LockWith + +IndexReader::LockWith::LockWith(CL_NS(store)::LuceneLock* lock, CL_NS(store)::Directory* dir) + : CL_NS(store)::LuceneLockWith<IndexReader*>(lock, IndexWriter::COMMIT_LOCK_TIMEOUT) +{ + this->directory = dir; +} + +// #pragma mark -- IndexReader::CommitLockWith + +IndexReader::CommitLockWith::CommitLockWith(CL_NS(store)::LuceneLock* lock, IndexReader* r) + : CL_NS(store)::LuceneLockWith<void>(lock,IndexWriter::COMMIT_LOCK_TIMEOUT) + , reader(r) +{ +} + +void IndexReader::CommitLockWith::doBody() +{ + reader->doCommit(); + reader->segmentInfos->write(reader->getDirectory()); +} + +CL_NS_END diff --git a/src/assistant/3rdparty/clucene/src/CLucene/index/IndexReader.h b/src/assistant/3rdparty/clucene/src/CLucene/index/IndexReader.h new file mode 100644 index 000000000..352d16e80 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/index/IndexReader.h @@ -0,0 +1,485 @@ +/* + * Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team + * + * Distributable under the terms of either the Apache License (Version 2.0) or + * the GNU Lesser General Public License, as specified in the COPYING file. + * + * Changes are Copyright(C) 2007, 2008 by Nokia Corporation and/or its subsidiary(-ies), all rights reserved. +*/ +#ifndef _lucene_index_IndexReader_ +#define _lucene_index_IndexReader_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include <QtCore/QString> + +#include "CLucene/store/Directory.h" +#include "CLucene/store/FSDirectory.h" +#include "CLucene/store/Lock.h" +#include "CLucene/document/Document.h" +#include "CLucene/index/TermVector.h" +#include "SegmentInfos.h" +#include "Terms.h" + + +CL_NS_DEF(index) + + +/** IndexReader is an abstract class, providing an interface for accessing an + index. Search of an index is done entirely through this abstract interface, + so that any subclass which implements it is searchable. + + <p> Concrete subclasses of IndexReader are usually constructed with a call to + one of the static <code>open()</code> methods, e.g. {@link #open(String)}. + + <p> For efficiency, in this API documents are often referred to via + <i>document numbers</i>, non-negative integers which each name a unique + document in the index. These document numbers are ephemeral--they may change + as documents are added to and deleted from an index. Clients should thus not + rely on a given document having the same number between sessions. + + <p> An IndexReader can be opened on a directory for which an IndexWriter is + opened already, but it cannot be used to delete documents from the index then. +*/ +class IndexReader : LUCENE_BASE +{ +public: + //Callback for classes that need to know if IndexReader is closing. + typedef void (*CloseCallback)(IndexReader*, void*); + + class CloseCallbackCompare:public CL_NS(util)::Compare::_base{ + public: + bool operator()( CloseCallback t1, CloseCallback t2 ) const{ + return t1 > t2; + } + static void doDelete(CloseCallback dummy){ + } + }; + + + enum FieldOption { + // all fields + ALL = 1, + // all indexed fields + INDEXED = 2, + // all fields which are not indexed + UNINDEXED = 4, + // all fields which are indexed with termvectors enables + INDEXED_WITH_TERMVECTOR = 8, + // all fields which are indexed but don't have termvectors enabled + INDEXED_NO_TERMVECTOR = 16, + // all fields where termvectors are enabled. Please note that only standard termvector fields are returned + TERMVECTOR = 32, + // all field with termvectors wiht positions enabled + TERMVECTOR_WITH_POSITION = 64, + // all fields where termvectors with offset position are set + TERMVECTOR_WITH_OFFSET = 128, + // all fields where termvectors with offset and position values set + TERMVECTOR_WITH_POSITION_OFFSET = 256 + }; + + +private: + bool stale; + bool hasChanges; + bool closeDirectory; + bool directoryOwner; + + SegmentInfos* segmentInfos; + CL_NS(store)::Directory* directory; + CL_NS(store)::LuceneLock* writeLock; + + typedef CL_NS(util)::CLSet<CloseCallback, void*, CloseCallbackCompare, + CloseCallbackCompare> CloseCallbackMap; + CloseCallbackMap closeCallbacks; + + /** Internal use. Implements commit */ + virtual void doCommit() = 0; + + /** + * Tries to acquire the WriteLock on this directory. + * this method is only valid if this IndexReader is directory owner. + * + * @throws IOException If WriteLock cannot be acquired. + */ + void aquireWriteLock(); +protected: + /** + * Constructor used if IndexReader is not owner of its directory. + * This is used for IndexReaders that are used within other IndexReaders that take care or locking directories. + * + * @param directory Directory where IndexReader files reside. + */ + IndexReader(CL_NS(store)::Directory* dir); + + /** + * Constructor used if IndexReader is owner of its directory. + * If IndexReader is owner of its directory, it locks its directory in case of write operations. + * + * @param directory Directory where IndexReader files reside. + * @param segmentInfos Used for write-l + * @param closeDirectory + */ + IndexReader(CL_NS(store)::Directory* directory, SegmentInfos* segmentInfos, bool closeDirectory); + + + /// Implements close. + virtual void doClose() = 0; + + /** Implements setNorm in subclass.*/ + virtual void doSetNorm(int32_t doc, const TCHAR* field, uint8_t value) = 0; + + /** Implements actual undeleteAll() in subclass. */ + virtual void doUndeleteAll() = 0; + + + /** Implements deletion of the document numbered <code>docNum</code>. + * Applications should call {@link #deleteDocument(int32_t)} or {@link #deleteDocuments(Term*)}. + */ + virtual void doDelete(const int32_t docNum) = 0; + +public: + + DEFINE_MUTEX(THIS_LOCK) + + ///Do not access this directly, only public so that MultiReader can access it + virtual void commit(); + + + /** Undeletes all documents currently marked as deleted in this index.*/ + void undeleteAll(); + + /** + * Get a list of unique field names that exist in this index and have the specified + * field option information. + * @param fldOption specifies which field option should be available for the returned fields + * @return Collection of Strings indicating the names of the fields. + * @see IndexReader.FieldOption + */ + virtual void getFieldNames(FieldOption fldOption, CL_NS(util)::StringArrayWithDeletor& retarray) = 0; + + _CL_DEPRECATED( getFieldNames(FieldOption, StringArrayWithDeletor&) ) virtual TCHAR** getFieldNames(); + _CL_DEPRECATED( getFieldNames(FieldOption, StringArrayWithDeletor&) ) virtual TCHAR** getFieldNames(bool indexed); + + /** Returns the byte-encoded normalization factor for the named field of + * every document. This is used by the search code to score documents. + * + * The number of bytes returned is the size of the IndexReader->maxDoc() + * MEMORY: The values are cached, so don't delete the returned byte array. + * @see Field#setBoost(qreal) + */ + virtual uint8_t* norms(const TCHAR* field) = 0; + + + /** Reads the byte-encoded normalization factor for the named field of every + * document. This is used by the search code to score documents. + * + * @see Field#setBoost(qreal) + */ + virtual void norms(const TCHAR* field, uint8_t* bytes) = 0; + + /** Expert: Resets the normalization factor for the named field of the named + * document. + * + * @see #norms(TCHAR*) + * @see Similarity#decodeNorm(uint8_t) + */ + void setNorm(int32_t doc, const TCHAR* field, qreal value); + + /** Expert: Resets the normalization factor for the named field of the named + * document. The norm represents the product of the field's {@link + * Field#setBoost(qreal) boost} and its {@link Similarity#lengthNorm(TCHAR*, + * int32_t) length normalization}. Thus, to preserve the length normalization + * values when resetting this, one should base the new value upon the old. + * + * @see #norms(TCHAR*) + * @see Similarity#decodeNorm(uint8_t) + */ + void setNorm(int32_t doc, const TCHAR* field, uint8_t value); + + /// Release the write lock, if needed. + virtual ~IndexReader(); + + /// Returns an IndexReader reading the index in an FSDirectory in the named path. + static IndexReader* open(const QString& path); + + /// Returns an IndexReader reading the index in the given Directory. + static IndexReader* open( CL_NS(store)::Directory* directory, bool closeDirectory=false); + + /** + * Returns the time the index in the named directory was last modified. + * Do not use this to check whether the reader is still up-to-date, use + * {@link #isCurrent()} instead. + */ + static uint64_t lastModified(const QString& directory); + + /** + * Returns the time the index in the named directory was last modified. + * Do not use this to check whether the reader is still up-to-date, use + * {@link #isCurrent()} instead. + */ + static uint64_t lastModified(const CL_NS(store)::Directory* directory); + + + /** + * Reads version number from segments files. The version number is + * initialized with a timestamp and then increased by one for each change of + * the index. + * + * @param directory where the index resides. + * @return version number. + * @throws IOException if segments file cannot be read + */ + static int64_t getCurrentVersion(CL_NS(store)::Directory* directory); + + /** + * Reads version number from segments files. The version number is + * initialized with a timestamp and then increased by one for each change of + * the index. + * + * @param directory where the index resides. + * @return version number. + * @throws IOException if segments file cannot be read + */ + static int64_t getCurrentVersion(const QString& directory); + + /** + * Version number when this IndexReader was opened. + */ + int64_t getVersion(); + + /** + * Check whether this IndexReader still works on a current version of the index. + * If this is not the case you will need to re-open the IndexReader to + * make sure you see the latest changes made to the index. + * + * @throws IOException + */ + bool isCurrent(); + + + /** + * Return an array of term frequency vectors for the specified document. + * The array contains a vector for each vectorized field in the document. + * Each vector contains terms and frequencies for all terms in a given vectorized field. + * If no such fields existed, the method returns null. The term vectors that are + * returned my either be of type TermFreqVector or of type TermPositionsVector if + * positions or offsets have been stored. + * + * @param docNumber document for which term frequency vectors are returned + * @return array of term frequency vectors. May be null if no term vectors have been + * stored for the specified document. + * @throws IOException if index cannot be accessed + * @see org.apache.lucene.document.Field.TermVector + */ + virtual bool getTermFreqVectors(int32_t docNumber, Array<TermFreqVector*>& result) =0; + + /** + * Return a term frequency vector for the specified document and field. The + * returned vector contains terms and frequencies for the terms in + * the specified field of this document, if the field had the storeTermVector + * flag set. If termvectors had been stored with positions or offsets, a + * TermPositionsVector is returned. + * + * @param docNumber document for which the term frequency vector is returned + * @param field field for which the term frequency vector is returned. + * @return term frequency vector May be null if field does not exist in the specified + * document or term vector was not stored. + * @throws IOException if index cannot be accessed + * @see org.apache.lucene.document.Field.TermVector + */ + virtual TermFreqVector* getTermFreqVector(int32_t docNumber, const TCHAR* field) = 0; + + /** + * Returns <code>true</code> if an index exists at the specified directory. + * If the directory does not exist or if there is no index in it. + * @param directory the directory to check for an index + * @return <code>true</code> if an index exists; <code>false</code> otherwise + */ + static bool indexExists(const QString& directory); + + /** + * Returns <code>true</code> if an index exists at the specified directory. + * If the directory does not exist or if there is no index in it. + * @param directory the directory to check for an index + * @return <code>true</code> if an index exists; <code>false</code> otherwise + * @throws IOException if there is a problem with accessing the index + */ + static bool indexExists(const CL_NS(store)::Directory* directory); + + /** Returns the number of documents in this index. */ + virtual int32_t numDocs() = 0; + + /** Returns one greater than the largest possible document number. + * This may be used to, e.g., determine how big to allocate an array which + * will have an element for every document number in an index. + */ + virtual int32_t maxDoc() const = 0; + + /** Gets the stored fields of the <code>n</code><sup>th</sup> + * <code>Document</code> in this index. + * The fields are not cleared before retrieving the document, so the + * object should be new or just cleared. + */ + virtual bool document(int32_t n, CL_NS(document)::Document*) =0; + + _CL_DEPRECATED( document(i, document) ) CL_NS(document)::Document* document(const int32_t n); + + /** Returns true if document <i>n</i> has been deleted */ + virtual bool isDeleted(const int32_t n) = 0; + + /** Returns true if any documents have been deleted */ + virtual bool hasDeletions() const = 0; + + /** Returns true if there are norms stored for this field. */ + virtual bool hasNorms(const TCHAR* field); + + /** Returns an enumeration of all the terms in the index. + * The enumeration is ordered by Term.compareTo(). Each term + * is greater than all that precede it in the enumeration. + * @memory Caller must clean up + */ + virtual TermEnum* terms() const =0; + + /** Returns an enumeration of all terms after a given term. + * The enumeration is ordered by Term.compareTo(). Each term + * is greater than all that precede it in the enumeration. + * @memory Caller must clean up + */ + virtual TermEnum* terms(const Term* t) const = 0; + + /** Returns the number of documents containing the term <code>t</code>. */ + virtual int32_t docFreq(const Term* t) const = 0; + + /* Returns an unpositioned TermPositions enumerator. + * @memory Caller must clean up + */ + virtual TermPositions* termPositions() const = 0; + + /** Returns an enumeration of all the documents which contain + * <code>term</code>. For each document, in addition to the document number + * and frequency of the term in that document, a list of all of the ordinal + * positions of the term in the document is available. Thus, this method + * implements the mapping: + * + * <p><ul> + * Term => <docNum, freq, + * <pos<sub>1</sub>, pos<sub>2</sub>, ... + * pos<sub>freq-1</sub>> + * ><sup>*</sup> + * </ul> + * <p> This positional information faciliates phrase and proximity searching. + * <p>The enumeration is ordered by document number. Each document number is + * greater than all that precede it in the enumeration. + * @memory Caller must clean up + */ + TermPositions* termPositions(Term* term) const; + + /** Returns an unpositioned {@link TermDocs} enumerator. + * @memory Caller must clean up + */ + virtual TermDocs* termDocs() const = 0; + + /** Returns an enumeration of all the documents which contain + * <code>term</code>. For each document, the document number, the frequency of + * the term in that document is also provided, for use in search scoring. + * Thus, this method implements the mapping: + * <p><ul>Term => <docNum, freq><sup>*</sup></ul> + * <p>The enumeration is ordered by document number. Each document number + * is greater than all that precede it in the enumeration. + * @memory Caller must clean up + */ + TermDocs* termDocs(Term* term) const; + + /** Deletes the document numbered <code>docNum</code>. Once a document is + * deleted it will not appear in TermDocs or TermPostitions enumerations. + * Attempts to read its field with the {@link #document} + * method will result in an error. The presence of this document may still be + * reflected in the {@link #docFreq} statistic, though + * this will be corrected eventually as the index is further modified. + */ + void deleteDocument(const int32_t docNum); + + ///@deprecated. Use deleteDocument instead. + _CL_DEPRECATED( deleteDocument ) void deleteDoc(const int32_t docNum) + { deleteDocument(docNum); } + + /** Deletes all documents containing <code>term</code>. + * This is useful if one uses a document field to hold a unique ID string for + * the document. Then to delete such a document, one merely constructs a + * term with the appropriate field and the unique ID string as its text and + * passes it to this method. + * See {@link #deleteDocument(int)} for information about when this deletion will + * become effective. + * @return the number of documents deleted + */ + int32_t deleteDocuments(Term* term); + + ///@deprecated. Use deleteDocuments instead. + _CL_DEPRECATED( deleteDocuments ) int32_t deleteTerm(Term* term){ return deleteDocuments(term); } + + /** + * Closes files associated with this index and also saves any new deletions to disk. + * No other methods should be called after this has been called. + */ + void close(); + + ///Checks if the index in the named directory is currently locked. + static bool isLocked(CL_NS(store)::Directory* directory); + + ///Checks if the index in the named directory is currently locked. + static bool isLocked(const QString& directory); + + + ///Forcibly unlocks the index in the named directory. + ///Caution: this should only be used by failure recovery code, + ///when it is known that no other process nor thread is in fact + ///currently accessing this index. + static void unlock(CL_NS(store)::Directory* directory); + static void unlock(const QString& path); + + /** Returns the directory this index resides in. */ + CL_NS(store)::Directory* getDirectory() { return directory; } + + /** Returns true if the file is a lucene filename (based on extension or filename) */ + static bool isLuceneFile(const QString& filename); + + /** + * For classes that need to know when the IndexReader closes (such as caches, etc), + * should pass their callback function to this. + */ + void addCloseCallback(CloseCallback callback, void* parameter); + +protected: + class LockWith : public CL_NS(store)::LuceneLockWith<IndexReader*> + { + public: + LockWith(CL_NS(store)::LuceneLock* lock, CL_NS(store)::Directory* dir); + + //Reads the segmentinfo file and depending on the number of segments found + //it returns a MultiReader or a SegmentReader + IndexReader* doBody(); + + private: + CL_NS(store)::Directory* directory; + }; + friend class IndexReader::LockWith; + + class CommitLockWith : public CL_NS(store)::LuceneLockWith<void> + { + public: + CommitLockWith(CL_NS(store)::LuceneLock* lock, IndexReader* r); + void doBody(); + + private: + IndexReader* reader; + }; + friend class IndexReader::CommitLockWith; +}; + +CL_NS_END +#endif + + diff --git a/src/assistant/3rdparty/clucene/src/CLucene/index/IndexWriter.cpp b/src/assistant/3rdparty/clucene/src/CLucene/index/IndexWriter.cpp new file mode 100644 index 000000000..5504cf6fa --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/index/IndexWriter.cpp @@ -0,0 +1,697 @@ +/* + * Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team + * + * Distributable under the terms of either the Apache License (Version 2.0) or + * the GNU Lesser General Public License, as specified in the COPYING file. + * + * Changes are Copyright(C) 2007, 2008 by Nokia Corporation and/or its subsidiary(-ies), all rights reserved. +*/ +#include "CLucene/StdHeader.h" +#include "IndexWriter.h" + +#include "CLucene/document/Document.h" +#include "CLucene/store/Directory.h" +#include "CLucene/store/Lock.h" +#include "CLucene/util/VoidList.h" +#include "DocumentWriter.h" +#include "SegmentInfos.h" +#include "SegmentMerger.h" + +CL_NS_USE(store) +CL_NS_USE(util) +CL_NS_USE(document) +CL_NS_USE(analysis) +CL_NS_DEF(index) + +const QLatin1String IndexWriter::WRITE_LOCK_NAME("write.lock"); +const QLatin1String IndexWriter::COMMIT_LOCK_NAME("commit.lock"); + +IndexWriter::IndexWriter(const QString& path, Analyzer* a, const bool create, + const bool _closeDir) + : directory(FSDirectory::getDirectory(path, create)) + , analyzer(a) + , segmentInfos(true) + , closeDir(_closeDir) +{ + //Func - Constructor + // Constructs an IndexWriter for the index in path. + //Pre - path != NULL and contains a named directory path + // a holds a valid reference to an analyzer and analyzes the text to + // be indexed create indicates if the indexWriter must create a new + // index located at path or just open it + //Post - If create is true, then a new, empty index has been created in + // path, replacing the index already there, if any. The named + // directory path is owned by this Instance + + CND_PRECONDITION(!path.isEmpty(), "path is NULL"); + + //Continue initializing the instance by _IndexWriter + _IndexWriter(create); +} + +IndexWriter::IndexWriter(Directory* d, Analyzer* a, const bool create, + const bool _closeDir) + : directory(_CL_POINTER(d)) + , analyzer(a) + , segmentInfos(true) + , closeDir(_closeDir) +{ + //Func - Constructor + // Constructs an IndexWriter for the index in path. + //Pre - d contains a valid reference to a directory + // a holds a valid reference to an analyzer and analyzes the text to + // be indexed create indicates if the indexWriter must create a new + // index located at path or just open it + //Post - If create is true, then a new, empty index has been created in + // path, replacing the index already there, if any. The directory d + // is not owned by this Instance + + //Continue initializing the instance by _IndexWriter + _IndexWriter ( create ); +} + +void IndexWriter::_IndexWriter(const bool create) +{ + //Func - Initialises the instances + //Pre - create indicates if the indexWriter must create a new index + // located at path or just open it + + similarity = CL_NS(search)::Similarity::getDefault(); + + useCompoundFile = true; + if ( directory->getDirectoryType() == RAMDirectory::DirectoryType() ) + useCompoundFile = false; + + //Create a ramDirectory + ramDirectory = _CLNEW TransactionalRAMDirectory; + + CND_CONDITION(ramDirectory != NULL, "ramDirectory is NULL"); + + //Initialize the writeLock to + writeLock = NULL; + + //initialise the settings... + maxFieldLength = DEFAULT_MAX_FIELD_LENGTH; + mergeFactor = DEFAULT_MERGE_FACTOR; + maxMergeDocs = DEFAULT_MAX_MERGE_DOCS; + writeLockTimeout = WRITE_LOCK_TIMEOUT; + commitLockTimeout = COMMIT_LOCK_TIMEOUT; + minMergeDocs = DEFAULT_MAX_BUFFERED_DOCS; + termIndexInterval = DEFAULT_TERM_INDEX_INTERVAL; + + //Create a new lock using the name "write.lock" + LuceneLock* newLock = directory->makeLock(IndexWriter::WRITE_LOCK_NAME); + + //Condition check to see if newLock has been allocated properly + CND_CONDITION(newLock != NULL, + "No memory could be allocated for LuceneLock newLock"); + + //Try to obtain a write lock + if (!newLock->obtain(writeLockTimeout)){ + //Write lock could not be obtained so delete it + _CLDELETE(newLock); + //Reset the instance + _finalize(); + //throw an exception because no writelock could be created or obtained + _CLTHROWA(CL_ERR_IO, "Index locked for write or no write access." ); + } + + //The Write Lock has been obtained so save it for later use + this->writeLock = newLock; + + //Create a new lock using the name "commit.lock" + LuceneLock* lock = directory->makeLock(IndexWriter::COMMIT_LOCK_NAME); + + //Condition check to see if lock has been allocated properly + CND_CONDITION(lock != NULL, "No memory could be allocated for LuceneLock lock"); + + LockWith2 with(lock, commitLockTimeout, this, NULL, create); + { + SCOPED_LOCK_MUTEX(directory->THIS_LOCK) // in- & inter-process sync + with.run(); + } + + //Release the commit lock + _CLDELETE(lock); + + isOpen = true; +} + +IndexWriter::~IndexWriter() +{ + //Func - Destructor + //Pre - true + //Post - The instance has been destroyed + close(); + _finalize(); +} + +void IndexWriter::close() +{ + //Func - Flushes all changes to an index, closes all associated files, and + // closes the directory that the index is stored in. + //Pre - closeDir indicates if the directory must be closed or not + //Post - All the changes have been flushed to disk and the write lock has + // been released. The ramDirectory has also been closed. The + // directory has been closed if the reference count of the directory + // reaches zero + + SCOPED_LOCK_MUTEX(THIS_LOCK) + if (isOpen) { + //Flush the Ram Segments + flushRamSegments(); + //Close the ram directory + if (ramDirectory != NULL) { + ramDirectory->close(); + _CLDECDELETE(ramDirectory); + } + + //Check if this instance must close the directory + if (closeDir) + directory->close(); + _CLDECDELETE(directory); + + // release write lock + if (writeLock != NULL) { + writeLock->release(); + _CLDELETE(writeLock); + } + isOpen = false; + } +} + +void IndexWriter::_finalize() +{ + //Func - Releases all the resources of the instance + //Pre - true + //Post - All the releases have been released + + if(writeLock != NULL) { + //release write lock + writeLock->release(); + _CLDELETE( writeLock ); + } + + //Delete the ramDirectory + if (ramDirectory != NULL) { + ramDirectory->close(); + _CLDECDELETE(ramDirectory); + } +} + +int32_t IndexWriter::docCount() +{ + //Func - Counts the number of documents in the index + //Pre - true + //Post - The number of documents have been returned + + SCOPED_LOCK_MUTEX(THIS_LOCK) + + //Initialize count + int32_t count = 0; + + //Iterate through all segmentInfos + for (int32_t i = 0; i < segmentInfos.size(); i++) { + //Get the i-th SegmentInfo + SegmentInfo* si = segmentInfos.info(i); + //Retrieve the number of documents of the segment and add it to count + count += si->docCount; + } + return count; +} + +void IndexWriter::addDocument(Document* doc, Analyzer* analyzer) +{ + //Func - Adds a document to the index + //Pre - doc contains a valid reference to a document + // ramDirectory != NULL + //Post - The document has been added to the index of this IndexWriter + CND_PRECONDITION(ramDirectory != NULL, "ramDirectory is NULL"); + + if (analyzer == NULL) + analyzer = this->analyzer; + + ramDirectory->transStart(); + try { + QString segmentName = newSegmentName(); + CND_CONDITION(!segmentName.isEmpty(), "segmentName is NULL"); + try { + //Create the DocumentWriter using a ramDirectory and analyzer + // supplied by the IndexWriter (this). + DocumentWriter* dw = _CLNEW DocumentWriter(ramDirectory, analyzer, + this ); + CND_CONDITION(dw != NULL, "dw is NULL"); + try { + //Add the client-supplied document to the new segment. + dw->addDocument(segmentName, doc); + } _CLFINALLY ( + _CLDELETE(dw); + ); + + //Create a new SegmentInfo instance about this new segment. + SegmentInfo* si = _CLNEW SegmentInfo(segmentName, 1, ramDirectory); + CND_CONDITION(si != NULL, "Si is NULL"); + { + SCOPED_LOCK_MUTEX(THIS_LOCK) + + //Add the info object for this particular segment to the list + // of all segmentInfos-> + segmentInfos.add(si); + + //Check to see if the segments must be merged + maybeMergeSegments(); + } + } _CLFINALLY() + } catch (...) { + ramDirectory->transAbort(); + throw; +} + ramDirectory->transCommit(); +} + +void IndexWriter::optimize() +{ + //Func - Optimizes the index for which this Instance is responsible + //Pre - true + //Post - + SCOPED_LOCK_MUTEX(THIS_LOCK) + + //Flush the RamSegments to disk + flushRamSegments(); + while (segmentInfos.size() > 1 + || (segmentInfos.size() == 1 + && (SegmentReader::hasDeletions(segmentInfos.info(0)) + || segmentInfos.info(0)->getDir()!=directory + || (useCompoundFile + && (!SegmentReader::usesCompoundFile(segmentInfos.info(0)) + || SegmentReader::hasSeparateNorms(segmentInfos.info(0))))))) { + int32_t minSegment = segmentInfos.size() - mergeFactor; + mergeSegments(minSegment < 0 ? 0 : minSegment); + } +} + + +QString IndexWriter::newSegmentName() +{ + SCOPED_LOCK_MUTEX(THIS_LOCK) + return QLatin1Char('_') + QString::number(segmentInfos.counter++, 36); +} + +void IndexWriter::flushRamSegments() +{ + //Func - Merges all RAM-resident segments. + //Pre - ramDirectory != NULL + //Post - The RAM-resident segments have been merged to disk + + CND_PRECONDITION(ramDirectory != NULL, "ramDirectory is NULL"); + + int32_t minSegment = segmentInfos.size()-1; //don't make this unsigned... + CND_CONDITION(minSegment >= -1, "minSegment must be >= -1"); + + int32_t docCount = 0; + //Iterate through all the segements and check if the directory is a ramDirectory + while (minSegment >= 0 && + segmentInfos.info(minSegment)->getDir() == ramDirectory) { + docCount += segmentInfos.info(minSegment)->docCount; + minSegment--; + } + if (minSegment < 0 || // add one FS segment? + (docCount + segmentInfos.info(minSegment)->docCount) > mergeFactor || + !(segmentInfos.info(segmentInfos.size()-1)->getDir() == ramDirectory)) + minSegment++; + + CND_CONDITION(minSegment >= 0, "minSegment must be >= 0"); + if (minSegment >= segmentInfos.size()) + return; // none to merge + mergeSegments(minSegment); +} + +void IndexWriter::maybeMergeSegments() { + //Func - Incremental Segment Merger + //Pre - + //Post - + + int64_t targetMergeDocs = minMergeDocs; + + // find segments smaller than current target size + while (targetMergeDocs <= maxMergeDocs) { + int32_t minSegment = segmentInfos.size(); + int32_t mergeDocs = 0; + + while (--minSegment >= 0) { + SegmentInfo* si = segmentInfos.info(minSegment); + if (si->docCount >= targetMergeDocs) + break; + mergeDocs += si->docCount; + } + + if (mergeDocs >= targetMergeDocs){ + // found a merge to do + mergeSegments(minSegment+1); + }else + break; + + //increase target size + targetMergeDocs *= mergeFactor; + } +} + +void IndexWriter::mergeSegments(const uint32_t minSegment) +{ + mergeSegments(minSegment, segmentInfos.size()); +} + +void IndexWriter::mergeSegments(const uint32_t minSegment, const uint32_t end) +{ + CLVector<SegmentReader*> segmentsToDelete(false); + QString mergedName = newSegmentName(); +#ifdef _CL_DEBUG_INFO + fprintf(_CL_DEBUG_INFO, "merging segments\n"); +#endif + SegmentMerger merger(this, mergedName); + for (size_t i = minSegment; i < end; i++) { + SegmentInfo* si = segmentInfos.info(i); +#ifdef _CL_DEBUG_INFO + fprintf(_CL_DEBUG_INFO, " %s (%d docs)\n", + si->name.toLocal8Bit().constData(), si->docCount); +#endif + SegmentReader* reader = _CLNEW SegmentReader(si); + merger.add(reader); + // if we own the directory + if ((reader->getDirectory() == this->directory) + || (reader->getDirectory() == this->ramDirectory)) { + // queue segment for deletion + segmentsToDelete.push_back(reader); + } + } + + int32_t mergedDocCount = merger.merge(); + +#ifdef _CL_DEBUG_INFO + fprintf(_CL_DEBUG_INFO, "\n into %s (%d docs)\n", + mergedName.toLocal8Bit().constData(), mergedDocCount); +#endif + + segmentInfos.clearto(minSegment);// remove old infos & add new + segmentInfos.add(_CLNEW SegmentInfo(mergedName, mergedDocCount, directory)); + + // close readers before we attempt to delete now-obsolete segments + merger.closeReaders(); + + LuceneLock* lock = directory->makeLock(IndexWriter::COMMIT_LOCK_NAME); + LockWith2 with (lock, commitLockTimeout, this, &segmentsToDelete, true); + { + SCOPED_LOCK_MUTEX(directory->THIS_LOCK) // in- & inter-process sync + with.run(); + } + _CLDELETE( lock ); + + if (useCompoundFile) { + QStringList filesToDelete; + merger.createCompoundFile(mergedName + QLatin1String(".tmp"), filesToDelete); + + LuceneLock* lock = directory->makeLock(IndexWriter::COMMIT_LOCK_NAME); + LockWithCFS with(lock, commitLockTimeout, directory, this, mergedName, + filesToDelete); + { + SCOPED_LOCK_MUTEX(directory->THIS_LOCK) // in- & inter-process sync + with.run(); + } + _CLDELETE(lock); + } +} + +void IndexWriter::deleteSegments(CLVector<SegmentReader*>* segments) +{ + QStringList deletable; + + {//scope delete deleteArray object + QStringList deleteArray; + readDeleteableFiles(deleteArray); + deleteFiles(deleteArray, deletable); // try to delete deleteable + } + + QStringList files; + for (uint32_t i = 0; i < segments->size(); i++) { + SegmentReader* reader = (*segments)[i]; + files.clear(); + reader->files(files); + if (reader->getDirectory() == this->directory) + deleteFiles(files, deletable); // try to delete our files + else + deleteFiles(files, reader->getDirectory()); // delete, eg, RAM files + } + + writeDeleteableFiles(deletable); // note files we can't delete +} + +void IndexWriter::deleteFiles(const QStringList& files) +{ + QStringList currentDeletable; + readDeleteableFiles(currentDeletable); + + // try to delete deleteable + QStringList deletable; + deleteFiles(currentDeletable, deletable); + + // try to delete our files + deleteFiles(files, deletable); + + // note files we can't delete + writeDeleteableFiles(deletable); +} + +void IndexWriter::readDeleteableFiles(QStringList& result) +{ + if (!directory->fileExists(QLatin1String("deletable"))) + return; + + IndexInput* input = directory->openInput(QLatin1String("deletable")); + try { + // read file names + TCHAR tname[CL_MAX_PATH]; + for (int32_t i = input->readInt(); i > 0; i--) { + int32_t read = input->readString(tname, CL_MAX_PATH); + result.push_back(QString::fromWCharArray(tname, read)); + } + } _CLFINALLY ( + input->close(); + _CLDELETE(input); + ); +} + +void IndexWriter::deleteFiles(const QStringList& files, QStringList& deletable) +{ + QStringList::const_iterator itr; + for (itr = files.begin(); itr != files.end(); ++itr) { + if (!getDirectory()->fileExists((*itr))) + continue; + + if (!getDirectory()->deleteFile((*itr), false)) { + if (directory->fileExists((*itr))) { +#ifdef _CL_DEBUG_INFO + fprintf(_CL_DEBUG_INFO, "%s; Will re-try later.\n", err.what()); +#endif + // add to deletable + deletable.push_back((*itr)); + } + } + } +} + +void IndexWriter::deleteFiles(const QStringList& files, Directory* directory) +{ + QStringList::const_iterator itr; + for (itr = files.begin(); itr != files.end(); ++itr) + directory->deleteFile((*itr), true); +} + +void IndexWriter::writeDeleteableFiles(const QStringList& files) +{ + IndexOutput* output = directory->createOutput(QLatin1String("deleteable.new")); + try { + output->writeInt(files.size()); + + TCHAR tfile[CL_MAX_PATH]; + QStringList::const_iterator itr; + for (itr = files.begin(); itr != files.end(); ++itr) { + tfile[(*itr).toWCharArray(tfile)] = '\0'; + output->writeString(tfile, _tcslen(tfile)); + } + } _CLFINALLY ( + output->close(); + _CLDELETE(output); + ); + + directory->renameFile(QLatin1String("deleteable.new"), + QLatin1String("deletable")); +} + +void IndexWriter::addIndexes(Directory** dirs) +{ + //Func - Add several indexes located in different directories into the current + // one managed by this instance + //Pre - dirs != NULL and contains directories of several indexes + // dirsLength > 0 and contains the number of directories + //Post - The indexes located in the directories in dirs have been merged with + // the pre(current) index. The Resulting index has also been optimized + + SCOPED_LOCK_MUTEX(THIS_LOCK) + + CND_PRECONDITION(dirs != NULL, "dirs is NULL"); + + // start with zero or 1 seg so optimize the current + optimize(); + + int32_t start = segmentInfos.size(); + + //Iterate through the directories + for (int32_t i = 0; dirs[i] != NULL; ++i) { + // DSR: Changed SegmentInfos constructor arg (see bug discussion below). + SegmentInfos sis(false); + sis.read(dirs[i]); + for (int32_t j = 0; j < sis.size(); j++) + segmentInfos.add(sis.info(j)); // add each info + } + + // commented out by tbusch to solve a bug and to be conform with + // java lucene + + // merge newly added segments in log(n) passes + //while (segmentInfos.size() > start + mergeFactor) { + // for (int32_t base = start; base < segmentInfos.size(); base++) { + // int32_t end = min(segmentInfos.size(), base + mergeFactor); + // if (end - base > 1) + // mergeSegments(base, end); + // } + //} + + // cleanup + optimize(); +} + + +void IndexWriter::addIndexes(IndexReader** readers) +{ + SCOPED_LOCK_MUTEX(THIS_LOCK) + optimize(); // start with zero or 1 seg + + QString mergedName = newSegmentName(); + SegmentMerger merger(this, mergedName); + + CLVector<SegmentReader*> segmentsToDelete; + SegmentReader* sReader = NULL; + if (segmentInfos.size() == 1) { // add existing index, if any + sReader = _CLNEW SegmentReader(segmentInfos.info(0)); + merger.add(sReader); + segmentsToDelete.push_back(sReader); // queue segment for deletion + } + + int32_t readersLength = 0; + while (readers[readersLength] != NULL) + merger.add(readers[readersLength++]); + + int32_t docCount = merger.merge(); // merge 'em + + // pop old infos & add new + segmentInfos.clearto(0); + segmentInfos.add(_CLNEW SegmentInfo(mergedName, docCount, directory)); + + if (sReader != NULL) { + sReader->close(); + _CLDELETE(sReader); + } + + LuceneLock* lock = directory->makeLock(IndexWriter::COMMIT_LOCK_NAME); + LockWith2 with(lock, commitLockTimeout, this, &segmentsToDelete, true); + { + // in- & inter-process sync + SCOPED_LOCK_MUTEX(directory->THIS_LOCK) + with.run(); + } + _CLDELETE(lock); + + if (useCompoundFile) { + QStringList filesToDelete; + merger.createCompoundFile(mergedName + QLatin1String(".tmp"), + filesToDelete); + + LuceneLock* cfslock = directory->makeLock(IndexWriter::COMMIT_LOCK_NAME); + LockWithCFS with(cfslock, commitLockTimeout, directory, this, mergedName, + filesToDelete); + { + // in- & inter-process sync + SCOPED_LOCK_MUTEX(directory->THIS_LOCK) + with.run(); + } + _CLDELETE(cfslock); + } +} + +// #pragma mark -- IndexWriter::LockWith2 + +IndexWriter::LockWith2::LockWith2(CL_NS(store)::LuceneLock* lock, + int64_t lockWaitTimeout, + IndexWriter* indexWriter, + CL_NS(util)::CLVector<SegmentReader*>* std, + bool _create) + : CL_NS(store)::LuceneLockWith<void>(lock, lockWaitTimeout) + , create(_create) + , writer(indexWriter) + , segmentsToDelete(std) +{ +} + +void IndexWriter::LockWith2::doBody() +{ + //Func - Writes segmentInfos to or reads segmentInfos from disk + //Pre - writer != NULL + //Post - if create is true then segementInfos has been written to disk + // otherwise segmentInfos has been read from disk + + CND_PRECONDITION(writer != NULL, "writer is NULL"); + + if (create) { + writer->segmentInfos.write(writer->getDirectory()); + // delete now-unused segments + if (segmentsToDelete != NULL) + writer->deleteSegments(segmentsToDelete); + } else { + writer->segmentInfos.read(writer->getDirectory()); + } +} + +// #pragma mark -- IndexWriter::LockWithCFS + +IndexWriter::LockWithCFS::LockWithCFS(CL_NS(store)::LuceneLock* lock, + int64_t lockWaitTimeout, + CL_NS(store)::Directory* dir, + IndexWriter* indexWriter, + const QString& segmentName, + const QStringList& ftd) + : CL_NS(store)::LuceneLockWith<void>(lock, lockWaitTimeout) + , segName(segmentName) + , writer(indexWriter) + , directory(dir) + , filesToDelete(ftd) +{ +} + +void IndexWriter::LockWithCFS::doBody() +{ + //Func - Writes segmentInfos to or reads segmentInfos from disk + //Pre - writer != NULL + //Post - if create is true then segementInfos has been written to disk + // otherwise segmentInfos has been read from disk + + CND_PRECONDITION(directory != NULL, "directory is NULL"); + CND_PRECONDITION(!segName.isEmpty(), "mergedName is NULL"); + + // make compound file visible for SegmentReaders + directory->renameFile(segName + QLatin1String(".tmp"), + segName + QLatin1String(".cfs")); + // delete now unused files of segment + writer->deleteFiles(filesToDelete); +} + +CL_NS_END diff --git a/src/assistant/3rdparty/clucene/src/CLucene/index/IndexWriter.h b/src/assistant/3rdparty/clucene/src/CLucene/index/IndexWriter.h new file mode 100644 index 000000000..80476c864 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/index/IndexWriter.h @@ -0,0 +1,425 @@ +/* + * Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team + * + * Distributable under the terms of either the Apache License (Version 2.0) or + * the GNU Lesser General Public License, as specified in the COPYING file. + * + * Changes are Copyright(C) 2007, 2008 by Nokia Corporation and/or its subsidiary(-ies), all rights reserved. +*/ +#ifndef _lucene_index_IndexWriter_ +#define _lucene_index_IndexWriter_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include <QtCore/QString> +#include <QtCore/QStringList> + +#include "CLucene/analysis/AnalysisHeader.h" +#include "CLucene/util/VoidList.h" +#include "CLucene/search/Similarity.h" +#include "CLucene/store/Lock.h" +#include "CLucene/store/TransactionalRAMDirectory.h" + +#include "SegmentHeader.h" + +CL_NS_DEF(index) + +/** +An IndexWriter creates and maintains an index. + +The third argument to the +<a href="#IndexWriter(org.apache.lucene.store.Directory, org.apache.lucene.analysis.Analyzer, boolean)"><b>constructor</b></a> +determines whether a new index is created, or whether an existing index is +opened for the addition of new documents. + +In either case, documents are added with the <a +href="#addDocument(org.apache.lucene.document.Document)"><b>addDocument</b></a> method. +When finished adding documents, <a href="#close()"><b>close</b></a> should be called. + +<p>If an index will not have more documents added for a while and optimal search +performance is desired, then the <a href="#optimize()"><b>optimize</b></a> +method should be called before the index is closed. + +<p>Opening an IndexWriter creates a lock file for the directory in use. Trying to open +another IndexWriter on the same directory will lead to an IOException. The IOException +is also thrown if an IndexReader on the same directory is used to delete documents +from the index. + +@see IndexModifier IndexModifier supports the important methods of IndexWriter plus deletion +*/ +class IndexWriter : LUCENE_BASE +{ + class LockWith2 : public CL_NS(store)::LuceneLockWith<void> + { + public: + LockWith2(CL_NS(store)::LuceneLock* lock, + int64_t lockWaitTimeout, + IndexWriter* wr, + CL_NS(util)::CLVector<SegmentReader*>* std, + bool create); + + ~LockWith2() {} + + void doBody(); + + private: + bool create; + IndexWriter* writer; + CL_NS(util)::CLVector<SegmentReader*>* segmentsToDelete; + }; + friend class LockWith2; + + class LockWithCFS : public CL_NS(store)::LuceneLockWith<void> + { + public: + LockWithCFS(CL_NS(store)::LuceneLock* lock, + int64_t lockWaitTimeout, + CL_NS(store)::Directory* dir, + IndexWriter* wr, + const QString& segName, + const QStringList& ftd); + + ~LockWithCFS() {} + + void doBody(); + + private: + QString segName; + IndexWriter* writer; + CL_NS(store)::Directory* directory; + QStringList filesToDelete; + }; + friend class IndexWriter::LockWithCFS; + + // indicates if the writers is open - this way close can be called multiple + // times + bool isOpen; + + // how to analyze text + CL_NS(analysis)::Analyzer* analyzer; + + CL_NS(search)::Similarity* similarity; // how to normalize + + /** Use compound file setting. Normally defaults to true, except when + * using a RAMDirectory. This minimizes the number of files used. + * Setting this to false may improve indexing performance, but + * may also cause file handle problems. + */ + bool useCompoundFile; + bool closeDir; + + // for temp segs + CL_NS(store)::TransactionalRAMDirectory* ramDirectory; + + CL_NS(store)::LuceneLock* writeLock; + + void _IndexWriter(const bool create); + + void _finalize(); + + // where this index resides + CL_NS(store)::Directory* directory; + + + int32_t getSegmentsCounter() { return segmentInfos.counter; } + int32_t maxFieldLength; + int32_t mergeFactor; + int32_t minMergeDocs; + int32_t maxMergeDocs; + int32_t termIndexInterval; + + int64_t writeLockTimeout; + int64_t commitLockTimeout; +public: + DEFINE_MUTEX(THIS_LOCK) + + // Release the write lock, if needed. + SegmentInfos segmentInfos; + + // Release the write lock, if needed. + ~IndexWriter(); + + /** + * The Java implementation of Lucene silently truncates any tokenized + * field if the number of tokens exceeds a certain threshold. Although + * that threshold is adjustable, it is easy for the client programmer + * to be unaware that such a threshold exists, and to become its + * unwitting victim. + * CLucene implements a less insidious truncation policy. Up to + * DEFAULT_MAX_FIELD_LENGTH tokens, CLucene behaves just as JLucene + * does. If the number of tokens exceeds that threshold without any + * indication of a truncation preference by the client programmer, + * CLucene raises an exception, prompting the client programmer to + * explicitly set a truncation policy by adjusting maxFieldLength. + */ + LUCENE_STATIC_CONSTANT(int32_t, DEFAULT_MAX_FIELD_LENGTH = 10000); + LUCENE_STATIC_CONSTANT(int32_t, FIELD_TRUNC_POLICY__WARN = -1); + int32_t getMaxFieldLength() const{ return maxFieldLength; } + void setMaxFieldLength(int32_t val){ maxFieldLength = val; } + + /** + * Default value is 10. Change using {@link #setMaxBufferedDocs(int)}. + */ + LUCENE_STATIC_CONSTANT(int32_t, DEFAULT_MAX_BUFFERED_DOCS = 10); + /** Determines the minimal number of documents required before the buffered + * in-memory documents are merging and a new Segment is created. + * Since Documents are merged in a {@link RAMDirectory}, + * large value gives faster indexing. At the same time, mergeFactor limits + * the number of files open in a FSDirectory. + * + * <p> The default value is DEFAULT_MAX_BUFFERED_DOCS.*/ + void setMaxBufferedDocs(int32_t val){ minMergeDocs = val; } + /** + * @see #setMaxBufferedDocs + */ + int32_t getMaxBufferedDocs(){ return minMergeDocs; } + + /** + * Default value for the write lock timeout (1,000). + */ + LUCENE_STATIC_CONSTANT(int64_t, WRITE_LOCK_TIMEOUT = 1000); + /** + * Sets the maximum time to wait for a write lock (in milliseconds). + */ + void setWriteLockTimeout(int64_t writeLockTimeout) + { this->writeLockTimeout = writeLockTimeout; } + /** + * @see #setWriteLockTimeout + */ + int64_t getWriteLockTimeout() { return writeLockTimeout; } + + /** + * Default value for the commit lock timeout (10,000). + */ + LUCENE_STATIC_CONSTANT(int64_t, COMMIT_LOCK_TIMEOUT = 10000); + /** + * Sets the maximum time to wait for a commit lock (in milliseconds). + */ + void setCommitLockTimeout(int64_t commitLockTimeout) + { this->commitLockTimeout = commitLockTimeout; } + /** + * @see #setCommitLockTimeout + */ + int64_t getCommitLockTimeout() { return commitLockTimeout; } + + static const QLatin1String WRITE_LOCK_NAME; //"write.lock"; + static const QLatin1String COMMIT_LOCK_NAME; //"commit.lock"; + + /** + * Default value is 10. Change using {@link #setMergeFactor(int)}. + */ + LUCENE_STATIC_CONSTANT(int32_t, DEFAULT_MERGE_FACTOR = 10); + /* Determines how often segment indices are merged by addDocument(). With + * smaller values, less RAM is used while indexing, and searches on + * unoptimized indices are faster, but indexing speed is slower. With larger + * values more RAM is used while indexing and searches on unoptimized indices + * are slower, but indexing is faster. Thus larger values (> 10) are best + * for batched index creation, and smaller values (< 10) for indices that are + * interactively maintained. + * + * <p>This must never be less than 2. The default value is 10. + */ + int32_t getMergeFactor() const{ return mergeFactor; } + void setMergeFactor(int32_t val){ mergeFactor = val; } + + + /** Expert: The fraction of terms in the "dictionary" which should be stored + * in RAM. Smaller values use more memory, but make searching slightly + * faster, while larger values use less memory and make searching slightly + * slower. Searching is typically not dominated by dictionary lookup, so + * tweaking this is rarely useful. + */ + LUCENE_STATIC_CONSTANT(int32_t, DEFAULT_TERM_INDEX_INTERVAL = 128); + /** Expert: Set the interval between indexed terms. Large values cause less + * memory to be used by IndexReader, but slow random-access to terms. Small + * values cause more memory to be used by an IndexReader, and speed + * random-access to terms. + * + * This parameter determines the amount of computation required per query + * term, regardless of the number of documents that contain that term. In + * particular, it is the maximum number of other terms that must be + * scanned before a term is located and its frequency and position information + * may be processed. In a large index with user-entered query terms, query + * processing time is likely to be dominated not by term lookup but rather + * by the processing of frequency and positional data. In a small index + * or when many uncommon query terms are generated (e.g., by wildcard + * queries) term lookup may become a dominant cost. + * + * In particular, <code>numUniqueTerms/interval</code> terms are read into + * memory by an IndexReader, and, on average, <code>interval/2</code> terms + * must be scanned for each random term access. + * + * @see #DEFAULT_TERM_INDEX_INTERVAL + */ + void setTermIndexInterval(int32_t interval) { termIndexInterval = interval; } + /** Expert: Return the interval between indexed terms. + * + * @see #setTermIndexInterval(int) + */ + int32_t getTermIndexInterval() { return termIndexInterval; } + + /** Determines the minimal number of documents required before the buffered + * in-memory documents are merging and a new Segment is created. + * Since Documents are merged in a {@link RAMDirectory}, + * large value gives faster indexing. At the same time, mergeFactor limits + * the number of files open in a FSDirectory. + * + * <p> The default value is 10.*/ + int32_t getMinMergeDocs() const{ return minMergeDocs; } + void setMinMergeDocs(int32_t val){ minMergeDocs = val; } + + /** Determines the largest number of documents ever merged by addDocument(). + * Small values (e.g., less than 10,000) are best for interactive indexing, + * as this limits the length of pauses while indexing to a few seconds. + * Larger values are best for batched indexing and speedier searches. + * + * <p>The default value is {@link #DEFAULT_MAX_MERGE_DOCS}. + */ + LUCENE_STATIC_CONSTANT(int32_t, DEFAULT_MAX_MERGE_DOCS = LUCENE_INT32_MAX_SHOULDBE); + /**Determines the largest number of documents ever merged by addDocument(). + * Small values (e.g., less than 10,000) are best for interactive indexing, + * as this limits the length of pauses while indexing to a few seconds. + * Larger values are best for batched indexing and speedier searches. + * + * <p>The default value is {@link Integer#MAX_VALUE}. + */ + int32_t getMaxMergeDocs() const{ return maxMergeDocs; } + void setMaxMergeDocs(int32_t val){ maxMergeDocs = val; } + + /** + * Constructs an IndexWriter for the index in <code>path</code>. + * Text will be analyzed with <code>a</code>. If <code>create</code> + * is true, then a new, empty index will be created in + * <code>path</code>, replacing the index already there, if any. + * + * @param path the path to the index directory + * @param a the analyzer to use + * @param create <code>true</code> to create the index or overwrite + * the existing one; <code>false</code> to append to the existing + * index + * @throws IOException if the directory cannot be read/written to, or + * if it does not exist, and <code>create</code> is + * <code>false</code> + */ + IndexWriter(const QString& path, CL_NS(analysis)::Analyzer* a, + const bool create, const bool closeDir = true); + + + /**Constructs an IndexWriter for the index in <code>d</code>. Text will be + * analyzed with <code>a</code>. If <code>create</code> is true, then a new, + * empty index will be created in <code>d</code>, replacing the index already + * there, if any. + */ + IndexWriter(CL_NS(store)::Directory* d, CL_NS(analysis)::Analyzer* a, + const bool create, const bool closeDir = false); + + // Flushes all changes to an index, closes all associated files, and closes + // the directory that the index is stored in. + void close(); + + // Returns the number of documents currently in this index. synchronized + int32_t docCount(); + + + // Adds a document to this index, using the provided analyzer instead of + // the value of {@link #getAnalyzer()}. If the document contains more than + // {@link #setMaxFieldLength(int)} terms for a given field, the remainder + // are discarded. + void addDocument(CL_NS(document)::Document* doc, + CL_NS(analysis)::Analyzer* analyzer = NULL); + + + // Merges all segments together into a single segment, optimizing an index + // for search. synchronized + void optimize(); + + + /**Merges all segments from an array of indices into this index. + * + * <p>This may be used to parallelize batch indexing. A large document + * collection can be broken into sub-collections. Each sub-collection can be + * indexed in parallel, on a different thread, process or machine. The + * complete index can then be created by merging sub-collection indices + * with this method. + * + * <p>After this completes, the index is optimized. + *@synchronized + */ + void addIndexes(CL_NS(store)::Directory** dirs); + + /** Merges the provided indexes into this index. + * <p>After this completes, the index is optimized. </p> + * <p>The provided IndexReaders are not closed.</p> + */ + void addIndexes(IndexReader** readers); + + + /** Returns the directory this index resides in. */ + CL_NS(store)::Directory* getDirectory() { return directory; } + + /** Get the current setting of whether to use the compound file format. + * Note that this just returns the value you set with setUseCompoundFile(boolean) + * or the default. You cannot use this to query the status of an existing index. + * @see #setUseCompoundFile(boolean) + */ + bool getUseCompoundFile() { return useCompoundFile; } + + /** Setting to turn on usage of a compound file. When on, multiple files + * for each segment are merged into a single file once the segment creation + * is finished. This is done regardless of what directory is in use. + */ + void setUseCompoundFile(bool value) { useCompoundFile = value; } + + + /** Expert: Set the Similarity implementation used by this IndexWriter. + * + * @see Similarity#setDefault(Similarity) + */ + void setSimilarity(CL_NS(search)::Similarity* similarity) + { this->similarity = similarity; } + + /** Expert: Return the Similarity implementation used by this IndexWriter. + * + * <p>This defaults to the current value of {@link Similarity#getDefault()}. + */ + CL_NS(search)::Similarity* getSimilarity() { return this->similarity; } + + /** Returns the analyzer used by this index. */ + CL_NS(analysis)::Analyzer* getAnalyzer() { return analyzer; } + +private: + /** Merges all RAM-resident segments. */ + void flushRamSegments(); + + /** Incremental segment merger. */ + void maybeMergeSegments(); + + // Pops segments off of segmentInfos stack down to minSegment, merges them, + // and pushes the merged index onto the top of the segmentInfos stack. + void mergeSegments(const uint32_t minSegment); + + // Merges the named range of segments, replacing them in the stack with a + // single segment. + void mergeSegments(const uint32_t minSegment, const uint32_t end); + + // Some operating systems (e.g. Windows) don't permit a file to be deleted + // while it is opened for read (e.g. by another process or thread). So we + // assume that when a delete fails it is because the file is open in another + // process, and queue the file for subsequent deletion. + void deleteSegments(CL_NS(util)::CLVector<SegmentReader*>* segments); + + void deleteFiles(const QStringList& files); + void readDeleteableFiles(QStringList& files); + void deleteFiles(const QStringList& files, QStringList& deletable); + void deleteFiles(const QStringList& files, CL_NS(store)::Directory* directory); + void writeDeleteableFiles(const QStringList& files); + + // synchronized + QString newSegmentName(); +}; + +CL_NS_END + +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/index/MultiReader.cpp b/src/assistant/3rdparty/clucene/src/CLucene/index/MultiReader.cpp new file mode 100644 index 000000000..1260d04dc --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/index/MultiReader.cpp @@ -0,0 +1,722 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#include "CLucene/StdHeader.h" +#include "MultiReader.h" + +#include "IndexReader.h" +#include "CLucene/document/Document.h" +#include "Terms.h" +#include "SegmentMergeQueue.h" + +CL_NS_USE(store) +CL_NS_USE(util) +CL_NS_DEF(index) + +MultiReader::MultiReader(IndexReader** subReaders): + IndexReader(subReaders == NULL || subReaders[0] == NULL ? NULL : subReaders[0]->getDirectory()), + normsCache(true, true) +{ + initialize(subReaders); +} + +MultiReader::MultiReader(Directory* directory, SegmentInfos* sis, IndexReader** subReaders): + IndexReader(directory, sis, false), + normsCache(true, true) +{ + initialize(subReaders); +} + + +MultiReader::~MultiReader() { +//Func - Destructor +//Pre - true +//Post - The instance has been destroyed all IndexReader instances +// this instance managed have been destroyed to + + _CLDELETE_ARRAY(ones); + _CLDELETE_ARRAY(starts); + + //Iterate through the subReaders and destroy each reader + if (subReaders && subReadersLength > 0) { + for (int32_t i = 0; i < subReadersLength; i++) { + _CLDELETE(subReaders[i]); + } + } + //Destroy the subReaders array + _CLDELETE_ARRAY(subReaders); +} + +void MultiReader::initialize(IndexReader** subReaders){ + this->subReadersLength = 0; + this->subReaders = subReaders; + + //count the subReaders size + if ( subReaders != NULL ){ + while ( subReaders[subReadersLength] != NULL ){ + subReadersLength++; + } + } + _maxDoc = 0; + _numDocs = -1; + ones = NULL; + + starts = _CL_NEWARRAY(int32_t,subReadersLength + 1); // build starts array + for (int32_t i = 0; i < subReadersLength; i++) { + starts[i] = _maxDoc; + + // compute maxDocs + _maxDoc += subReaders[i]->maxDoc(); + if (subReaders[i]->hasDeletions()) + _hasDeletions = true; + } + starts[subReadersLength] = _maxDoc; +} + +bool MultiReader::getTermFreqVectors(int32_t n, Array<TermFreqVector*>& result){ + int32_t i = readerIndex(n); // find segment num + return subReaders[i]->getTermFreqVectors(n - starts[i], result); // dispatch to segment +} + +TermFreqVector* MultiReader::getTermFreqVector(int32_t n, const TCHAR* field){ + int32_t i = readerIndex(n); // find segment num + return subReaders[i]->getTermFreqVector(n - starts[i], field); +} + + +int32_t MultiReader::numDocs() { + SCOPED_LOCK_MUTEX(THIS_LOCK) + if (_numDocs == -1) { // check cache + int32_t n = 0; // cache miss--recompute + for (int32_t i = 0; i < subReadersLength; i++) + n += subReaders[i]->numDocs(); // sum from readers + _numDocs = n; + } + return _numDocs; +} + +int32_t MultiReader::maxDoc() const { + return _maxDoc; +} + +bool MultiReader::document(int32_t n, CL_NS(document)::Document* doc){ + int32_t i = readerIndex(n); // find segment num + return subReaders[i]->document(n - starts[i],doc); // dispatch to segment reader +} + +bool MultiReader::isDeleted(const int32_t n) { + int32_t i = readerIndex(n); // find segment num + return subReaders[i]->isDeleted(n - starts[i]); // dispatch to segment reader +} + +uint8_t* MultiReader::norms(const TCHAR* field){ + SCOPED_LOCK_MUTEX(THIS_LOCK) + uint8_t* bytes; + bytes = normsCache.get(field); + if (bytes != NULL){ + return bytes; // cache hit + } + + if ( !hasNorms(field) ) + return fakeNorms(); + + bytes = _CL_NEWARRAY(uint8_t,maxDoc()); + for (int32_t i = 0; i < subReadersLength; i++) + subReaders[i]->norms(field, bytes + starts[i]); + + //Unfortunately the data in the normCache can get corrupted, since it's being loaded with string + //keys that may be deleted while still in use by the map. To prevent this field is duplicated + //and then stored in the normCache + TCHAR* key = STRDUP_TtoT(field); + //update cache + normsCache.put(key, bytes); + + return bytes; +} + +void MultiReader::norms(const TCHAR* field, uint8_t* result) { + SCOPED_LOCK_MUTEX(THIS_LOCK) + uint8_t* bytes = normsCache.get(field); + if (bytes==NULL && !hasNorms(field)) + bytes=fakeNorms(); + + if (bytes != NULL){ // cache hit + int32_t len = maxDoc(); + memcpy(result,bytes,len * sizeof(int32_t)); + } + + for (int32_t i = 0; i < subReadersLength; i++) // read from segments + subReaders[i]->norms(field, result + starts[i]); +} + + +void MultiReader::doSetNorm(int32_t n, const TCHAR* field, uint8_t value){ + normsCache.remove(field); // clear cache + int32_t i = readerIndex(n); // find segment num + subReaders[i]->setNorm(n-starts[i], field, value); // dispatch +} + +TermEnum* MultiReader::terms() const { + return _CLNEW MultiTermEnum(subReaders, starts, NULL); +} + +TermEnum* MultiReader::terms(const Term* term) const { + return _CLNEW MultiTermEnum(subReaders, starts, term); +} + +int32_t MultiReader::docFreq(const Term* t) const { + int32_t total = 0; // sum freqs in Multi + for (int32_t i = 0; i < subReadersLength; i++) + total += subReaders[i]->docFreq(t); + return total; +} + +TermDocs* MultiReader::termDocs() const { + TermDocs* ret = _CLNEW MultiTermDocs(subReaders, starts); + return ret; +} + +TermPositions* MultiReader::termPositions() const { + TermPositions* ret = (TermPositions*)_CLNEW MultiTermPositions(subReaders, starts); + return ret; +} + +void MultiReader::doDelete(const int32_t n) { + _numDocs = -1; // invalidate cache + int32_t i = readerIndex(n); // find segment num + subReaders[i]->deleteDocument(n - starts[i]); // dispatch to segment reader + _hasDeletions = true; +} + +int32_t MultiReader::readerIndex(const int32_t n) const { // find reader for doc n: + int32_t lo = 0; // search starts array + int32_t hi = subReadersLength - 1; // for first element less + // than n, return its index + while (hi >= lo) { + int32_t mid = (lo + hi) >> 1; + int32_t midValue = starts[mid]; + if (n < midValue) + hi = mid - 1; + else if (n > midValue) + lo = mid + 1; + else{ // found a match + while (mid+1 < subReadersLength && starts[mid+1] == midValue) { + mid++; // scan to last match + } + return mid; + } + } + return hi; +} + +bool MultiReader::hasNorms(const TCHAR* field) { + for (int i = 0; i < subReadersLength; i++) { + if (subReaders[i]->hasNorms(field)) + return true; + } + return false; +} +uint8_t* MultiReader::fakeNorms() { + if (ones==NULL) + ones=SegmentReader::createFakeNorms(maxDoc()); + return ones; +} + +void MultiReader::doUndeleteAll(){ + for (int32_t i = 0; i < subReadersLength; i++) + subReaders[i]->undeleteAll(); + _hasDeletions = false; + _numDocs = -1; +} +void MultiReader::doCommit() { + for (int32_t i = 0; i < subReadersLength; i++) + subReaders[i]->commit(); +} + +void MultiReader::doClose() { + SCOPED_LOCK_MUTEX(THIS_LOCK) + for (int32_t i = 0; i < subReadersLength; i++){ + subReaders[i]->close(); + } +} + + +void MultiReader::getFieldNames(FieldOption fldOption, StringArrayWithDeletor& retarray){ + StringArrayWithDeletor temp; + CLHashList<TCHAR*> hashList; + for (int32_t i = 0; i < subReadersLength; i++) { + IndexReader* reader = subReaders[i]; + reader->getFieldNames(fldOption, temp); + + //create a unique list of names. + StringArrayWithDeletor::iterator itr = temp.begin(); + while ( itr != temp.end() ){ + if ( hashList.find(*itr) == hashList.end() ) + hashList.insert(STRDUP_TtoT(*itr)); + itr++; + } + } + //move the items into the return + CLHashList<TCHAR*>::iterator itr = hashList.begin(); + while ( itr != hashList.end() ){ + retarray.push_back(*itr);//no need to copy, already done! + itr++; + } +} + + +MultiTermDocs::MultiTermDocs(){ +//Func - Default constructor +// Initialises an empty MultiTermDocs. +// This constructor is needed to allow the constructor of MultiTermPositions +// initialise the instance by itself +//Pre - true +//Post - An empty + + subReaders = NULL; + subReadersLength = 0; + starts = NULL; + base = 0; + pointer = 0; + current = NULL; + term = NULL; + readerTermDocs = NULL; +} + +MultiTermDocs::MultiTermDocs(IndexReader** r, const int32_t* s){ +//Func - Constructor +//Pre - if r is NULL then rLen must be 0 else if r != NULL then rLen > 0 +// s != NULL +//Post - The instance has been created + + //count readers + subReadersLength = 0; + subReaders = r; + + CND_PRECONDITION(s != NULL, "s is NULL"); + + if ( subReaders != NULL ){ + while ( subReaders[subReadersLength] != NULL ) + subReadersLength++; + } + + starts = s; + base = 0; + pointer = 0; + current = NULL; + term = NULL; + + readerTermDocs = NULL; + + //Check if there are subReaders + if(subReaders != NULL && subReadersLength > 0){ + readerTermDocs = _CL_NEWARRAY(TermDocs*, subReadersLength+1); + + CND_CONDITION(readerTermDocs != NULL,"No memory could be allocated for readerTermDocs"); + + //Initialize the readerTermDocs pointer array to NULLs + for ( int32_t i=0;i<subReadersLength+1;i++){ + readerTermDocs[i]=NULL; + } + } +} + +MultiTermDocs::~MultiTermDocs(){ +//Func - Destructor +//Pre - true +//Post - The instance has been destroyed + + close(); +} + + +TermPositions* MultiTermDocs::__asTermPositions(){ + return NULL; +} + +int32_t MultiTermDocs::doc() const { + CND_PRECONDITION(current!=NULL,"current==NULL, check that next() was called"); + return base + current->doc(); +} +int32_t MultiTermDocs::freq() const { + CND_PRECONDITION(current!=NULL,"current==NULL, check that next() was called"); + return current->freq(); +} + +void MultiTermDocs::seek(TermEnum* termEnum){ + seek(termEnum->term(false)); +} + +void MultiTermDocs::seek( Term* tterm) { +//Func - Resets the instance for a new search +//Pre - tterm != NULL +//Post - The instance has been reset for a new search + + CND_PRECONDITION(tterm != NULL, "tterm is NULL"); + + //Assigning tterm is done as below for a reason + //The construction ensures that if seek is called from within + //MultiTermDocs with as argument this->term (seek(this->term)) that the assignment + //will succeed and all referencecounters represent the correct situation + + //Get a pointer from tterm and increase its reference counter + Term *TempTerm = _CL_POINTER(tterm); + + //Finialize term to ensure we decrease the reference counter of the instance which term points to + _CLDECDELETE(term); + + //Assign TempTerm to term + term = TempTerm; + + base = 0; + pointer = 0; + current = NULL; +} + +bool MultiTermDocs::next() { + if (current != NULL && current->next()) { + return true; + } else if (pointer < subReadersLength) { + base = starts[pointer]; + current = termDocs(pointer++); + return next(); + } else + return false; +} + +int32_t MultiTermDocs::read(int32_t* docs, int32_t* freqs, int32_t length) { + while (true) { + while (current == NULL) { + if (pointer < subReadersLength) { // try next segment + base = starts[pointer]; + current = termDocs(pointer++); + } else { + return 0; + } + } + int32_t end = current->read(docs, freqs,length); + if (end == 0) { // none left in segment + current = NULL; + } else { // got some + int32_t b = base; // adjust doc numbers + for (int32_t i = 0; i < end; i++) + docs[i] += b; + return end; + } + } +} + +bool MultiTermDocs::skipTo(const int32_t target) { + do { + if (!next()) + return false; + } while (target > doc()); + return true; +} + +void MultiTermDocs::close() { +//Func - Closes all MultiTermDocs managed by this instance +//Pre - true +//Post - All the MultiTermDocs have been closed + + + //Check if readerTermDocs is valid + if (readerTermDocs){ + TermDocs* curTD = NULL; + //iterate through the readerTermDocs array + for (int32_t i = 0; i < subReadersLength; i++) { + //Retrieve the i-th TermDocs instance + curTD = readerTermDocs[i]; + + //Check if it is a valid pointer + if (curTD != NULL) { + //Close it + curTD->close(); + _CLDELETE(curTD); + } + } + + _CLDELETE_ARRAY(readerTermDocs); + } + + //current previously pointed to a member of readerTermDocs; ensure that + //it doesn't now point to invalid memory. + current = NULL; + base = 0; + pointer = 0; + + _CLDECDELETE(term); +} + +TermDocs* MultiTermDocs::termDocs(const IndexReader* reader) const { + TermDocs* ret = reader->termDocs(); + return ret; +} + +TermDocs* MultiTermDocs::termDocs(const int32_t i) const { + if (term == NULL) + return NULL; + TermDocs* result = readerTermDocs[i]; + if (result == NULL){ + readerTermDocs[i] = termDocs(subReaders[i]); + result = readerTermDocs[i]; + } + result->seek(term); + + return result; +} + + +MultiTermEnum::MultiTermEnum( + IndexReader** subReaders, const int32_t *starts, const Term* t){ +//Func - Constructor +// Opens all enumerations of all readers +//Pre - readers != NULL and contains an array of IndexReader instances each responsible for +// reading a single segment +// subReadersLength >= 0 and represents the number of readers in the readers array +// starts is an array of +//Post - An instance of has been created + +//Pre - if readers is NULL then subReadersLength must be 0 else if readers != NULL then subReadersLength > 0 +// s != NULL +//Post - The instance has been created + + int32_t subReadersLength = 0; + if ( subReaders != NULL ){ + while ( subReaders[subReadersLength] != NULL ) + subReadersLength++; + } + CND_PRECONDITION(starts != NULL,"starts is NULL"); + + //Temporary variables + IndexReader* reader = NULL; + TermEnum* termEnum = NULL; + SegmentMergeInfo* smi = NULL; + _docFreq = 0; + _term = NULL; + queue = _CLNEW SegmentMergeQueue(subReadersLength); + + CND_CONDITION (queue != NULL, "Could not allocate memory for queue"); + + //iterate through all the readers + for ( int32_t i=0;i<subReadersLength;i++ ) { + //Get the i-th reader + reader = subReaders[i]; + + //Check if the enumeration must start from term t + if (t != NULL) { + //termEnum is an enumeration of terms starting at or after the named term t + termEnum = reader->terms(t); + }else{ + //termEnum is an enumeration of all the Terms and TermInfos in the set. + termEnum = reader->terms(); + } + + //Instantiate an new SegmentMerginfo + smi = _CLNEW SegmentMergeInfo(starts[i], termEnum, reader); + + // Note that in the call termEnum->getTerm(false) below false is required because + // otherwise a reference is leaked. By passing false getTerm is + // ordered to return an unowned reference instead. (Credits for DSR) + if (t == NULL ? smi->next() : termEnum->term(false) != NULL){ + // initialize queue + queue->put(smi); + } else{ + //Close the SegmentMergeInfo + smi->close(); + //And have it deleted + _CLDELETE(smi); + } + } + + //Check if the queue has elements + if (t != NULL && queue->size() > 0) { + next(); + } +} + +MultiTermEnum::~MultiTermEnum(){ +//Func - Destructor +//Pre - true +//Post - All the resource have been freed and the instance has been deleted + + //Close the enumeration + close(); + + //Delete the queue + _CLDELETE(queue); +} + +bool MultiTermEnum::next(){ +//Func - Move the current term to the next in the set of enumerations +//Pre - true +//Post - Returns true if term has been moved to the next in the set of enumerations +// Returns false if this was not possible + + SegmentMergeInfo* top = queue->top(); + if (top == NULL) { + _CLDECDELETE(_term); + _term = NULL; + return false; + } + + //The getTerm method requires the client programmer to indicate whether he + // owns the returned reference, so we can discard ours + // right away. + _CLDECDELETE(_term); + + //Assign term the term of top and make sure the reference counter is increased + _term = _CL_POINTER(top->term); + _docFreq = 0; + + //Find the next term + while (top != NULL && _term->compareTo(top->term) == 0) { + //don't delete, this is the top + queue->pop(); + // increment freq + _docFreq += top->termEnum->docFreq(); + if (top->next()){ + // restore queue + queue->put(top); + }else{ + // done with a segment + top->close(); + _CLDELETE(top); + } + top = queue->top(); + } + + return true; +} + + +Term* MultiTermEnum::term() { +//Func - Returns the current term of the set of enumerations +//Pre - pointer is true or false and indicates if the reference counter +// of term must be increased or not +// next() must have been called once! +//Post - pointer = true -> term has been returned with an increased reference counter +// pointer = false -> term has been returned + + return _CL_POINTER(_term); +} + +Term* MultiTermEnum::term(bool pointer) { + if ( pointer ) + return _CL_POINTER(_term); + else + return _term; +} + +int32_t MultiTermEnum::docFreq() const { +//Func - Returns the document frequency of the current term in the set +//Pre - termInfo != NULL +// next() must have been called once +//Post - The document frequency of the current enumerated term has been returned + + return _docFreq; +} + + +void MultiTermEnum::close() { +//Func - Closes the set of enumerations in the queue +//Pre - queue holds a valid reference to a SegmentMergeQueue +//Post - The queue has been closed all SegmentMergeInfo instance have been deleted by +// the closing of the queue +// term has been finalized and reset to NULL + + // Needed when this enumeration hasn't actually been exhausted yet + _CLDECDELETE(_term); + + //Close the queue This will destroy all SegmentMergeInfo instances! + queue->close(); + +} + + + + + +MultiTermPositions::MultiTermPositions(IndexReader** r, const int32_t* s){ +//Func - Constructor +//Pre - if r is NULL then rLen must be 0 else if r != NULL then rLen > 0 +// s != NULL +//Post - The instance has been created + + subReaders = r; + subReadersLength = 0; + if ( subReaders != NULL ){ + while ( subReaders[subReadersLength] != NULL ) + subReadersLength ++ ; + } + + CND_PRECONDITION(s != NULL, "s is NULL"); + + starts = s; + base = 0; + pointer = 0; + current = NULL; + term = NULL; + + readerTermDocs = NULL; + + //Check if there are readers + if(subReaders != NULL && subReadersLength > 0){ + readerTermDocs = (TermDocs**)_CL_NEWARRAY(SegmentTermPositions*,subReadersLength); + + CND_CONDITION(readerTermDocs != NULL,"No memory could be allocated for readerTermDocs"); + + //Initialize the readerTermDocs pointer array + for ( int32_t i=0;i<subReadersLength;i++){ + readerTermDocs[i]=NULL; + } + } +} + + +TermDocs* MultiTermPositions::__asTermDocs(){ + return (TermDocs*) this; +} +TermPositions* MultiTermPositions::__asTermPositions(){ + return (TermPositions*) this; +} + + +TermDocs* MultiTermPositions::termDocs(const IndexReader* reader) const { +// Here in the MultiTermPositions class, we want this->current to always +// be a SegmentTermPositions rather than merely a SegmentTermDocs. +// To that end, we override the termDocs(IndexReader&) method to produce +// a SegmentTermPositions via the underlying reader's termPositions method +// rather merely producing a SegmentTermDocs via the reader's termDocs +// method. + + TermPositions* tp = reader->termPositions(); + TermDocs* ret = tp->__asTermDocs(); + + CND_CONDITION(ret != NULL, + "Dynamic downcast in MultiTermPositions::termDocs from" + " TermPositions to TermDocs failed." + ); + return ret; + } + +int32_t MultiTermPositions::nextPosition() { + //Func - + //Pre - current != NULL + //Post - + CND_PRECONDITION(current != NULL,"current is NULL"); + + TermPositions* curAsTP = current->__asTermPositions(); + + CND_CONDITION(curAsTP != NULL, + "Dynamic downcast in MultiTermPositions::nextPosition from" + " SegmentTermDocs to TermPositions failed." + ) + return curAsTP->nextPosition(); +} + + +CL_NS_END diff --git a/src/assistant/3rdparty/clucene/src/CLucene/index/MultiReader.h b/src/assistant/3rdparty/clucene/src/CLucene/index/MultiReader.h new file mode 100644 index 000000000..1d76814e1 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/index/MultiReader.h @@ -0,0 +1,202 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _lucene_index_MultiReader +#define _lucene_index_MultiReader + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include "SegmentHeader.h" +#include "IndexReader.h" +#include "CLucene/document/Document.h" +#include "Terms.h" +#include "SegmentMergeQueue.h" + +CL_NS_DEF(index) + +/** An IndexReader which reads multiple indexes, appending their content. +*/ +class MultiTermDocs:public virtual TermDocs { +private: +protected: + TermDocs** readerTermDocs; + + IndexReader** subReaders; + int32_t subReadersLength; + const int32_t* starts; + Term* term; + + int32_t base; + int32_t pointer; + + TermDocs* current; // == segTermDocs[pointer] +public: + MultiTermDocs(); + MultiTermDocs(IndexReader** subReaders, const int32_t* s); + virtual ~MultiTermDocs(); + + int32_t doc() const; + int32_t freq() const; + + void seek(TermEnum* termEnum); + void seek(Term* tterm); + bool next(); + + /** Optimized implementation. */ + int32_t read(int32_t* docs, int32_t* freqs, int32_t length); + + /** As yet unoptimized implementation. */ + bool skipTo(const int32_t target); + + void close(); + + virtual TermPositions* __asTermPositions(); +protected: + virtual TermDocs* termDocs(const IndexReader* reader) const; +private: + TermDocs* termDocs(const int32_t i) const; + +}; + + +//MultiTermEnum represents the enumeration of all terms of all readers +class MultiTermEnum:public TermEnum { +private: + SegmentMergeQueue* queue; + + Term* _term; + int32_t _docFreq; +public: + //Constructor + //Opens all enumerations of all readers + MultiTermEnum(IndexReader** subReaders, const int32_t* starts, const Term* t); + + //Destructor + ~MultiTermEnum(); + + //Move the current term to the next in the set of enumerations + bool next(); + + //Returns a pointer to the current term of the set of enumerations + Term* term(); + Term* term(bool pointer); + + //Returns the document frequency of the current term in the set + int32_t docFreq() const; + + //Closes the set of enumerations in the queue + void close(); + + + const char* getObjectName(){ return MultiTermEnum::getClassName(); } + static const char* getClassName(){ return "MultiTermEnum"; } +}; + + +class MultiTermPositions:public MultiTermDocs,public TermPositions { +public: + MultiTermPositions(IndexReader** subReaders, const int32_t* s); + ~MultiTermPositions() {}; + int32_t nextPosition(); + + + virtual TermDocs* __asTermDocs(); + virtual TermPositions* __asTermPositions(); +protected: + TermDocs* termDocs(const IndexReader* reader) const; +}; + + +class MultiReader:public IndexReader{ +private: + bool _hasDeletions; + IndexReader** subReaders; + int32_t subReadersLength; + int32_t* starts; // 1st docno for each segment + + CL_NS(util)::CLHashtable<const TCHAR*,uint8_t*, + CL_NS(util)::Compare::TChar, + CL_NS(util)::Equals::TChar, + CL_NS(util)::Deletor::tcArray, + CL_NS(util)::Deletor::Array<uint8_t> > normsCache; + int32_t _maxDoc; + int32_t _numDocs; + void initialize(IndexReader** subReaders); + + int32_t readerIndex(const int32_t n) const; + + bool hasNorms(const TCHAR* field); + uint8_t* ones; + uint8_t* fakeNorms(); +protected: + void doSetNorm(int32_t n, const TCHAR* field, uint8_t value); + void doUndeleteAll(); + void doCommit(); + // synchronized + void doClose(); + + // synchronized + void doDelete(const int32_t n); +public: + /** Construct reading the named set of readers. */ + MultiReader(CL_NS(store)::Directory* directory, SegmentInfos* sis, IndexReader** subReaders); + + /** + * <p>Construct a MultiReader aggregating the named set of (sub)readers. + * Directory locking for delete, undeleteAll, and setNorm operations is + * left to the subreaders. </p> + * <p>Note that all subreaders are closed if this Multireader is closed.</p> + * @param subReaders set of (sub)readers + * @throws IOException + */ + MultiReader(IndexReader** subReaders); + + ~MultiReader(); + + /** Return an array of term frequency vectors for the specified document. + * The array contains a vector for each vectorized field in the document. + * Each vector vector contains term numbers and frequencies for all terms + * in a given vectorized field. + * If no such fields existed, the method returns null. + */ + bool getTermFreqVectors(int32_t n, Array<TermFreqVector*>& result); + TermFreqVector* getTermFreqVector(int32_t n, const TCHAR* field); + + + // synchronized + int32_t numDocs(); + + int32_t maxDoc() const; + + bool document(int32_t n, CL_NS(document)::Document* doc); + + bool isDeleted(const int32_t n); + bool hasDeletions() const{ return _hasDeletions; } + + // synchronized + uint8_t* norms(const TCHAR* field); + void norms(const TCHAR* field, uint8_t* result); + + TermEnum* terms() const; + TermEnum* terms(const Term* term) const; + + //Returns the document frequency of the current term in the set + int32_t docFreq(const Term* t=NULL) const; + TermDocs* termDocs() const; + TermPositions* termPositions() const; + + + /** + * @see IndexReader#getFieldNames(IndexReader.FieldOption fldOption) + */ + void getFieldNames(FieldOption fldOption, CL_NS(util)::StringArrayWithDeletor& retarray); +}; + + +CL_NS_END +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/index/SegmentHeader.h b/src/assistant/3rdparty/clucene/src/CLucene/index/SegmentHeader.h new file mode 100644 index 000000000..00b08991d --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/index/SegmentHeader.h @@ -0,0 +1,314 @@ +/* +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +* +* Changes are Copyright(C) 2007, 2008 by Nokia Corporation and/or its subsidiary(-ies), all rights reserved. +*/ +#ifndef _lucene_index_SegmentHeader_ +#define _lucene_index_SegmentHeader_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include <QtCore/QString> +#include <QtCore/QStringList> + +#include "SegmentInfos.h" +#include "CLucene/util/BitSet.h" +#include "CLucene/util/VoidMap.h" +#include "Term.h" +#include "FieldInfos.h" +#include "FieldsReader.h" +#include "IndexReader.h" +#include "TermInfosReader.h" +#include "CompoundFile.h" +#include "CLucene/util/ThreadLocal.h" + +CL_NS_DEF(index) + +class SegmentReader; + +class SegmentTermDocs : public virtual TermDocs +{ + int32_t _doc; + int32_t skipInterval; + int32_t numSkips; + int32_t skipCount; + CL_NS(store)::IndexInput* skipStream; + int32_t skipDoc; + int64_t freqPointer; + int64_t proxPointer; + int64_t skipPointer; + bool haveSkipped; + +protected: + // SegmentReader parent + const SegmentReader* parent; + CL_NS(store)::IndexInput* freqStream; + int32_t count; + int32_t df; + int32_t _freq; + CL_NS(util)::BitSet* deletedDocs; +public: + virtual ~SegmentTermDocs(); + + virtual void seek(TermEnum* termEnum); + virtual void seek(Term* term); + virtual void seek(const TermInfo* ti); + + virtual void close(); + virtual int32_t doc()const; + virtual int32_t freq()const; + + virtual bool next(); + + /** Optimized implementation. */ + virtual int32_t read(int32_t* docs, int32_t* freqs, int32_t length); + + /** Optimized implementation. */ + virtual bool skipTo(const int32_t target); + + virtual TermPositions* __asTermPositions(); + + // \param Parent must be a segment reader + SegmentTermDocs( const SegmentReader* Parent); +protected: + virtual void skippingDoc(){} + virtual void skipProx(int64_t proxPointer){} +}; + + +class SegmentTermPositions : public SegmentTermDocs, public TermPositions +{ +private: + CL_NS(store)::IndexInput* proxStream; + int32_t proxCount; + int32_t position; + +public: + // \param Parent must be a segment reader + SegmentTermPositions(const SegmentReader* Parent); + ~SegmentTermPositions(); + + void seek(const TermInfo* ti); + void close(); + int32_t nextPosition(); + bool next(); + int32_t read(int32_t* docs, int32_t* freqs, int32_t length); + virtual TermDocs* __asTermDocs(); + virtual TermPositions* __asTermPositions(); + + //resolve SegmentTermDocs/TermPositions ambiguity + void seek(Term* term){ SegmentTermDocs::seek(term); } + void seek(TermEnum* termEnum){ SegmentTermDocs::seek(termEnum); } + int32_t doc() const{ return SegmentTermDocs::doc(); } + int32_t freq() const{ return SegmentTermDocs::freq(); } + bool skipTo(const int32_t target){ return SegmentTermDocs::skipTo(target); } + +protected: + void skippingDoc(); + /** Called by super.skipTo(). */ + void skipProx(int64_t proxPointer); +}; + +// An IndexReader responsible for reading 1 segment of an index +class SegmentReader : public IndexReader +{ + /** + * The class Norm represents the normalizations for a field. + * These normalizations are read from an IndexInput in into an array of bytes called bytes + */ + class Norm : LUCENE_BASE + { + int32_t number; + SegmentReader* reader; + QString segment; // segment name + + public: + CL_NS(store)::IndexInput* in; + uint8_t* bytes; + bool dirty; + //Constructor + Norm(CL_NS(store)::IndexInput* instrm, int32_t number, + SegmentReader* reader, const QString& segment); + //Destructor + ~Norm(); + + void reWrite(); + }; + friend class SegmentReader::Norm; + + //Holds the name of the segment that is being read + QString segment; + + //Indicates if there are documents marked as deleted + bool deletedDocsDirty; + bool normsDirty; + bool undeleteAll; + + //Holds all norms for all fields in the segment + typedef CL_NS(util)::CLHashtable<const TCHAR*, + Norm*,CL_NS(util)::Compare::TChar, CL_NS(util)::Equals::TChar> NormsType; + NormsType _norms; + + uint8_t* ones; + uint8_t* fakeNorms(); + + // Compound File Reader when based on a compound file segment + CompoundFileReader* cfsReader; + // Reads the Field Info file + FieldsReader* fieldsReader; + TermVectorsReader* termVectorsReaderOrig; + CL_NS(util)::ThreadLocal<TermVectorsReader*, + CL_NS(util)::Deletor::Object<TermVectorsReader> >termVectorsLocal; + + void initialize(SegmentInfo* si); + + // Create a clone from the initial TermVectorsReader and store it in the + // ThreadLocal. @return TermVectorsReader + TermVectorsReader* getTermVectorsReader(); + +protected: + // Marks document docNum as deleted + void doDelete(const int32_t docNum); + void doUndeleteAll(); + void doCommit(); + void doSetNorm(int32_t doc, const TCHAR* field, uint8_t value); + + // can return null if norms aren't stored + uint8_t* getNorms(const TCHAR* field); + +public: + /** + Func - Constructor. + Opens all files of a segment + .fnm -> Field Info File + Field names are stored in the field info file, with suffix .fnm. + .frq -> Frequency File + The .frq file contains the lists of documents which contain + each term, along with the frequency of the term in that document. + .prx -> Prox File + The prox file contains the lists of positions that each term occurs + at within documents. + .tis -> Term Info File + This file is sorted by Term. Terms are ordered first lexicographically + by the term's field name, and within that lexicographically by the term's text. + .del -> Deletion File + The .del file is optional, and only exists when a segment contains deletions + .f[0-9]* -> Norm File + Contains s, for each document, a byte that encodes a value that is + multiplied into the score for hits on that field: + */ + SegmentReader(SegmentInfo* si); + + SegmentReader(SegmentInfos* sis, SegmentInfo* si); + // Destructor. + virtual ~SegmentReader(); + + // Closes all streams to the files of a single segment + void doClose(); + + // Checks if a segment managed by SegmentInfo si has deletions + static bool hasDeletions(const SegmentInfo* si); + bool hasDeletions() const; + bool hasNorms(const TCHAR* field) const; + + // Returns all file names managed by this SegmentReader + void files(QStringList& retarray); + // Returns an enumeration of all the Terms and TermInfos in the set. + TermEnum* terms() const; + // Returns an enumeration of terms starting at or after the named term t + TermEnum* terms(const Term* t) const; + + // Gets the document identified by n + bool document(int32_t n, CL_NS(document)::Document* doc); + + // Checks if the n-th document has been marked deleted + bool isDeleted(const int32_t n); + + // Returns an unpositioned TermDocs enumerator. + TermDocs* termDocs() const; + // Returns an unpositioned TermPositions enumerator. + TermPositions* termPositions() const; + + // Returns the number of documents which contain the term t + int32_t docFreq(const Term* t) const; + + // Returns the actual number of documents in the segment + int32_t numDocs(); + // Returns the number of all the documents in the segment including the + // ones that have been marked deleted + int32_t maxDoc() const; + + // Returns the bytes array that holds the norms of a named field. + // Returns fake norms if norms aren't available + uint8_t* norms(const TCHAR* field); + + // Reads the Norms for field from disk + void norms(const TCHAR* field, uint8_t* bytes); + + // concatenating segment with ext and x + QString SegmentName(const QString& ext, const int32_t x = -1); + // Creates a filename in buffer by concatenating segment with ext and x + void SegmentName(QString& buffer, int32_t bufferLen, const QString& ext, + const int32_t x = -1); + + /** + * @see IndexReader#getFieldNames(IndexReader.FieldOption fldOption) + */ + void getFieldNames(FieldOption fldOption, CL_NS(util)::StringArrayWithDeletor& retarray); + + static bool usesCompoundFile(SegmentInfo* si); + + /** Return a term frequency vector for the specified document and field. The + * vector returned contains term numbers and frequencies for all terms in + * the specified field of this document, if the field had storeTermVector + * flag set. If the flag was not set, the method returns null. + * @throws IOException + */ + TermFreqVector* getTermFreqVector(int32_t docNumber, const TCHAR* field = NULL); + + /** Return an array of term frequency vectors for the specified document. + * The array contains a vector for each vectorized field in the document. + * Each vector vector contains term numbers and frequencies for all terms + * in a given vectorized field. + * If no such fields existed, the method returns null. + * @throws IOException + */ + bool getTermFreqVectors(int32_t docNumber, Array<TermFreqVector*>& result); + +private: + //Open all norms files for all fields + void openNorms(CL_NS(store)::Directory* cfsDir); + //Closes all norms files + void closeNorms(); + + // a bitVector that manages which documents have been deleted + CL_NS(util)::BitSet* deletedDocs; + // an IndexInput to the frequency file + CL_NS(store)::IndexInput* freqStream; + // For reading the fieldInfos file + FieldInfos* fieldInfos; + // For reading the Term Dictionary .tis file + TermInfosReader* tis; + // an IndexInput to the prox file + CL_NS(store)::IndexInput* proxStream; + + static bool hasSeparateNorms(SegmentInfo* si); + static uint8_t* createFakeNorms(int32_t size); + + // allow various classes to access the internals of this. this allows us + // to have a more tight idea of the package + friend class IndexReader; + friend class IndexWriter; + friend class SegmentTermDocs; + friend class SegmentTermPositions; + friend class MultiReader; +}; + +CL_NS_END + +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/index/SegmentInfos.cpp b/src/assistant/3rdparty/clucene/src/CLucene/index/SegmentInfos.cpp new file mode 100644 index 000000000..f62c4061a --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/index/SegmentInfos.cpp @@ -0,0 +1,259 @@ +/* + * Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team + * + * Distributable under the terms of either the Apache License (Version 2.0) or + * the GNU Lesser General Public License, as specified in the COPYING file. + * + * Changes are Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies). +*/ +#include "CLucene/StdHeader.h" +#include "SegmentInfos.h" + +#include "CLucene/store/Directory.h" +#include "CLucene/util/Misc.h" + +CL_NS_USE(store) +CL_NS_USE(util) +CL_NS_DEF(index) + +SegmentInfo::SegmentInfo(const QString& Name, const int32_t DocCount, + CL_NS(store)::Directory* Dir) + : docCount(DocCount) + , dir(Dir) +{ + //Func - Constructor. Initialises SegmentInfo. + //Pre - Name holds the unique name in the directory Dir + // DocCount holds the number of documents in the segment + // Dir holds the Directory where the segment resides + //Post - The instance has been created. name contains the duplicated string + // Name. docCount = DocCount and dir references Dir + name = Name; +} + +SegmentInfo::~SegmentInfo() +{ +} + +SegmentInfos::SegmentInfos(bool _deleteMembers) + : deleteMembers(_deleteMembers) +{ + //Func - Constructor + //Pre - deleteMembers indicates if the instance to be created must delete + // all SegmentInfo instances it manages when the instance is destroyed + // or not true -> must delete, false may not delete + //Post - An instance of SegmentInfos has been created. + + //initialize counter to 0 + counter = 0; + version = Misc::currentTimeMillis(); +} + +SegmentInfos::~SegmentInfos() +{ + //Func - Destructor + //Pre - true + //Post - The instance has been destroyed. Depending on the constructor used + // the SegmentInfo instances that this instance managed have been + // deleted or not. + + if (deleteMembers) { + segmentInfosType::iterator it; + for (it = infos.begin(); it != infos.end(); ++it) + _CLLDELETE(*it); + } + //Clear the list of SegmentInfo instances - make sure everything is deleted + infos.clear(); +} + +SegmentInfo* SegmentInfos::info(int32_t i) const +{ + //Func - Returns a reference to the i-th SegmentInfo in the list. + //Pre - i >= 0 + //Post - A reference to the i-th SegmentInfo instance has been returned + + CND_PRECONDITION(i >= 0, "i contains negative number"); + + //Get the i-th SegmentInfo instance + SegmentInfo *ret = infos.value(i, 0); + + //Condition check to see if the i-th SegmentInfo has been retrieved + CND_CONDITION(ret != NULL, "No SegmentInfo instance found"); + + return ret; +} + +void SegmentInfos::clearto(size_t _min) +{ + // Make sure we actually need to remove + if (infos.size() > _min) { + segmentInfosType::iterator itr; + segmentInfosType::iterator eitr = infos.end(); + segmentInfosType::iterator bitr = infos.begin() + _min; + + for(itr = bitr; itr != eitr; ++itr) + _CLLDELETE((*itr)); + infos.erase(bitr, eitr); + } +} + +void SegmentInfos::add(SegmentInfo* info) +{ + infos.push_back(info); +} + +int32_t SegmentInfos::size() const +{ + return infos.size(); +} + +void SegmentInfos::read(Directory* directory) +{ + //Func - Reads segments file that resides in directory. + //Pre - directory contains a valid reference + //Post - The segments file has been read and for each segment found + // a SegmentsInfo intance has been created and stored. + + //Open an IndexInput to the segments file and check if valid + IndexInput* input = directory->openInput(QLatin1String("segments")); + if (input) { + try { + int32_t format = input->readInt(); + // file contains explicit format info + if (format < 0) { + // check that it is a format we can understand + if (format < FORMAT) { + TCHAR err[30]; + _sntprintf(err, 30, _T("Unknown format version: %d"), format); + _CLTHROWT(CL_ERR_Runtime, err); + } + // read version + version = input->readLong(); + // read counter + counter = input->readInt(); + } else { + // file is in old format without explicit format info + counter = format; + } + + //Temporary variable for storing the name of the segment + char aname[CL_MAX_PATH] = { 0 }; + TCHAR tname[CL_MAX_PATH] = { 0 }; + + //read segmentInfos + for (int32_t i = input->readInt(); i > 0; --i) { + // read the name of the segment + input->readString(tname, CL_MAX_PATH); + STRCPY_TtoA(aname, tname, CL_MAX_PATH); + + //Instantiate a new SegmentInfo Instance + SegmentInfo* si = _CLNEW SegmentInfo(QLatin1String(aname), + input->readInt(), directory); + + //Condition check to see if si points to an instance + CND_CONDITION(si != NULL, "Memory allocation for si failed") ; + + //store SegmentInfo si + infos.push_back(si); + } + + if (format >= 0) { + // in old format the version number may be at the end of the file + if (input->getFilePointer() >= input->length()) { + // old file format without version number + version = Misc::currentTimeMillis(); + } else { + // read version + version = input->readLong(); + } + } + } _CLFINALLY ( + //destroy the inputStream input. The destructor of IndexInput will + //also close the Inputstream input + _CLDELETE(input); + ); + } +} + +void SegmentInfos::write(Directory* directory) +{ + //Func - Writes a new segments file based upon the SegmentInfo instances it manages + //Pre - directory is a valid reference to a Directory + //Post - The new segment has been written to disk + + //Open an IndexOutput to the segments file and check if valid + IndexOutput* output = directory->createOutput(QLatin1String("segments.new")); + if (output) { + try { + // write FORMAT + output->writeInt(FORMAT); + // every write changes the index + output->writeLong(++version); + // Write the counter + output->writeInt(counter); + + // Write the number of SegmentInfo Instances which is equal to the number + // of segments in directory as each SegmentInfo manages a single segment + output->writeInt(infos.size()); + + //temporary value for wide segment name + TCHAR tname[CL_MAX_PATH]; + + //Iterate through all the SegmentInfo instances + for (uint32_t i = 0; i < infos.size(); ++i) { + //Retrieve the SegmentInfo + SegmentInfo *si = infos.value(i, 0); + //Condition check to see if si has been retrieved + CND_CONDITION(si != NULL, "No SegmentInfo instance found"); + + //Write the name of the current segment + int32_t count = si->name.toWCharArray(tname); + tname[count] = '\0'; + output->writeString(tname, _tcslen(tname)); + + //Write the number of documents in the segment + output->writeInt(si->docCount); + } + } _CLFINALLY( + output->close(); + _CLDELETE(output); + ); + + // install new segment info + directory->renameFile(QLatin1String("segments.new"), + QLatin1String("segments")); + } +} + + +int64_t SegmentInfos::readCurrentVersion(Directory* directory) +{ + int32_t format = 0; + int64_t version = 0; + IndexInput* input = directory->openInput(QLatin1String("segments")); + try { + format = input->readInt(); + if (format < 0){ + if (format < FORMAT) { + TCHAR err[30]; + _sntprintf(err, 30, _T("Unknown format version: %d"), format); + _CLTHROWT(CL_ERR_Runtime, err); + } + // read version + version = input->readLong(); + } + } _CLFINALLY ( + input->close(); + _CLDELETE(input); + ); + + if (format < 0) + return version; + + // We cannot be sure about the format of the file. Therefore we have to + // read the whole file and cannot simply seek to the version entry. + SegmentInfos segmentInfos; + segmentInfos.read(directory); + return segmentInfos.getVersion(); +} + +CL_NS_END diff --git a/src/assistant/3rdparty/clucene/src/CLucene/index/SegmentInfos.h b/src/assistant/3rdparty/clucene/src/CLucene/index/SegmentInfos.h new file mode 100644 index 000000000..ce7183820 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/index/SegmentInfos.h @@ -0,0 +1,128 @@ +/* + * Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team + * + * Distributable under the terms of either the Apache License (Version 2.0) or + * the GNU Lesser General Public License, as specified in the COPYING file. + * + * Changes are Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies). +*/ +#ifndef _lucene_index_SegmentInfos_ +#define _lucene_index_SegmentInfos_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include <QtCore/QString> +#include <QtCore/QVector> + +#include "CLucene/store/Directory.h" + +CL_NS_DEF(index) + +class SegmentInfo : LUCENE_BASE +{ +public: + SegmentInfo(const QString& Name, const int32_t DocCount, + CL_NS(store)::Directory* Dir); + ~SegmentInfo(); + + ///Gets the Directory where the segment resides + CL_NS(store)::Directory* getDir() const { return dir; } + + //Unique name in directory dir + QString name; + + //Number of docs in the segment + const int32_t docCount; + +private: + //Directory where the segment resides + CL_NS(store)::Directory* dir; +}; + +typedef QVector<SegmentInfo*> segmentInfosType; + +//SegmentInfos manages a list of SegmentInfo instances +//Each SegmentInfo contains information about a segment in a directory. +// +//The active segments in the index are stored in the segment info file. +//An index only has a single file in this format, and it is named "segments". +//This lists each segment by name, and also contains the size of each segment. +//The format of the file segments is defined as follows: +// +// SegCount +//Segments --> SegCount, <SegName, SegSize> +// +//SegCount, SegSize --> UInt32 +// +//SegName --> String +// +//SegName is the name of the segment, and is used as the file name prefix +//for all of the files that compose the segment's index. +// +//SegSize is the number of documents contained in the segment index. +// +//Note: +//At http://jakarta.apache.org/lucene/docs/fileformats.html the definition +//of all file formats can be found. Note that java lucene currently +//defines Segments as follows: +// +//Segments --> Format, Version, SegCount, <SegName, SegSize>SegCount +// +//Format, SegCount, SegSize --> UInt32 +// +//Format and Version have not been implemented yet +class SegmentInfos : LUCENE_BASE +{ +public: + SegmentInfos(bool deleteMembers = true); + ~SegmentInfos(); + + //delete and clears objects 'from' from to 'to' + void clearto(size_t to); + + //count of segment infos + int32_t size() const; + + //add a segment info + void add(SegmentInfo* info); + + //Returns a reference to the i-th SegmentInfo in the list. + SegmentInfo* info(int32_t i) const; + + // version number when this SegmentInfos was generated. + int64_t getVersion() const { return version; } + + static int64_t readCurrentVersion(CL_NS(store)::Directory* directory); + + //Reads segments file that resides in directory + void read(CL_NS(store)::Directory* directory); + + //Writes a new segments file based upon the SegmentInfo instances it manages + void write(CL_NS(store)::Directory* directory); + +private: + // The file format version, a negative number. + // Works since counter, the old 1st entry, is always >= 0 + LUCENE_STATIC_CONSTANT(int32_t, FORMAT = -1); + + // counts how often the index has been changed by adding or deleting docs. + // starting with the current time in milliseconds forces to create unique + // version numbers. + int64_t version; + + segmentInfosType infos; + + // used to name new segments + int32_t counter; + + // allow IndexWriter to use counter + friend class IndexWriter; + + bool deleteMembers; +}; + +CL_NS_END + +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/index/SegmentMergeInfo.cpp b/src/assistant/3rdparty/clucene/src/CLucene/index/SegmentMergeInfo.cpp new file mode 100644 index 000000000..85ac784ad --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/index/SegmentMergeInfo.cpp @@ -0,0 +1,104 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#include "CLucene/StdHeader.h" +#include "SegmentMergeInfo.h" + +#include "SegmentTermEnum.h" +#include "SegmentHeader.h" + +CL_NS_DEF(index) + +SegmentMergeInfo::SegmentMergeInfo(const int32_t b, TermEnum* te, IndexReader* r): + reader(r),termEnum(te),base(b), docMap(NULL) { +//Func - Constructor +//Pre - b >= 0 +// te contains a valid reference to a SegmentTermEnum instance +// r contains a valid reference to a SegmentReader instance +//Post - The instance has been created + + CND_PRECONDITION(b >= 0, "b is a negative number"); + + postings=NULL; + term = te->term(); +} + +SegmentMergeInfo::~SegmentMergeInfo(){ +//Func - Destructor +//Pre - true +//Post - The instance has been destroyed + + close(); +} + +int32_t* SegmentMergeInfo::getDocMap(){ + if ( docMap == NULL ){ + // build array which maps document numbers around deletions + if (reader->hasDeletions()) { + //Get the total number of documents managed by the reader including the deleted ones + int32_t maxDoc = reader->maxDoc(); + //Create a map for all documents + docMap = _CL_NEWARRAY(int32_t,maxDoc); + int32_t j = 0; + //Iterate through all the document numbers + for (int32_t i = 0; i < maxDoc; i++) { + //Check if document i is marked deleted + if (reader->isDeleted(i)){ + //Document i has not been marked deleted so assign -1 + docMap[i] = -1; + }else{ + docMap[i] = j++; + } + } + } + } + return docMap; +} + +TermPositions* SegmentMergeInfo::getPositions() { + if (postings == NULL) { + postings = reader->termPositions(); + } + return postings; +} + + +bool SegmentMergeInfo::next() { +//Func - Moves the current term of the enumeration termEnum to the next and term +// points to this new current term +//Pre - true +//Post - Returns true if the term has been moved to the next otherwise false + if (termEnum->next()) { + _CLDECDELETE(term); + term = termEnum->term(); + return true; + } else { + _CLDECDELETE(term); //TODO: test HighFreqTerms errors with this + term = NULL; + return false; + } +} + +void SegmentMergeInfo::close() { +//Func - Closes the the resources +//Pre - true +//Post - The resources have been closed + + //First make sure posting has been closed + if ( postings != NULL ){ + postings->close(); + _CLVDELETE(postings); //todo: not a clucene object... should be + } + + if ( termEnum != NULL ){ + termEnum->close(); + _CLDELETE(termEnum); + } + _CLDECDELETE(term); + _CLDELETE_ARRAY(docMap); +} + +CL_NS_END diff --git a/src/assistant/3rdparty/clucene/src/CLucene/index/SegmentMergeInfo.h b/src/assistant/3rdparty/clucene/src/CLucene/index/SegmentMergeInfo.h new file mode 100644 index 000000000..7ffd46ade --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/index/SegmentMergeInfo.h @@ -0,0 +1,47 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _lucene_index_SegmentMergeInfo_ +#define _lucene_index_SegmentMergeInfo_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include "SegmentTermEnum.h" +#include "SegmentHeader.h" + +CL_NS_DEF(index) +class SegmentMergeInfo:LUCENE_BASE { + int32_t* docMap; // maps around deleted docs + TermPositions* postings; +public: + TermEnum* termEnum; + Term* term; + int32_t base; + IndexReader* reader; + + //Constructor + SegmentMergeInfo(const int32_t b, TermEnum* te, IndexReader* r); + + //Destructor + ~SegmentMergeInfo(); + + //Moves the current term of the enumeration termEnum to the next and term + //points to this new current term + bool next(); + + //Closes the the resources + void close(); + + // maps around deleted docs + int32_t* getDocMap(); + + TermPositions* getPositions(); +}; +CL_NS_END +#endif + diff --git a/src/assistant/3rdparty/clucene/src/CLucene/index/SegmentMergeQueue.cpp b/src/assistant/3rdparty/clucene/src/CLucene/index/SegmentMergeQueue.cpp new file mode 100644 index 000000000..879781287 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/index/SegmentMergeQueue.cpp @@ -0,0 +1,74 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#include "CLucene/StdHeader.h" +#include "SegmentMergeQueue.h" + +#include "SegmentMergeInfo.h" +CL_NS_DEF(index) + + + SegmentMergeQueue::SegmentMergeQueue(const int32_t size) { + //Func - Constructor + // Creates a queue of length size + //Pre - size >= 0 + //Post - The queue has been created of length size + + //BVK: bug. changed condition from size > 0 to size >= 0 + //if size is 0, as it is when retrieving a TermEnum + //from an empty index this should this should not fail. + CND_PRECONDITION(size >= 0, "size is too small"); + + //Call the initialize method of its superclass. The boolean value passed here + //indicates that the superclass PriorityQueue takes the responsibility to have its elements deleted + //The destructor of SegmentMergInfo will make sure that each intstance it will be closed properly + //before it is deleted + initialize(size,true); + } + + SegmentMergeQueue::~SegmentMergeQueue(){ + //Func - Destructor + // Does nothing as its parent class will clean up everything + //Pre - true + //Post - true + close(); + } + + void SegmentMergeQueue::close() { + //Func - Closes and destroyes all SegmentMergeInfo Instances in the queue + //Pre - true + //post - All SegmentMergeInfo Instances in the queue have been closed and deleted + // The queue is now empty but can still be used + + //call the clear method of the parent class PriorityQueue + clear(); + } + + bool SegmentMergeQueue::lessThan(SegmentMergeInfo* stiA, SegmentMergeInfo* stiB) { + //Func - Overloaded method that implements the lessThan operator for the parent class + // This method is used by the parent class Priority queue to reorder its internal + // data structures. This implementation check if stiA is less than the current term of stiB. + //Pre - stiA != NULL + // stiB != NULL + //Post - true is returned if stiA < stiB otherwise false + + CND_PRECONDITION(stiA != NULL, "stiA is NULL"); + CND_PRECONDITION(stiB != NULL, "stiB is NULL"); + + //Compare the two terms + int32_t comparison = stiA->term->compareTo(stiB->term); + //Check if they match + if (comparison == 0){ //todo: can we do an optimized compare here? compare using equals, then compare properly? + //If the match check if the base of stiA is smaller than the base of stiB + //Note that different bases means that the terms of stiA an stiB ly in different segments + return stiA->base < stiB->base; + }else{ + //Terms didn't match so return the difference in positions + return comparison < 0; + } + } + +CL_NS_END diff --git a/src/assistant/3rdparty/clucene/src/CLucene/index/SegmentMergeQueue.h b/src/assistant/3rdparty/clucene/src/CLucene/index/SegmentMergeQueue.h new file mode 100644 index 000000000..faa690252 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/index/SegmentMergeQueue.h @@ -0,0 +1,38 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _lucene_index_SegmentMergeQueue_ +#define _lucene_index_SegmentMergeQueue_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include "CLucene/util/PriorityQueue.h" +#include "SegmentMergeInfo.h" + +CL_NS_DEF(index) + class SegmentMergeQueue :public CL_NS(util)::PriorityQueue<SegmentMergeInfo*,CL_NS(util)::Deletor::Object<SegmentMergeInfo> > { + public: + //Constructor + //Creates a queue of length size + SegmentMergeQueue(const int32_t size); + + //Destructor + //Does nothing as its parent class will clean up everything + ~SegmentMergeQueue(); + + //Closes and destroyes all SegmentMergeInfo Instances in the queue + void close(); + protected: + //Overloaded method that implements the lessThan operator for the parent class + //This method is used by the parent class Priority queue to reorder its internal + //data structures. This implementation check if stiA is less than the current term of stiB. + bool lessThan(SegmentMergeInfo* stiA, SegmentMergeInfo* stiB); + + }; +CL_NS_END +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/index/SegmentMerger.cpp b/src/assistant/3rdparty/clucene/src/CLucene/index/SegmentMerger.cpp new file mode 100644 index 000000000..40814da0c --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/index/SegmentMerger.cpp @@ -0,0 +1,723 @@ +/* + * Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team + * + * Distributable under the terms of either the Apache License (Version 2.0) or + * the GNU Lesser General Public License, as specified in the COPYING file. + * + * Changes are Copyright(C) 2007, 2008 by Nokia Corporation and/or its subsidiary(-ies), all rights reserved. +*/ +#include "CLucene/StdHeader.h" +#include "SegmentMerger.h" + +CL_NS_USE(util) +CL_NS_USE(document) +CL_NS_USE(store) +CL_NS_DEF(index) + +// File extensions of old-style index files +int COMPOUND_EXTENSIONS_LENGTH = 7; +const char* COMPOUND_EXTENSIONS = "fnm\0" "frq\0" "prx\0" "fdx\0" "fdt\0" "tii\0" "tis\0"; + +int VECTOR_EXTENSIONS_LENGTH = 3; +const char* VECTOR_EXTENSIONS = "tvx\0" "tvd\0" "tvf\0"; + +SegmentMerger::SegmentMerger(IndexWriter* writer, const QString& name) +{ + //Func - Constructor + //Pre - dir holds a valid reference to a Directory + // name != NULL + //Post - Instance has been created + + CND_PRECONDITION(!name.isEmpty(), "name is NULL"); + + freqOutput = NULL; + proxOutput = NULL; + termInfosWriter = NULL; + queue = NULL; + fieldInfos = NULL; + useCompoundFile = writer->getUseCompoundFile(); + skipBuffer = _CLNEW CL_NS(store)::RAMIndexOutput(); + + segment = name; + directory = writer->getDirectory(); + termIndexInterval = writer->getTermIndexInterval(); + + lastSkipDoc=0; + lastSkipFreqPointer=0; + lastSkipProxPointer=0; + skipInterval=0; +} + +SegmentMerger::~SegmentMerger() +{ + //Func - Destructor + //Pre - true + //Post - The instance has been destroyed + + //Clear the readers set + readers.clear(); + + //Delete field Infos + _CLDELETE(fieldInfos); + //Close and destroy the IndexOutput to the Frequency File + if (freqOutput != NULL) { + freqOutput->close(); + _CLDELETE(freqOutput); + } + //Close and destroy the IndexOutput to the Prox File + if (proxOutput != NULL) { + proxOutput->close(); + _CLDELETE(proxOutput); + } + //Close and destroy the termInfosWriter + if (termInfosWriter != NULL) { + termInfosWriter->close(); + _CLDELETE(termInfosWriter); + } + //Close and destroy the queue + if (queue != NULL) { + queue->close(); + _CLDELETE(queue); + } + //close and destory the skipBuffer + if (skipBuffer != NULL) { + skipBuffer->close(); + _CLDELETE(skipBuffer); + } +} + +void SegmentMerger::add(IndexReader* reader) +{ + //Func - Adds a IndexReader to the set of readers + //Pre - reader contains a valid reference to a IndexReader + //Post - The SegementReader reader has been added to the set of readers + + readers.push_back(reader); +} + +IndexReader* SegmentMerger::segmentReader(const int32_t i) +{ + //Func - Returns a reference to the i-th IndexReader + //Pre - 0 <= i < readers.size() + //Post - A reference to the i-th IndexReader has been returned + + CND_PRECONDITION(i >= 0, "i is a negative number"); + CND_PRECONDITION((size_t)i < readers.size(), + "i is bigger than the number of IndexReader instances"); + + //Retrieve the i-th IndexReader + IndexReader* ret = readers[i]; + CND_CONDITION(ret != NULL, "No IndexReader found"); + + return ret; +} + +int32_t SegmentMerger::merge() +{ + int32_t value = mergeFields(); + mergeTerms(); + mergeNorms(); + + if (fieldInfos->hasVectors()) + mergeVectors(); + + return value; +} + +void SegmentMerger::closeReaders() +{ + for (uint32_t i = 0; i < readers.size(); i++) { + // close readers + IndexReader* reader = readers[i]; + reader->close(); + } +} + +void SegmentMerger::createCompoundFile(const QString& filename, QStringList& files) +{ + CompoundFileWriter* cfsWriter = _CLNEW CompoundFileWriter(directory, filename); + + { //msvc6 scope fix + // Basic files + for (int32_t i = 0; i < COMPOUND_EXTENSIONS_LENGTH; i++) { + files.push_back(Misc::qjoin(segment, QLatin1String("."), + QLatin1String(COMPOUND_EXTENSIONS+(i*4)))); + } + } + + { //msvc6 scope fix + // Field norm files + for (int32_t i = 0; i < fieldInfos->size(); i++) { + FieldInfo* fi = fieldInfos->fieldInfo(i); + if (fi->isIndexed && !fi->omitNorms) { + TCHAR tbuf[10]; + char abuf[10]; + _i64tot(i, tbuf, 10); + STRCPY_TtoA(abuf, tbuf, 10); + + files.push_back(Misc::qjoin(segment, QLatin1String(".f"), + QLatin1String(abuf))); + } + } + } + + // Vector files + if (fieldInfos->hasVectors()) { + for (int32_t i = 0; i < VECTOR_EXTENSIONS_LENGTH; i++) { + files.push_back(Misc::qjoin(segment, QLatin1String("."), + QLatin1String(VECTOR_EXTENSIONS+(i*4)))); + } + } + + { //msvc6 scope fix + // Now merge all added files + for (size_t i=0;i<files.size();i++) { + cfsWriter->addFile(files[i]); + } + } + + // Perform the merge + cfsWriter->close(); + _CLDELETE(cfsWriter); +} + +void SegmentMerger::addIndexed(IndexReader* reader, FieldInfos* fieldInfos, + StringArrayWithDeletor& names, bool storeTermVectors, + bool storePositionWithTermVector, bool storeOffsetWithTermVector) +{ + StringArrayWithDeletor::const_iterator itr = names.begin(); + while (itr != names.end()) { + fieldInfos->add(*itr, true, + storeTermVectors, storePositionWithTermVector, + storeOffsetWithTermVector, !reader->hasNorms(*itr)); + ++itr; + } +} + +int32_t SegmentMerger::mergeFields() +{ + //Func - Merge the fields of all segments + //Pre - true + //Post - The field infos and field values of all segments have been merged. + + //Create a new FieldInfos + fieldInfos = _CLNEW FieldInfos(); // merge field names + + //Condition check to see if fieldInfos points to a valid instance + CND_CONDITION(fieldInfos != NULL, "Memory allocation for fieldInfos failed"); + + IndexReader* reader = NULL; + + int32_t docCount = 0; + + //Iterate through all readers + for (uint32_t i = 0; i < readers.size(); i++) { + //get the i-th reader + reader = readers[i]; + //Condition check to see if reader points to a valid instance + CND_CONDITION(reader != NULL,"No IndexReader found"); + + StringArrayWithDeletor tmp; + + tmp.clear(); + reader->getFieldNames(IndexReader::TERMVECTOR_WITH_POSITION_OFFSET, tmp); + addIndexed(reader, fieldInfos, tmp, true, true, true); + + tmp.clear(); + reader->getFieldNames(IndexReader::TERMVECTOR_WITH_POSITION, tmp); + addIndexed(reader, fieldInfos, tmp, true, true, false); + + tmp.clear(); + reader->getFieldNames(IndexReader::TERMVECTOR_WITH_OFFSET, tmp); + addIndexed(reader, fieldInfos, tmp, true, false, true); + + tmp.clear(); + reader->getFieldNames(IndexReader::TERMVECTOR, tmp); + addIndexed(reader, fieldInfos, tmp, true, false, false); + + tmp.clear(); + reader->getFieldNames(IndexReader::INDEXED, tmp); + addIndexed(reader, fieldInfos, tmp, false, false, false); + + tmp.clear(); + reader->getFieldNames(IndexReader::UNINDEXED, tmp); + if (tmp.size() > 0) { + TCHAR** arr = _CL_NEWARRAY(TCHAR*,tmp.size()+1); + tmp.toArray(arr); + fieldInfos->add((const TCHAR**)arr, false); + _CLDELETE_ARRAY(arr); + //no need to delete the contents, since tmp is responsible for it + } + } + + //Create the filename of the new FieldInfos file + QString buf = Misc::segmentname(segment, QLatin1String(".fnm")); + //Write the new FieldInfos file to the directory + fieldInfos->write(directory, buf); + + // merge field values + // Instantiate Fieldswriter which will write in directory for the segment + // name segment using the new merged fieldInfos + FieldsWriter* fieldsWriter = _CLNEW FieldsWriter(directory, segment, fieldInfos); + + //Condition check to see if fieldsWriter points to a valid instance + CND_CONDITION(fieldsWriter != NULL, "Memory allocation for fieldsWriter failed"); + + try { + IndexReader* reader = NULL; + int32_t maxDoc = 0; + //Iterate through all readers + for (uint32_t i = 0; i < readers.size(); i++) { + // get the i-th reader + reader = readers[i]; + + + // Condition check to see if reader points to a valid instance + CND_CONDITION(reader != NULL, "No IndexReader found"); + + // Get the total number documents including the documents that have + // been marked deleted + int32_t maxDoc = reader->maxDoc(); + + //document buffer + Document doc; + + //Iterate through all the documents managed by the current reader + for (int32_t j = 0; j < maxDoc; j++) { + //Check if the j-th document has been deleted, if so skip it + if (!reader->isDeleted(j)) { + //Get the document + if (reader->document(j, &doc)) { + //Add the document to the new FieldsWriter + fieldsWriter->addDocument(&doc); + docCount++; + //doc is cleard for re-use + doc.clear(); + } + } + } + } + } _CLFINALLY ( + //Close the fieldsWriter + fieldsWriter->close(); + //And have it deleted as it not used any more + _CLDELETE(fieldsWriter); + ); + + return docCount; +} + +void SegmentMerger::mergeVectors() +{ + TermVectorsWriter* termVectorsWriter = + _CLNEW TermVectorsWriter(directory, segment, fieldInfos); + + try { + for (uint32_t r = 0; r < readers.size(); r++) { + IndexReader* reader = readers[r]; + int32_t maxDoc = reader->maxDoc(); + for (int32_t docNum = 0; docNum < maxDoc; docNum++) { + // skip deleted docs + if (reader->isDeleted(docNum)) + continue; + + Array<TermFreqVector*> tmp; + if (reader->getTermFreqVectors(docNum, tmp)) + termVectorsWriter->addAllDocVectors(tmp); + tmp.deleteAll(); + } + } + } _CLFINALLY ( + _CLDELETE(termVectorsWriter); + ); +} + + +void SegmentMerger::mergeTerms() +{ + //Func - Merge the terms of all segments + //Pre - fieldInfos != NULL + //Post - The terms of all segments have been merged + + CND_PRECONDITION(fieldInfos != NULL, "fieldInfos is NULL"); + + try{ + //create a filename for the new Frequency File for segment + QString buf = Misc::segmentname(segment, QLatin1String(".frq")); + //Open an IndexOutput to the new Frequency File + freqOutput = directory->createOutput(buf); + + //create a filename for the new Prox File for segment + buf = Misc::segmentname(segment, QLatin1String(".prx")); + //Open an IndexOutput to the new Prox File + proxOutput = directory->createOutput(buf); + + //Instantiate a new termInfosWriter which will write in directory + //for the segment name segment using the new merged fieldInfos + termInfosWriter = _CLNEW TermInfosWriter(directory, segment, fieldInfos, + termIndexInterval); + + //Condition check to see if termInfosWriter points to a valid instance + CND_CONDITION(termInfosWriter != NULL, + "Memory allocation for termInfosWriter failed"); + + skipInterval = termInfosWriter->skipInterval; + queue = _CLNEW SegmentMergeQueue(readers.size()); + + //And merge the Term Infos + mergeTermInfos(); + } _CLFINALLY ( + //Close and destroy the IndexOutput to the Frequency File + if (freqOutput != NULL) { + freqOutput->close(); _CLDELETE(freqOutput); + } + + //Close and destroy the IndexOutput to the Prox File + if (proxOutput != NULL) + { + proxOutput->close(); + _CLDELETE(proxOutput); + } + + //Close and destroy the termInfosWriter + if (termInfosWriter != NULL) { + termInfosWriter->close(); + _CLDELETE(termInfosWriter); + } + + //Close and destroy the queue + if (queue != NULL) { + queue->close(); + _CLDELETE(queue); + } + ); +} + +void SegmentMerger::mergeTermInfos() +{ + //Func - Merges all TermInfos into a single segment + //Pre - true + //Post - All TermInfos have been merged into a single segment + + //Condition check to see if queue points to a valid instance + CND_CONDITION(queue != NULL, "Memory allocation for queue failed"); + + //base is the id of the first document in a segment + int32_t base = 0; + + IndexReader* reader = NULL; + SegmentMergeInfo* smi = NULL; + + //iterate through all the readers + for (uint32_t i = 0; i < readers.size(); i++) { + //Get the i-th reader + reader = readers[i]; + + //Condition check to see if reader points to a valid instance + CND_CONDITION(reader != NULL, "No IndexReader found"); + + //Get the term enumeration of the reader + TermEnum* termEnum = reader->terms(); + //Instantiate a new SegmentMerginfo for the current reader and enumeration + smi = _CLNEW SegmentMergeInfo(base, termEnum, reader); + + //Condition check to see if smi points to a valid instance + CND_CONDITION(smi != NULL, "Memory allocation for smi failed") ; + + //Increase the base by the number of documents that have not been marked deleted + //so base will contain a new value for the first document of the next iteration + base += reader->numDocs(); + //Get the next current term + if (smi->next()) { + //Store the SegmentMergeInfo smi with the initialized SegmentTermEnum TermEnum + //into the queue + queue->put(smi); + } else { + //Apparently the end of the TermEnum of the SegmentTerm has been reached so + //close the SegmentMergeInfo smi + smi->close(); + //And destroy the instance and set smi to NULL (It will be used later in this method) + _CLDELETE(smi); + } + } + + //Instantiate an array of SegmentMergeInfo instances called match + SegmentMergeInfo** match = _CL_NEWARRAY(SegmentMergeInfo*,readers.size()+1); + + //Condition check to see if match points to a valid instance + CND_CONDITION(match != NULL, "Memory allocation for match failed") ; + + SegmentMergeInfo* top = NULL; + + //As long as there are SegmentMergeInfo instances stored in the queue + while (queue->size() > 0) { + int32_t matchSize = 0; + + // pop matching terms + + //Pop the first SegmentMergeInfo from the queue + match[matchSize++] = queue->pop(); + //Get the Term of match[0] + Term* term = match[0]->term; + + //Condition check to see if term points to a valid instance + CND_CONDITION(term != NULL,"term is NULL") ; + + //Get the current top of the queue + top = queue->top(); + + //For each SegmentMergInfo still in the queue + //Check if term matches the term of the SegmentMergeInfo instances in the queue + while (top != NULL && term->equals(top->term)) { + //A match has been found so add the matching SegmentMergeInfo to the match array + match[matchSize++] = queue->pop(); + //Get the next SegmentMergeInfo + top = queue->top(); + } + match[matchSize]=NULL; + + //add new TermInfo + mergeTermInfo(match); //matchSize + + //Restore the SegmentTermInfo instances in the match array back into the queue + while (matchSize > 0) { + smi = match[--matchSize]; + + //Condition check to see if smi points to a valid instance + CND_CONDITION(smi != NULL, "smi is NULL"); + + //Move to the next term in the enumeration of SegmentMergeInfo smi + if (smi->next()) { + //There still are some terms so restore smi in the queue + queue->put(smi); + + } else { + //Done with a segment + //No terms anymore so close this SegmentMergeInfo instance + smi->close(); + _CLDELETE(smi); + } + } + } + + _CLDELETE_ARRAY(match); +} + +void SegmentMerger::mergeTermInfo(SegmentMergeInfo** smis) +{ + //Func - Merge the TermInfo of a term found in one or more segments. + //Pre - smis != NULL and it contains segments that are positioned at the same term. + // n is equal to the number of SegmentMergeInfo instances in smis + // freqOutput != NULL + // proxOutput != NULL + //Post - The TermInfo of a term has been merged + + CND_PRECONDITION(smis != NULL, "smis is NULL"); + CND_PRECONDITION(freqOutput != NULL, "freqOutput is NULL"); + CND_PRECONDITION(proxOutput != NULL, "proxOutput is NULL"); + + //Get the file pointer of the IndexOutput to the Frequency File + int64_t freqPointer = freqOutput->getFilePointer(); + //Get the file pointer of the IndexOutput to the Prox File + int64_t proxPointer = proxOutput->getFilePointer(); + + //Process postings from multiple segments all positioned on the same term. + int32_t df = appendPostings(smis); + + int64_t skipPointer = writeSkip(); + + //df contains the number of documents across all segments where this term was found + if (df > 0) { + //add an entry to the dictionary with pointers to prox and freq files + termInfo.set(df, freqPointer, proxPointer, (int32_t)(skipPointer - freqPointer)); + //Precondition check for to be sure that the reference to + //smis[0]->term will be valid + CND_PRECONDITION(smis[0]->term != NULL, "smis[0]->term is NULL"); + //Write a new TermInfo + termInfosWriter->add(smis[0]->term, &termInfo); + } +} + + +int32_t SegmentMerger::appendPostings(SegmentMergeInfo** smis) +{ + //Func - Process postings from multiple segments all positioned on the + // same term. Writes out merged entries into freqOutput and + // the proxOutput streams. + //Pre - smis != NULL and it contains segments that are positioned at the same term. + // n is equal to the number of SegmentMergeInfo instances in smis + // freqOutput != NULL + // proxOutput != NULL + //Post - Returns number of documents across all segments where this term was found + + CND_PRECONDITION(smis != NULL, "smis is NULL"); + CND_PRECONDITION(freqOutput != NULL, "freqOutput is NULL"); + CND_PRECONDITION(proxOutput != NULL, "proxOutput is NULL"); + + int32_t lastDoc = 0; + int32_t df = 0; //Document Counter + + resetSkip(); + SegmentMergeInfo* smi = NULL; + + //Iterate through all SegmentMergeInfo instances in smis + int32_t i = 0; + while ((smi=smis[i]) != NULL) { + //Get the i-th SegmentMergeInfo + + //Condition check to see if smi points to a valid instance + CND_PRECONDITION(smi != NULL, " is NULL"); + + //Get the term positions + TermPositions* postings = smi->getPositions(); + //Get the base of this segment + int32_t base = smi->base; + //Get the docMap so we can see which documents have been deleted + int32_t* docMap = smi->getDocMap(); + //Seek the termpost + postings->seek(smi->termEnum); + while (postings->next()) { + int32_t doc = postings->doc(); + //Check if there are deletions + if (docMap != NULL) + doc = docMap[doc]; // map around deletions + doc += base; // convert to merged space + + //Condition check to see doc is eaqual to or bigger than lastDoc + CND_CONDITION(doc >= lastDoc,"docs out of order"); + + //Increase the total frequency over all segments + df++; + + if ((df % skipInterval) == 0) { + bufferSkip(lastDoc); + } + + //Calculate a new docCode + //use low bit to flag freq=1 + int32_t docCode = (doc - lastDoc) << 1; + lastDoc = doc; + + //Get the frequency of the Term + int32_t freq = postings->freq(); + if (freq == 1) { + //write doc & freq=1 + freqOutput->writeVInt(docCode | 1); + } else { + //write doc + freqOutput->writeVInt(docCode); + //write frequency in doc + freqOutput->writeVInt(freq); + } + + int32_t lastPosition = 0; + // write position deltas + for (int32_t j = 0; j < freq; j++) { + //Get the next position + int32_t position = postings->nextPosition(); + //Write the difference between position and the last position + proxOutput->writeVInt(position - lastPosition); + lastPosition = position; + } + } + + i++; + } + + //Return total number of documents across all segments where term was found + return df; +} + +void SegmentMerger::resetSkip() +{ + skipBuffer->reset(); + lastSkipDoc = 0; + lastSkipFreqPointer = freqOutput->getFilePointer(); + lastSkipProxPointer = proxOutput->getFilePointer(); +} + +void SegmentMerger::bufferSkip(int32_t doc) +{ + int64_t freqPointer = freqOutput->getFilePointer(); + int64_t proxPointer = proxOutput->getFilePointer(); + + skipBuffer->writeVInt(doc - lastSkipDoc); + skipBuffer->writeVInt((int32_t) (freqPointer - lastSkipFreqPointer)); + skipBuffer->writeVInt((int32_t) (proxPointer - lastSkipProxPointer)); + + lastSkipDoc = doc; + lastSkipFreqPointer = freqPointer; + lastSkipProxPointer = proxPointer; +} + +int64_t SegmentMerger::writeSkip() +{ + int64_t skipPointer = freqOutput->getFilePointer(); + skipBuffer->writeTo(freqOutput); + return skipPointer; +} + +// Func - Merges the norms for all fields +// Pre - fieldInfos != NULL +// Post - The norms for all fields have been merged +void SegmentMerger::mergeNorms() +{ + CND_PRECONDITION(fieldInfos != NULL, "fieldInfos is NULL"); + + //iterate through all the Field Infos instances + for (int32_t i = 0; i < fieldInfos->size(); i++) { + //Get the i-th FieldInfo + FieldInfo* fi = fieldInfos->fieldInfo(i); + //Is this Field indexed? + if (fi->isIndexed && !fi->omitNorms) { + //Create and Instantiate an IndexOutput to that norm file + QString buf = Misc::segmentname(segment, QLatin1String(".f"), i); + IndexOutput* output = directory->createOutput(buf); + + //Condition check to see if output points to a valid instance + CND_CONDITION(output != NULL, "No Outputstream retrieved"); + + uint8_t* input = NULL; + try { + for (uint32_t j = 0; j < readers.size(); ++j) { + // get the next index reader + condition check + IndexReader* reader = readers[j]; + CND_CONDITION(reader != NULL, "No reader found"); + + // Get the total number of documents including the documents + // that have been marked deleted + int32_t maxDoc = reader->maxDoc(); + if (maxDoc > 0) { + // if there are docs, allocate buffer to read it's norms + uint8_t* data = (uint8_t*)realloc(input, maxDoc * + sizeof(uint8_t)); + if (data) { + input = data; + memset(input, 0, maxDoc * sizeof(uint8_t)); + // Get an IndexInput to the norm file for this + // field in this segment + reader->norms(fi->name, input); + + //Iterate through all the documents + for(int32_t k = 0; k < maxDoc; k++) { + //Check if document k is deleted + if (!reader->isDeleted(k)) { + //write the new norm + output->writeByte(input[k]); + } + } + } + } + } + } _CLFINALLY ( + if (output != NULL) { + output->close(); + _CLDELETE(output); + } + free(input); + ); + } + } +} + +CL_NS_END diff --git a/src/assistant/3rdparty/clucene/src/CLucene/index/SegmentMerger.h b/src/assistant/3rdparty/clucene/src/CLucene/index/SegmentMerger.h new file mode 100644 index 000000000..230843b00 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/index/SegmentMerger.h @@ -0,0 +1,169 @@ +/* + * Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team + * + * Distributable under the terms of either the Apache License (Version 2.0) or + * the GNU Lesser General Public License, as specified in the COPYING file. + * + * Changes are Copyright(C) 2007, 2008 by Nokia Corporation and/or its subsidiary(-ies), all rights reserved. +*/ +#ifndef _lucene_index_SegmentMerger_ +#define _lucene_index_SegmentMerger_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include <QtCore/QString> +#include <QtCore/QStringList> + +#include "CLucene/store/Directory.h" +#include "CLucene/store/RAMDirectory.h" +#include "CLucene/util/VoidList.h" +#include "SegmentMergeInfo.h" +#include "SegmentMergeQueue.h" +#include "IndexWriter.h" +#include "FieldInfos.h" +#include "FieldsWriter.h" +#include "TermInfosWriter.h" + +CL_NS_DEF(index) + +/** +* The SegmentMerger class combines two or more Segments, represented by an IndexReader ({@link #add}, +* into a single Segment. After adding the appropriate readers, call the merge method to combine the +* segments. +*<P> +* If the compoundFile flag is set, then the segments will be merged into a compound file. +* +* +* @see #merge +* @see #add +*/ +class SegmentMerger : LUCENE_BASE +{ + bool useCompoundFile; + + CL_NS(store)::RAMIndexOutput* skipBuffer; + int32_t lastSkipDoc; + int64_t lastSkipFreqPointer; + int64_t lastSkipProxPointer; + + void resetSkip(); + void bufferSkip(int32_t doc); + int64_t writeSkip(); + + //Directory of the segment + CL_NS(store)::Directory* directory; + //name of the new segment + QString segment; + //Set of IndexReaders + CL_NS(util)::CLVector<IndexReader*, + CL_NS(util)::Deletor::Object<IndexReader> > readers; + //Field Infos for t he FieldInfo instances of all fields + FieldInfos* fieldInfos; + + //The queue that holds SegmentMergeInfo instances + SegmentMergeQueue* queue; + //IndexOutput to the new Frequency File + CL_NS(store)::IndexOutput* freqOutput; + //IndexOutput to the new Prox File + CL_NS(store)::IndexOutput* proxOutput; + //Writes Terminfos that have been merged + TermInfosWriter* termInfosWriter; + TermInfo termInfo; //(new) minimize consing + + int32_t termIndexInterval; + int32_t skipInterval; + +public: + /** + * + * @param dir The Directory to merge the other segments into + * @param name The name of the new segment + * @param compoundFile true if the new segment should use a compoundFile + */ + SegmentMerger( IndexWriter* writer, const QString& name ); + + //Destructor + ~SegmentMerger(); + + /** + * Add an IndexReader to the collection of readers that are to be merged + * @param reader + */ + void add(IndexReader* reader); + + /** + * + * @param i The index of the reader to return + * @return The ith reader to be merged + */ + IndexReader* segmentReader(const int32_t i); + + /** + * Merges the readers specified by the {@link #add} method into the + * directory passed to the constructor + * @return The number of documents that were merged + * @throws IOException + */ + int32_t merge(); + /** + * close all IndexReaders that have been added. + * Should not be called before merge(). + * @throws IOException + */ + void closeReaders(); +private: + void addIndexed(IndexReader* reader, FieldInfos* fieldInfos, + CL_NS(util)::StringArrayWithDeletor& names, + bool storeTermVectors, bool storePositionWithTermVector, + bool storeOffsetWithTermVector); + + /** + * Merge the fields of all segments + * @return The number of documents in all of the readers + * @throws IOException + */ + int32_t mergeFields(); + + /** + * Merge the TermVectors from each of the segments into the new one. + * @throws IOException + */ + void mergeVectors(); + + /** Merge the terms of all segments */ + void mergeTerms(); + + /** Merges all TermInfos into a single segment */ + void mergeTermInfos(); + + /** Merge one term found in one or more segments. The array <code>smis</code> + * contains segments that are positioned at the same term. <code>N</code> + * is the number of cells in the array actually occupied. + * + * @param smis array of segments + * @param n number of cells in the array actually occupied + */ + void mergeTermInfo( SegmentMergeInfo** smis); + + /** Process postings from multiple segments all positioned on the + * same term. Writes out merged entries into freqOutput and + * the proxOutput streams. + * + * @param smis array of segments + * @param n number of cells in the array actually occupied + * @return number of documents across all segments where this term was found + */ + int32_t appendPostings(SegmentMergeInfo** smis); + + //Merges the norms for all fields + void mergeNorms(); + + void createCompoundFile(const QString& filename, QStringList& files); + friend class IndexWriter; //allow IndexWriter to use createCompoundFile +}; + +CL_NS_END + +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/index/SegmentReader.cpp b/src/assistant/3rdparty/clucene/src/CLucene/index/SegmentReader.cpp new file mode 100644 index 000000000..ba061714b --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/index/SegmentReader.cpp @@ -0,0 +1,816 @@ +/* + * Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team + * + * Distributable under the terms of either the Apache License (Version 2.0) or + * the GNU Lesser General Public License, as specified in the COPYING file. + * + * Changes are Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies). +*/ +#include "CLucene/StdHeader.h" +#include "SegmentHeader.h" + +#include "FieldInfos.h" +#include "FieldsReader.h" +#include "IndexReader.h" +#include "TermInfosReader.h" +#include "Terms.h" +#include "CLucene/search/Similarity.h" + +CL_NS_USE(util) +CL_NS_USE(store) +CL_NS_USE(document) +CL_NS_USE(search) +CL_NS_DEF(index) + +SegmentReader::Norm::Norm(IndexInput* instrm, int32_t n, SegmentReader* r, + const QString& seg) + : number(n) + , reader(r) + , segment(seg) + , in(instrm) + , bytes(NULL) + , dirty(false) +{ + //Func - Constructor + //Pre - instrm is a valid reference to an IndexInput + //Post - A Norm instance has been created with an empty bytes array + bytes = NULL; + dirty = false; +} + +SegmentReader::Norm::~Norm() +{ + //Func - Destructor + //Pre - true + //Post - The IndexInput in has been deleted (and closed by its destructor) + // and the array too. + + //Close and destroy the inputstream in-> The inputstream will be closed + // by its destructor. Note that the IndexInput 'in' actually is a pointer!!!!! + _CLDELETE(in); + + //Delete the bytes array + _CLDELETE_ARRAY(bytes); + +} + +void SegmentReader::Norm::reWrite() +{ + QString buf(segment + QLatin1String(".tmp")); + + // NOTE: norms are re-written in regular directory, not cfs + IndexOutput* out = reader->getDirectory()->createOutput(buf); + try { + out->writeBytes(bytes, reader->maxDoc()); + } _CLFINALLY ( + out->close(); + _CLDELETE(out) + ); + + QString fileName(segment); + if (reader->cfsReader == NULL) + fileName.append(QLatin1String(".f%1")).arg(number); + else // use a different file name if we have compound format + fileName.append(QLatin1String(".s%1")).arg(number); + + reader->getDirectory()->renameFile(buf, fileName); + this->dirty = false; +} + +SegmentReader::SegmentReader(SegmentInfo* si) + : IndexReader(si->getDir()) + , _norms(false, false) +{ + initialize(si); +} + +SegmentReader::SegmentReader(SegmentInfos* sis, SegmentInfo* si) + : IndexReader(si->getDir(), sis, false) + , _norms(false, false) +{ + initialize(si); +} + +void SegmentReader::initialize(SegmentInfo* si) +{ + //Pre - si-> is a valid reference to SegmentInfo instance + // identified by si-> + //Post - All files of the segment have been read + + deletedDocs = NULL; + ones = NULL; + //There are no documents yet marked as deleted + deletedDocsDirty = false; + + normsDirty=false; + undeleteAll=false; + + //Duplicate the name of the segment from SegmentInfo to segment + segment = si->name; + // make sure that all index files have been read or are kept open + // so that if an index update removes them we'll still have them + freqStream = NULL; + proxStream = NULL; + + //instantiate a buffer large enough to hold a directory path + QString buf; + + // Use compound file directory for some files, if it exists + Directory* cfsDir = getDirectory(); + SegmentName(buf, CL_MAX_PATH, QLatin1String(".cfs")); + if (cfsDir->fileExists(buf)) { + cfsReader = _CLNEW CompoundFileReader(cfsDir, buf); + cfsDir = cfsReader; + }else + cfsReader = NULL; + + // Create the name of the field info file with suffix .fnm in buf + SegmentName(buf, CL_MAX_PATH, QLatin1String(".fnm")); + fieldInfos = _CLNEW FieldInfos(cfsDir, buf ); + + // Condition check to see if fieldInfos points to a valid instance + CND_CONDITION(fieldInfos != NULL, + "No memory could be allocated for fieldInfos"); + + // Create the name of the frequence file with suffix .frq in buf + SegmentName(buf ,CL_MAX_PATH, QLatin1String(".frq")); + + // Open an IndexInput freqStream to the frequency file + freqStream = cfsDir->openInput( buf ); + + // Condition check to see if freqStream points to a valid instance and was + // able to open the frequency file + CND_CONDITION(freqStream != NULL, + "IndexInput freqStream could not open the frequency file"); + + // Create the name of the prox file with suffix .prx in buf + SegmentName(buf, CL_MAX_PATH, QLatin1String(".prx")); + + // Open an IndexInput proxStream to the prox file + proxStream = cfsDir->openInput( buf ); + + // Condition check to see if proxStream points to a valid instance and was + // able to open the prox file + CND_CONDITION(proxStream != NULL, + "IndexInput proxStream could not open proximity file"); + + // Instantiate a FieldsReader for reading the Field Info File + fieldsReader = _CLNEW FieldsReader(cfsDir, segment, fieldInfos); + + // Condition check to see if fieldsReader points to a valid instance + CND_CONDITION(fieldsReader != NULL, + "No memory could be allocated for fieldsReader"); + + //Instantiate a TermInfosReader for reading the Term Dictionary .tis file + tis = _CLNEW TermInfosReader(cfsDir, segment, fieldInfos); + + //Condition check to see if tis points to a valid instance + CND_CONDITION(tis != NULL,"No memory could be allocated for tis"); + + // Check if the segment has deletion according to the SegmentInfo instance + // si-> NOTE: the bitvector is stored using the regular directory, not cfs + if (hasDeletions(si)) { + //Create a deletion file with suffix .del + SegmentName(buf, CL_MAX_PATH, QLatin1String(".del")); + // Instantiate a BitVector that manages which documents have been deleted + deletedDocs = _CLNEW BitSet(getDirectory(), buf); + } + + // Open the norm file. There's a norm file for each indexed field with a + // byte for each document. The .f[0-9]* file contains, for each document, + // a byte that encodes a value that is multiplied into the score for hits + // on that field + openNorms(cfsDir); + + termVectorsReaderOrig = NULL; + if (fieldInfos->hasVectors()) // open term vector files only as needed + termVectorsReaderOrig = _CLNEW TermVectorsReader(cfsDir, segment, fieldInfos); +} + +SegmentReader::~SegmentReader() +{ + //Func - Destructor. + //Pre - doClose has been invoked! + //Post - the instance has been destroyed + + doClose(); //this means that index reader doesn't need to be closed manually + + _CLDELETE(fieldInfos); + _CLDELETE(fieldsReader); + _CLDELETE(tis); + _CLDELETE(freqStream); + _CLDELETE(proxStream); + _CLDELETE(deletedDocs); + _CLDELETE_ARRAY(ones); + _CLDELETE(termVectorsReaderOrig); + _CLDECDELETE(cfsReader); +} + +void SegmentReader::doCommit() +{ + QString bufdel(segment + QLatin1String(".del")); + + if (deletedDocsDirty) { // re-write deleted + QString buftmp(segment + QLatin1String(".tmp")); + deletedDocs->write(getDirectory(), buftmp); + getDirectory()->renameFile(buftmp, bufdel); + } + + if(undeleteAll && getDirectory()->fileExists(bufdel)) + getDirectory()->deleteFile(bufdel, true); + + if (normsDirty) { // re-write norms + NormsType::iterator itr = _norms.begin(); + while (itr != _norms.end()) { + Norm* norm = itr->second; + if (norm->dirty) { + norm->reWrite(); + } + ++itr; + } + } + deletedDocsDirty = false; + normsDirty = false; + undeleteAll = false; +} + +void SegmentReader::doClose() +{ + //Func - Closes all streams to the files of a single segment + //Pre - fieldsReader != NULL + // tis != NULL + //Post - All streams to files have been closed + + CND_PRECONDITION(fieldsReader != NULL, "fieldsReader is NULL"); + CND_PRECONDITION(tis != NULL, "tis is NULL"); + + //Close the fieldsReader + fieldsReader->close(); + //Close the TermInfosReader + tis->close(); + + //Close the frequency stream + if (freqStream != NULL){ + freqStream->close(); + } + //Close the prox stream + if (proxStream != NULL){ + proxStream->close(); + } + + //Close the norm file + closeNorms(); + + if (termVectorsReaderOrig != NULL) + termVectorsReaderOrig->close(); + + if (cfsReader != NULL) + cfsReader->close(); +} + +bool SegmentReader::hasDeletions() const +{ + return deletedDocs != NULL; +} + +//static +bool SegmentReader::usesCompoundFile(SegmentInfo* si) +{ + return si->getDir()->fileExists(si->name + QLatin1String(".cfs")); +} + +//static +bool SegmentReader::hasSeparateNorms(SegmentInfo* si) +{ + QString pattern(si->name); + pattern.append(QLatin1String(".s")); + size_t patternLength = pattern.length(); + + QStringList names = si->getDir()->list(); + foreach (const QString& name, names) { + int length = name.length(); + if (length > patternLength && name.left(patternLength) == pattern) { + if (name.at(patternLength) >= QLatin1Char('0') + && name.at(patternLength) <= QLatin1Char('9')) { + return true; + } + } + } + return false; +} + +bool SegmentReader::hasDeletions(const SegmentInfo* si) +{ + //Func - Static method + // Checks if a segment managed by SegmentInfo si-> has deletions + //Pre - si-> holds a valid reference to an SegmentInfo instance + //Post - if the segement contains deleteions true is returned otherwise flas + + //Check if the deletion file exists and return the result + QString f; + Misc::segmentname(f, CL_MAX_PATH, si->name, QLatin1String(".del"), -1); + return si->getDir()->fileExists(f); +} + +//synchronized +void SegmentReader::doDelete(const int32_t docNum) +{ + //Func - Marks document docNum as deleted + //Pre - docNum >=0 and DocNum < maxDoc() + // docNum contains the number of the document that must be + // marked deleted + //Post - The document identified by docNum has been marked deleted + + SCOPED_LOCK_MUTEX(THIS_LOCK) + + CND_PRECONDITION(docNum >= 0, "docNum is a negative number"); + CND_PRECONDITION(docNum < maxDoc(), + "docNum is bigger than the total number of documents"); + + //Check if deletedDocs exists + if (deletedDocs == NULL) { + deletedDocs = _CLNEW BitSet(maxDoc()); + + //Condition check to see if deletedDocs points to a valid instance + CND_CONDITION(deletedDocs != NULL, + "No memory could be allocated for deletedDocs"); + } + //Flag that there are documents marked deleted + deletedDocsDirty = true; + undeleteAll = false; + //Mark document identified by docNum as deleted + deletedDocs->set(docNum); + +} + +void SegmentReader::doUndeleteAll() +{ + _CLDELETE(deletedDocs); + deletedDocsDirty = false; + undeleteAll = true; +} + +void SegmentReader::files(QStringList& retarray) +{ + //Func - Returns all file names managed by this SegmentReader + //Pre - segment != NULL + //Post - All filenames managed by this SegmentRead have been returned + + CND_PRECONDITION(segment != NULL, "segment is NULL"); + + QString temp; + #define _ADD_SEGMENT(ext) \ + temp = SegmentName(ext); \ + if (getDirectory()->fileExists(temp)) \ + retarray.push_back(temp); + + //Add the name of the Field Info file + _ADD_SEGMENT(QLatin1String(".cfs")); + _ADD_SEGMENT(QLatin1String(".fnm")); + _ADD_SEGMENT(QLatin1String(".fdx")); + _ADD_SEGMENT(QLatin1String(".fdt")); + _ADD_SEGMENT(QLatin1String(".tii")); + _ADD_SEGMENT(QLatin1String(".tis")); + _ADD_SEGMENT(QLatin1String(".frq")); + _ADD_SEGMENT(QLatin1String(".prx")); + _ADD_SEGMENT(QLatin1String(".del")); + _ADD_SEGMENT(QLatin1String(".tvx")); + _ADD_SEGMENT(QLatin1String(".tvd")); + _ADD_SEGMENT(QLatin1String(".tvf")); + _ADD_SEGMENT(QLatin1String(".tvp")); + + //iterate through the field infos + for (int32_t i = 0; i < fieldInfos->size(); ++i) { + //Get the field info for the i-th field + FieldInfo* fi = fieldInfos->fieldInfo(i); + //Check if the field has been indexed + if (fi->isIndexed && !fi->omitNorms) { + QString name; + if (cfsReader == NULL) + name = SegmentName(QLatin1String(".f"), i); + else + name = SegmentName(QLatin1String(".s"), i); + + //The field has been indexed so add its norm file + if (getDirectory()->fileExists(name)) + retarray.push_back(name); + } + } +} + +TermEnum* SegmentReader::terms() const +{ + //Func - Returns an enumeration of all the Terms and TermInfos in the set. + //Pre - tis != NULL + //Post - An enumeration of all the Terms and TermInfos in the set has been returned + + CND_PRECONDITION(tis != NULL, "tis is NULL"); + + return tis->terms(); +} + +TermEnum* SegmentReader::terms(const Term* t) const +{ + //Func - Returns an enumeration of terms starting at or after the named term t + //Pre - t != NULL + // tis != NULL + //Post - An enumeration of terms starting at or after the named term t + + CND_PRECONDITION(t != NULL, "t is NULL"); + CND_PRECONDITION(tis != NULL, "tis is NULL"); + + return tis->terms(t); +} + +bool SegmentReader::document(int32_t n, Document* doc) +{ + //Func - Returns a document identified by n + //Pre - n >=0 and identifies the document n + //Post - if the document has been deleted then an exception has been thrown + // otherwise a reference to the found document has been returned + + SCOPED_LOCK_MUTEX(THIS_LOCK) + + CND_PRECONDITION(n >= 0, "n is a negative number"); + + //Check if the n-th document has been marked deleted + if (isDeleted(n)) + _CLTHROWA(CL_ERR_InvalidState, "attempt to access a deleted document" ); + + //Retrieve the n-th document + return fieldsReader->doc(n, doc); +} + +bool SegmentReader::isDeleted(const int32_t n) +{ + //Func - Checks if the n-th document has been marked deleted + //Pre - n >=0 and identifies the document n + //Post - true has been returned if document n has been deleted otherwise fralse + + SCOPED_LOCK_MUTEX(THIS_LOCK) + + CND_PRECONDITION(n >= 0, "n is a negative number"); + + //Is document n deleted + return (deletedDocs != NULL && deletedDocs->get(n)); +} + +TermDocs* SegmentReader::termDocs() const +{ + //Func - Returns an unpositioned TermDocs enumerator. + //Pre - true + //Post - An unpositioned TermDocs enumerator has been returned + + return _CLNEW SegmentTermDocs(this); +} + +TermPositions* SegmentReader::termPositions() const +{ + //Func - Returns an unpositioned TermPositions enumerator. + //Pre - true + //Post - An unpositioned TermPositions enumerator has been returned + + return _CLNEW SegmentTermPositions(this); +} + +int32_t SegmentReader::docFreq(const Term* t) const +{ + //Func - Returns the number of documents which contain the term t + //Pre - t holds a valid reference to a Term + //Post - The number of documents which contain term t has been returned + + //Get the TermInfo ti for Term t in the set + TermInfo* ti = tis->get(t); + //Check if an TermInfo has been returned + if (ti) { + //Get the frequency of the term + int32_t ret = ti->docFreq; + //TermInfo ti is not needed anymore so delete it + _CLDELETE( ti ); + //return the number of documents which containt term t + return ret; + } + + //No TermInfo returned so return 0 + return 0; +} + +int32_t SegmentReader::numDocs() +{ + //Func - Returns the actual number of documents in the segment + //Pre - true + //Post - The actual number of documents in the segments + + //Get the number of all the documents in the segment including the ones that have + //been marked deleted + int32_t n = maxDoc(); + + //Check if there any deleted docs + if (deletedDocs != NULL) + //Substract the number of deleted docs from the number returned by maxDoc + n -= deletedDocs->count(); + + //return the actual number of documents in the segment + return n; +} + +int32_t SegmentReader::maxDoc() const +{ + //Func - Returns the number of all the documents in the segment including + // the ones that have been marked deleted + //Pre - true + //Post - The total number of documents in the segment has been returned + + return fieldsReader->size(); +} + +void SegmentReader::getFieldNames(FieldOption fldOption, + StringArrayWithDeletor& retarray) +{ + size_t len = fieldInfos->size(); + for (size_t i = 0; i < len; i++) { + bool v = false; + FieldInfo* fi = fieldInfos->fieldInfo(i); + if (fldOption & IndexReader::ALL) { + v = true; + } else { + if (!fi->isIndexed && (fldOption & IndexReader::UNINDEXED)) { + v = true; + } + + if (fi->isIndexed && (fldOption & IndexReader::INDEXED)) { + v = true; + } + + if (fi->isIndexed && fi->storeTermVector == false + && (fldOption & IndexReader::INDEXED_NO_TERMVECTOR)) { + v = true; + } + + if ((fldOption & IndexReader::TERMVECTOR) + && fi->storeTermVector == true + && fi->storePositionWithTermVector == false + && fi->storeOffsetWithTermVector == false) { + v = true; + } + + if (fi->isIndexed && fi->storeTermVector + && (fldOption & IndexReader::INDEXED_WITH_TERMVECTOR)) { + v = true; + } + + if (fi->storePositionWithTermVector + && fi->storeOffsetWithTermVector == false + && (fldOption & IndexReader::TERMVECTOR_WITH_POSITION)) { + v = true; + } + + if (fi->storeOffsetWithTermVector + && fi->storePositionWithTermVector == false + && (fldOption & IndexReader::TERMVECTOR_WITH_OFFSET)) { + v = true; + } + + if ((fi->storeOffsetWithTermVector && fi->storePositionWithTermVector) + && (fldOption & IndexReader::TERMVECTOR_WITH_POSITION_OFFSET)) { + v = true; + } + } + + if (v) + retarray.push_back(STRDUP_TtoT(fi->name)); + } +} + +bool SegmentReader::hasNorms(const TCHAR* field) const +{ + return _norms.find(field) != _norms.end(); +} + + +void SegmentReader::norms(const TCHAR* field, uint8_t* bytes) +{ + //Func - Reads the Norms for field from disk starting at offset in the inputstream + //Pre - field != NULL + // bytes != NULL is an array of bytes which is to be used to read the norms into. + // it is advisable to have bytes initalized by zeroes! + //Post - The if an inputstream to the norm file could be retrieved the bytes have been read + // You are never sure whether or not the norms have been read into bytes properly!!!!!!!!!!!!!!!!! + + CND_PRECONDITION(field != NULL, "field is NULL"); + CND_PRECONDITION(bytes != NULL, "field is NULL"); + + SCOPED_LOCK_MUTEX(THIS_LOCK) + + Norm* norm = _norms.get(field); + if ( norm == NULL ){ + memcpy(bytes, fakeNorms(), maxDoc()); + return; + } + + if (norm->bytes != NULL) { // can copy from cache + memcpy(bytes, norm->bytes, maxDoc()); + return; + } + + IndexInput* _normStream = norm->in->clone(); + CND_PRECONDITION(_normStream != NULL, "normStream==NULL") + + // read from disk + try { + _normStream->seek(0); + _normStream->readBytes(bytes, maxDoc()); + } _CLFINALLY ( + //Have the normstream closed + _normStream->close(); + //Destroy the normstream + _CLDELETE( _normStream ); + ); +} + +uint8_t* SegmentReader::createFakeNorms(int32_t size) +{ + uint8_t* ones = _CL_NEWARRAY(uint8_t,size); + memset(ones, DefaultSimilarity::encodeNorm(1.0f), size); + return ones; +} + +uint8_t* SegmentReader::fakeNorms() +{ + if (ones == NULL) + ones = createFakeNorms(maxDoc()); + return ones; +} + +// can return null if norms aren't stored +uint8_t* SegmentReader::getNorms(const TCHAR* field) +{ + SCOPED_LOCK_MUTEX(THIS_LOCK) + Norm* norm = _norms.get(field); + if (norm == NULL) + return NULL; // not indexed, or norms not stored + + if (norm->bytes == NULL) { // value not yet read + uint8_t* bytes = _CL_NEWARRAY(uint8_t, maxDoc()); + norms(field, bytes); + norm->bytes = bytes; // cache it + } + return norm->bytes; +} + +uint8_t* SegmentReader::norms(const TCHAR* field) +{ + //Func - Returns the bytes array that holds the norms of a named field + //Pre - field != NULL and contains the name of the field for which the norms + // must be retrieved + //Post - If there was norm for the named field then a bytes array has been allocated + // and returned containing the norms for that field. If the named field is unknown NULL is returned. + + CND_PRECONDITION(field != NULL, "field is NULL"); + + SCOPED_LOCK_MUTEX(THIS_LOCK) + + uint8_t* bytes = getNorms(field); + if (bytes == NULL) + bytes = fakeNorms(); + return bytes; +} + +void SegmentReader::doSetNorm(int32_t doc, const TCHAR* field, uint8_t value) +{ + Norm* norm = _norms.get(field); + if (norm == NULL) // not an indexed field + return; + + norm->dirty = true; // mark it dirty + normsDirty = true; + + uint8_t* bits = norms(field); + bits[doc] = value; // set the value +} + +QString SegmentReader::SegmentName(const QString& ext, const int32_t x) +{ + //Func - Returns an allocated buffer in which it creates a filename by + // concatenating segment with ext and x + //Pre ext != NULL and holds the extension + // x contains a number + //Post - A buffer has been instantiated an when x = -1 buffer contains the concatenation of + // segment and ext otherwise buffer contains the contentation of segment, ext and x + + CND_PRECONDITION(!ext.isEmpty(), "ext is NULL"); + + QString buf; + SegmentName(buf, CL_MAX_PATH, ext, x); + return buf; +} + +void SegmentReader::SegmentName(QString& buffer, int32_t bufferLen, + const QString& ext, const int32_t x) +{ + //Func - Creates a filename in buffer by concatenating segment with ext and x + //Pre - buffer != NULL + // ext != NULL + // x contains a number + //Post - When x = -1 buffer contains the concatenation of segment and ext otherwise + // buffer contains the contentation of segment, ext and x + + CND_PRECONDITION(!segment.isEmpty(), "Segment is NULL"); + + Misc::segmentname(buffer, bufferLen, segment, ext, x); +} + +void SegmentReader::openNorms(Directory* cfsDir) +{ + //Func - Open all norms files for all fields + // Creates for each field a norm Instance with an open inputstream to + // a corresponding norm file ready to be read + //Pre - true + //Post - For each field a norm instance has been created with an open inputstream to + // a corresponding norm file ready to be read + + //Iterate through all the fields + for (int32_t i = 0; i < fieldInfos->size(); i++) { + //Get the FieldInfo for the i-th field + FieldInfo* fi = fieldInfos->fieldInfo(i); + //Check if the field is indexed + if (fi->isIndexed && !fi->omitNorms ) { + //Allocate a buffer + QString fileName; + + // look first if there are separate norms in compound format + SegmentName(fileName, CL_MAX_PATH, QLatin1String(".s"), fi->number); + Directory* d = getDirectory(); + if(!d->fileExists(fileName)){ + SegmentName(fileName, CL_MAX_PATH, QLatin1String(".f"), fi->number); + d = cfsDir; + } + + _norms.put(fi->name, _CLNEW Norm(d->openInput(fileName), + fi->number, this, segment)); + } + } +} + +void SegmentReader::closeNorms() +{ + //Func - Close all the norms stored in norms + //Pre - true + //Post - All the norms have been destroyed + + SCOPED_LOCK_MUTEX(_norms.THIS_LOCK) + + //Create an interator initialized at the beginning of norms + NormsType::iterator itr = _norms.begin(); + //Iterate through all the norms + while (itr != _norms.end()) { + // Get, delete the norm + _CLDELETE(itr->second); + // Move the interator to the next norm in the norms collection. + // Note ++ is an overloaded operator + ++itr; + } + _norms.clear(); //bvk: they're deleted, so clear them so that they are not re-used +} + +TermVectorsReader* SegmentReader::getTermVectorsReader() +{ + TermVectorsReader* tvReader = termVectorsLocal.get(); + if (tvReader == NULL) { + tvReader = termVectorsReaderOrig->clone(); + termVectorsLocal.set(tvReader); + } + return tvReader; +} + +TermFreqVector* SegmentReader::getTermFreqVector(int32_t docNumber, + const TCHAR* field) +{ + if (field) { + FieldInfo* fi = fieldInfos->fieldInfo(field); + // Check if this field is invalid or has no stored term vector + if (fi == NULL || !fi->storeTermVector || termVectorsReaderOrig == NULL) + return NULL; + } + + TermVectorsReader* termVectorsReader = getTermVectorsReader(); + if (termVectorsReader == NULL) + return NULL; + + return termVectorsReader->get(docNumber, field); +} + +bool SegmentReader::getTermFreqVectors(int32_t docNumber, + Array<TermFreqVector*>& result) +{ + if (termVectorsReaderOrig == NULL) + return false; + + TermVectorsReader* termVectorsReader = getTermVectorsReader(); + if (termVectorsReader == NULL) + return false; + + return termVectorsReader->get(docNumber, result); +} + +CL_NS_END diff --git a/src/assistant/3rdparty/clucene/src/CLucene/index/SegmentTermDocs.cpp b/src/assistant/3rdparty/clucene/src/CLucene/index/SegmentTermDocs.cpp new file mode 100644 index 000000000..50951e9ba --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/index/SegmentTermDocs.cpp @@ -0,0 +1,216 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#include "CLucene/StdHeader.h" +#include "SegmentHeader.h" + +#include "CLucene/store/IndexInput.h" +#include "Term.h" + +CL_NS_DEF(index) + + SegmentTermDocs::SegmentTermDocs(const SegmentReader* _parent){ + //Func - Constructor + //Pre - Paren != NULL + //Post - The instance has been created + + CND_PRECONDITION(_parent != NULL,"Parent is NULL"); + + parent = _parent; + deletedDocs = parent->deletedDocs; + + _doc = 0; + _freq = 0; + count = 0; + df = 0; + + skipInterval=0; + numSkips=0; + skipCount=0; + skipStream=NULL; + skipDoc=0; + freqPointer=0; + proxPointer=0; + skipPointer=0; + haveSkipped=false; + + freqStream = parent->freqStream->clone(); + skipInterval = parent->tis->getSkipInterval(); + } + + SegmentTermDocs::~SegmentTermDocs() { + //Func - Destructor + //Pre - true + //Post - The instance has been destroyed + + close(); + } + + TermPositions* SegmentTermDocs::__asTermPositions(){ + return NULL; + } + + void SegmentTermDocs::seek(Term* term) { + TermInfo* ti = parent->tis->get(term); + seek(ti); + _CLDELETE(ti); + } + + void SegmentTermDocs::seek(TermEnum* termEnum){ + TermInfo* ti=NULL; + + // use comparison of fieldinfos to verify that termEnum belongs to the same segment as this SegmentTermDocs + if ( termEnum->getObjectName() == SegmentTermEnum::getClassName() ){ + SegmentTermEnum* te = (SegmentTermEnum*)termEnum; + te->fieldInfos = parent->fieldInfos; + ti = te->getTermInfo(); + }else{ + ti = parent->tis->get(termEnum->term(false)); + } + + seek(ti); + _CLDELETE(ti); + } + void SegmentTermDocs::seek(const TermInfo* ti) { + count = 0; + if (ti == NULL) { + df = 0; + } else { + df = ti->docFreq; + _doc = 0; + skipDoc = 0; + skipCount = 0; + numSkips = df / skipInterval; + freqPointer = ti->freqPointer; + proxPointer = ti->proxPointer; + skipPointer = freqPointer + ti->skipOffset; + freqStream->seek(freqPointer); + haveSkipped = false; + } + } + + void SegmentTermDocs::close() { + + //Check if freqStream still exists + if (freqStream != NULL){ + freqStream->close(); //todo: items like these can probably be delete, because deleting the object also closes it...do everywhere + _CLDELETE( freqStream ); + } + if (skipStream != NULL){ + skipStream->close(); + _CLDELETE( skipStream ); + } + } + + int32_t SegmentTermDocs::doc()const { + return _doc; + } + int32_t SegmentTermDocs::freq()const { + return _freq; + } + + +bool SegmentTermDocs::next() +{ + while (true) { + if (count == df) + return false; + + uint32_t docCode = freqStream->readVInt(); + _doc += docCode >> 1; //unsigned shift + if ((docCode & 1) != 0) // if low bit is set + _freq = 1; // _freq is one + else + _freq = freqStream->readVInt(); // else read _freq + count++; + + if (deletedDocs == NULL || (_doc >= 0 && !deletedDocs->get(_doc))) + break; + skippingDoc(); + } + return true; +} + + +int32_t SegmentTermDocs::read(int32_t* docs, int32_t* freqs, int32_t length) +{ + int32_t i = 0; + // TODO: one optimization would be to get the pointer buffer for ram or mmap + // dirs and iterate over them instead of using readByte() intensive functions. + while (i < length && count < df) { + uint32_t docCode = freqStream->readVInt(); + _doc += docCode >> 1; + if ((docCode & 1) != 0) // if low bit is set + _freq = 1; // _freq is one + else + _freq = freqStream->readVInt(); // else read _freq + count++; + + if (deletedDocs == NULL || (_doc >= 0 && !deletedDocs->get(_doc))) { + docs[i] = _doc; + freqs[i] = _freq; + i++; + } + } + return i; +} + + bool SegmentTermDocs::skipTo(const int32_t target){ + if (df >= skipInterval) { // optimized case + if (skipStream == NULL) + skipStream = freqStream->clone(); // lazily clone + + if (!haveSkipped) { // lazily seek skip stream + skipStream->seek(skipPointer); + haveSkipped = true; + } + + // scan skip data + int32_t lastSkipDoc = skipDoc; + int64_t lastFreqPointer = freqStream->getFilePointer(); + int64_t lastProxPointer = -1; + int32_t numSkipped = -1 - (count % skipInterval); + + while (target > skipDoc) { + lastSkipDoc = skipDoc; + lastFreqPointer = freqPointer; + lastProxPointer = proxPointer; + + if (skipDoc != 0 && skipDoc >= _doc) + numSkipped += skipInterval; + + if(skipCount >= numSkips) + break; + + skipDoc += skipStream->readVInt(); + freqPointer += skipStream->readVInt(); + proxPointer += skipStream->readVInt(); + + skipCount++; + } + + // if we found something to skip, then skip it + if (lastFreqPointer > freqStream->getFilePointer()) { + freqStream->seek(lastFreqPointer); + skipProx(lastProxPointer); + + _doc = lastSkipDoc; + count += numSkipped; + } + + } + + // done skipping, now just scan + + do { + if (!next()) + return false; + } while (target > _doc); + return true; + } + + +CL_NS_END diff --git a/src/assistant/3rdparty/clucene/src/CLucene/index/SegmentTermEnum.cpp b/src/assistant/3rdparty/clucene/src/CLucene/index/SegmentTermEnum.cpp new file mode 100644 index 000000000..20e286fd1 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/index/SegmentTermEnum.cpp @@ -0,0 +1,389 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#include "CLucene/StdHeader.h" +#include "SegmentTermEnum.h" + +#include "Terms.h" +#include "FieldInfos.h" +#include "Term.h" +#include "TermInfo.h" +#include "TermInfosWriter.h" + +CL_NS_USE(store) +CL_NS_DEF(index) + + SegmentTermEnum::SegmentTermEnum(IndexInput* i, FieldInfos* fis, const bool isi): + fieldInfos(fis){ + //Func - Constructor + //Pre - i holds a reference to an instance of IndexInput + // fis holds a reference to an instance of FieldInfos + // isi + //Post - An instance of SegmentTermEnum has been created + input = i; + position = -1; + //Instantiate a Term with empty field, empty text and which is interned (see term.h what interned means) + _term = _CLNEW Term; + isIndex = isi; + termInfo = _CLNEW TermInfo(); + indexPointer = 0; + buffer = NULL; + bufferLength = 0; + prev = NULL; + formatM1SkipInterval = 0; + + //Set isClone to false as the instance is not clone of another instance + isClone = false; + + + int32_t firstInt = input->readInt(); + if (firstInt >= 0) { + // original-format file, without explicit format version number + format = 0; + size = firstInt; + + // back-compatible settings + indexInterval = 128; + skipInterval = LUCENE_INT32_MAX_SHOULDBE; // switch off skipTo optimization + + } else { + // we have a format version number + format = firstInt; + + // check that it is a format we can understand + if (format < TermInfosWriter::FORMAT){ + TCHAR err[30]; + _sntprintf(err,30,_T("Unknown format version: %d"), format); + _CLTHROWT(CL_ERR_Runtime,err); + } + + size = input->readLong(); // read the size + + if(format == -1){ + if (!isIndex) { + indexInterval = input->readInt(); + formatM1SkipInterval = input->readInt(); + } + // switch off skipTo optimization for file format prior to 1.4rc2 in order to avoid a bug in + // skipTo implementation of these versions + skipInterval = LUCENE_INT32_MAX_SHOULDBE; + }else{ + indexInterval = input->readInt(); + skipInterval = input->readInt(); + } + } + } + + SegmentTermEnum::SegmentTermEnum(const SegmentTermEnum& clone): + fieldInfos(clone.fieldInfos) + { + //Func - Constructor + // The instance is created by cloning all properties of clone + //Pre - clone holds a valid reference to SegmentTermEnum + //Post - An instance of SegmentTermEnum with the same properties as clone + + input = clone.input->clone(); + //Copy the postion from the clone + position = clone.position; + + if ( clone._term != NULL ){ + _term = _CLNEW Term; + _term->set(clone._term,clone._term->text()); + }else + _term = NULL; + isIndex = clone.isIndex; + termInfo = _CLNEW TermInfo(clone.termInfo); + indexPointer = clone.indexPointer; + buffer = clone.buffer==NULL?NULL:(TCHAR*)malloc(sizeof(TCHAR) * (clone.bufferLength+1)); + bufferLength = clone.bufferLength; + prev = clone.prev==NULL?NULL:_CLNEW Term(clone.prev->field(),clone.prev->text(),false); + size = clone.size; + + format = clone.format; + indexInterval= clone.indexInterval; + skipInterval = clone.skipInterval; + formatM1SkipInterval = clone.formatM1SkipInterval; + //Set isClone to true as this instance is a clone of another instance + isClone = true; + + //Copy the contents of buffer of clone to the buffer of this instance + if ( clone.buffer != NULL ) + memcpy(buffer,clone.buffer,bufferLength * sizeof(TCHAR)); + } + + SegmentTermEnum::~SegmentTermEnum(){ + //Func - Destructor + //Pre - true + //Post - The instance has been destroyed. If this instance was a clone + // then the inputstream is closed and deleted too. + + //todo: revisit this... close() should clean up most of everything. + + //Finalize prev + _CLDECDELETE(prev ); + //Finalize term + _CLDECDELETE( _term ); + + + //Delete the buffer if necessary + free(buffer); + //Delete termInfo if necessary + _CLDELETE(termInfo); + + //Check if this instance is a clone + if ( isClone ){ + //Close the inputstream + input->close(); + //delete the inputstream + _CLDELETE(input); + } + } + + bool SegmentTermEnum::next(){ + //Func - Moves the current of the set to the next in the set + //Pre - true + //Post - If the end has been reached NULL is returned otherwise the term has + // become the next Term in the enumeration + + //Increase position by and and check if the end has been reached + if (position++ >= size-1) { + //delete term + _CLDECDELETE(_term); + return false; + } + + //delete the previous enumerated term + Term* tmp=NULL; + if ( prev != NULL ){ + int32_t usage = prev->__cl_refcount; + if ( usage > 1 ){ + _CLDECDELETE(prev); //todo: tune other places try and delete its term + }else + tmp = prev; //we are going to re-use this term + } + //prev becomes the current enumerated term + prev = _term; + //term becomes the next term read from inputStream input + _term = readTerm(tmp); + + //Read docFreq, the number of documents which contain the term. + termInfo->docFreq = input->readVInt(); + //Read freqPointer, a pointer into the TermFreqs file (.frq) + termInfo->freqPointer += input->readVLong(); + + //Read proxPointer, a pointer into the TermPosition file (.prx). + termInfo->proxPointer += input->readVLong(); + + if(format == -1){ + // just read skipOffset in order to increment file pointer; + // value is never used since skipTo is switched off + if (!isIndex) { + if (termInfo->docFreq > formatM1SkipInterval) { + termInfo->skipOffset = input->readVInt(); + } + } + }else{ + if (termInfo->docFreq >= skipInterval) + termInfo->skipOffset = input->readVInt(); + } + + //Check if the enumeration is an index + if (isIndex) + //read index pointer + indexPointer += input->readVLong(); + + return true; + } + + Term* SegmentTermEnum::term() { + //Func - Returns the current term. + //Pre - pointer is true or false and indicates if the reference counter + // of term must be increased or not + // next() must have been called once! + //Post - pointer = true -> term has been returned with an increased reference counter + // pointer = false -> term has been returned + + return _CL_POINTER(_term); + } + Term* SegmentTermEnum::term(bool pointer) { + if ( pointer ) + return _CL_POINTER(_term); + else + return _term; + } + + void SegmentTermEnum::scanTo(const Term *term){ + //Func - Scan for Term without allocating new Terms + //Pre - term != NULL + //Post - The iterator term has been moved to the position where Term is expected to be + // in the enumeration + while ( term->compareTo(this->_term) > 0 && next()) + { + } + } + + void SegmentTermEnum::close() { + //Func - Closes the enumeration to further activity, freeing resources. + //Pre - true + //Post - The inputStream input has been closed + + input->close(); + } + + int32_t SegmentTermEnum::docFreq() const { + //Func - Returns the document frequency of the current term in the set + //Pre - termInfo != NULL + // next() must have been called once + //Post - The document frequency of the current enumerated term has been returned + + return termInfo->docFreq; + } + + void SegmentTermEnum::seek(const int64_t pointer, const int32_t p, Term* t, TermInfo* ti) { + //Func - Repositions term and termInfo within the enumeration + //Pre - pointer >= 0 + // p >= 0 and contains the new position within the enumeration + // t is a valid reference to a Term and is the new current term in the enumeration + // ti is a valid reference to a TermInfo and is corresponding TermInfo form the new + // current Term + //Post - term and terminfo have been repositioned within the enumeration + + //Reset the IndexInput input to pointer + input->seek(pointer); + //Assign the new position + position = p; + + //finalize the current term + if ( _term == NULL || _term->__cl_refcount > 1 ){ + _CLDECDELETE(_term); + //Get a pointer from t and increase the reference counter of t + _term = _CLNEW Term; //cannot use reference, because TermInfosReader uses non ref-counted array + } + _term->set(t,t->text()); + + //finalize prev + _CLDECDELETE(prev); + + //Change the current termInfo so it matches the new current term + termInfo->set(ti); + + //Have the buffer grown if needed + if ( bufferLength <= _term->textLength() ) + growBuffer(_term->textLength(), true ); // copy term text into buffer + else + _tcsncpy(buffer,_term->text(),bufferLength); //just copy the buffer + } + + TermInfo* SegmentTermEnum::getTermInfo()const { + //Func - Returns a clone of the current termInfo + //Pre - termInfo != NULL + // next() must have been called once + //Post - A clone of the current termInfo has been returned + + return _CLNEW TermInfo(*termInfo); //clone + } + + void SegmentTermEnum::getTermInfo(TermInfo* ti)const { + //Func - Retrieves a clone of termInfo through the reference ti + //Pre - ti contains a valid reference to TermInfo + // termInfo != NULL + // next() must have been called once + //Post - ti contains a clone of termInfo + + ti->set(termInfo); + } + + int64_t SegmentTermEnum::freqPointer()const { + //Func - Returns the freqpointer of the current termInfo + //Pre - termInfo != NULL + // next() must have been called once + //Post - The freqpointer of the current termInfo has been returned + + return termInfo->freqPointer; + } + + int64_t SegmentTermEnum::proxPointer()const { + //Func - Returns the proxPointer of the current termInfo + //Pre - termInfo != NULL + // next() must have been called once + //Post - the proxPointer of the current termInfo has been returned + + return termInfo->proxPointer; + } + + SegmentTermEnum* SegmentTermEnum::clone() const { + //Func - Returns a clone of this instance + //Pre - true + //Post - An clone of this instance has been returned + + return _CLNEW SegmentTermEnum(*this); + } + + Term* SegmentTermEnum::readTerm(Term* reuse) { + //Func - Reads the next term in the enumeration + //Pre - true + //Post - The next Term in the enumeration has been read and returned + + //Read the start position from the inputStream input + int32_t start = input->readVInt(); + //Read the length of term in the inputStream input + int32_t length = input->readVInt(); + + //Calculated the total lenght of bytes that buffer must be to contain the current + //chars in buffer and the new ones yet to be read + uint32_t totalLength = start + length; + + if (static_cast<uint32_t>(bufferLength) < totalLength+1) + growBuffer(totalLength, false); + + //Read a length number of characters into the buffer from position start in the inputStream input + input->readChars(buffer, start, length); + //Null terminate the string + buffer[totalLength] = 0; + + //Return a new Term + int32_t field = input->readVInt(); + const TCHAR* fieldname = fieldInfos->fieldName(field); + if ( reuse == NULL ) + reuse = _CLNEW Term; + + reuse->set(fieldname, buffer, false); + return reuse; + } + + void SegmentTermEnum::growBuffer(const uint32_t length, bool force_copy) { + //Func - Instantiate a buffer of length length+1 + //Pre - length > 0 + //Post - pre(buffer) has been deleted with its contents. A new buffer + // has been allocated of length length+1 and the text of term has been copied + // to buffer + //todo: we could guess that we will need to re-grow this + //buffer a few times...so start off with a reasonable grow + //value... + if ( bufferLength > length ) + return; + + //Store the new bufferLength + if ( length - bufferLength < LUCENE_SEGMENTTERMENUM_GROWSIZE ) + bufferLength = length+LUCENE_SEGMENTTERMENUM_GROWSIZE; + else + bufferLength = length+1; + + bool copy = buffer==NULL; + + //Instantiate the new buffer + 1 is needed for terminator '\0' + if ( buffer == NULL ) + buffer = (TCHAR*)malloc(sizeof(TCHAR) * (bufferLength+1)); + else + buffer = (TCHAR*)realloc(buffer, sizeof(TCHAR) * (bufferLength+1)); + + if ( copy || force_copy){ + //Copy the text of term into buffer + _tcsncpy(buffer,_term->text(),bufferLength); + } + } + +CL_NS_END diff --git a/src/assistant/3rdparty/clucene/src/CLucene/index/SegmentTermEnum.h b/src/assistant/3rdparty/clucene/src/CLucene/index/SegmentTermEnum.h new file mode 100644 index 000000000..0d50103f3 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/index/SegmentTermEnum.h @@ -0,0 +1,138 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _lucene_index_SegmentTermEnum_ +#define _lucene_index_SegmentTermEnum_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include "Terms.h" +#include "FieldInfos.h" +#include "TermInfo.h" + +CL_NS_DEF(index) + +/** + * SegmentTermEnum is an enumeration of all Terms and TermInfos + */ +class SegmentTermEnum:public TermEnum{ +private: + Term* _term; ///points to the current Term in the enumeration + TermInfo* termInfo; ///points to the TermInfo matching the current Term in the enumeration + + bool isIndex; ///Indicates if the Segment is a an index + bool isClone; ///Indicates if SegmentTermEnum is an orignal instance or + ///a clone of another SegmentTermEnum + + TCHAR* buffer; ///The buffer that contains the data read from the Term Infos File + uint32_t bufferLength; ///Length of the buffer + + int32_t format; + int32_t formatM1SkipInterval; + + CL_NS(store)::IndexInput* input; ///The IndexInput that reads from the Term Infos File + FieldInfos* fieldInfos; ///contains the Field Infos for the segment + int64_t size; ///The size of the enumeration + int64_t position; ///The position of the current (term) in the enumeration + int64_t indexPointer; + Term* prev; ///The previous current + int32_t indexInterval; + int32_t skipInterval; + + friend class TermInfosReader; + friend class SegmentTermDocs; +protected: + + /** + * Constructor. + * The instance is created by cloning all properties of clone + */ + SegmentTermEnum( const SegmentTermEnum& clone); + +public: + ///Constructor + SegmentTermEnum(CL_NS(store)::IndexInput* i, FieldInfos* fis, const bool isi ); + + ///Destructor + ~SegmentTermEnum(); + + /** + * Moves the current of the set to the next in the set + */ + bool next(); + + /** + * Returns a pointer to the current term. + */ + Term* term(); + /** + * Returns the current term. + */ + Term* term(bool pointer); + + /** + * Scan for Term term without allocating new Terms + */ + void scanTo(const Term *term); + + /** + * Closes the enumeration to further activity, freeing resources. + */ + void close(); + + /** + * Returns the document frequency of the current term in the set + */ + int32_t docFreq() const; + + /** + * Repositions term and termInfo within the enumeration + */ + void seek(const int64_t pointer, const int32_t p, Term* t, TermInfo* ti); + + /** + * Returns a clone of the current termInfo + */ + TermInfo* getTermInfo()const; + + /** + * Retrieves a clone of termInfo through the reference ti + */ + void getTermInfo(TermInfo* ti)const; + + /** + * Returns the freqPointer from the current TermInfo in the enumeration. + */ + int64_t freqPointer() const; + + /** + * Returns the proxPointer from the current TermInfo in the enumeration. + */ + int64_t proxPointer() const; + + /** + * Returns a clone of this instance + */ + SegmentTermEnum* clone() const; + + const char* getObjectName(){ return SegmentTermEnum::getClassName(); } + static const char* getClassName(){ return "SegmentTermEnum"; } + +private: + /** + * Reads the next term in the enumeration + */ + Term* readTerm(Term* reuse); + /** + * Instantiate a buffer of length length+1 + */ + void growBuffer(const uint32_t length, bool force_copy); + +}; +CL_NS_END +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/index/SegmentTermPositions.cpp b/src/assistant/3rdparty/clucene/src/CLucene/index/SegmentTermPositions.cpp new file mode 100644 index 000000000..e481838e9 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/index/SegmentTermPositions.cpp @@ -0,0 +1,101 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#include "CLucene/StdHeader.h" +#include "SegmentHeader.h" + +#include "Terms.h" + +CL_NS_USE(util) +CL_NS_DEF(index) + +SegmentTermPositions::SegmentTermPositions(const SegmentReader* _parent): + SegmentTermDocs(_parent){ +//Func - Constructor +//Pre - Parent != NULL +//Post - The instance has been created + + CND_PRECONDITION(_parent != NULL, "Parent is NULL"); + + proxStream = _parent->proxStream->clone(); + + CND_CONDITION(proxStream != NULL,"proxStream is NULL"); + + position = 0; + proxCount = 0; +} + +SegmentTermPositions::~SegmentTermPositions() { +//Func - Destructor +//Pre - true +//Post - The intance has been closed + close(); +} + +TermDocs* SegmentTermPositions::__asTermDocs(){ + return (TermDocs*) this; +} +TermPositions* SegmentTermPositions::__asTermPositions(){ + return (TermPositions*) this; +} + +void SegmentTermPositions::seek(const TermInfo* ti) { + SegmentTermDocs::seek(ti); + if (ti != NULL) + proxStream->seek(ti->proxPointer); + proxCount = 0; +} + +void SegmentTermPositions::close() { +//Func - Frees the resources +//Pre - true +//Post - The resources have been freed + + SegmentTermDocs::close(); + //Check if proxStream still exists + if(proxStream){ + proxStream->close(); + _CLDELETE( proxStream ); + } +} + +int32_t SegmentTermPositions::nextPosition() { + /* DSR:CL_BUG: Should raise exception if proxCount == 0 at the + ** beginning of this method, as in + ** if (--proxCount == 0) throw ...; + ** The JavaDocs for TermPositions.nextPosition declare this constraint, + ** but CLucene doesn't enforce it. */ + proxCount--; + return position += proxStream->readVInt(); +} + +bool SegmentTermPositions::next() { + for (int32_t f = proxCount; f > 0; f--) // skip unread positions + proxStream->readVInt(); + + if (SegmentTermDocs::next()) { // run super + proxCount = _freq; // note frequency + position = 0; // reset position + return true; + } + return false; +} + +int32_t SegmentTermPositions::read(int32_t* docs, int32_t* freqs, int32_t length) { + _CLTHROWA(CL_ERR_InvalidState,"TermPositions does not support processing multiple documents in one call. Use TermDocs instead."); +} + +void SegmentTermPositions::skippingDoc() { + for (int32_t f = _freq; f > 0; f--) // skip all positions + proxStream->readVInt(); +} + +void SegmentTermPositions::skipProx(int64_t proxPointer){ + proxStream->seek(proxPointer); + proxCount = 0; +} + +CL_NS_END diff --git a/src/assistant/3rdparty/clucene/src/CLucene/index/SegmentTermVector.cpp b/src/assistant/3rdparty/clucene/src/CLucene/index/SegmentTermVector.cpp new file mode 100644 index 000000000..5e9ac3c3b --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/index/SegmentTermVector.cpp @@ -0,0 +1,188 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#include "CLucene/StdHeader.h" +#include "TermVector.h" +#include "CLucene/util/StringBuffer.h" + +CL_NS_USE(util) +CL_NS_DEF(index) + +Array<int32_t> SegmentTermPositionVector::EMPTY_TERM_POS; + +SegmentTermVector::SegmentTermVector(const TCHAR* field, TCHAR** terms, Array<int32_t>* termFreqs) { + this->field = STRDUP_TtoT(field); + this->terms = terms; + this->termsLen = -1; //lazily get the size of the terms + this->termFreqs = termFreqs; +} + +SegmentTermVector::~SegmentTermVector(){ + _CLDELETE_CARRAY(field); + _CLDELETE_CARRAY_ALL(terms); + + _CLDELETE_ARRAY(termFreqs->values); + _CLDELETE(termFreqs); +} +TermPositionVector* SegmentTermVector::__asTermPositionVector(){ + return NULL; +} + +const TCHAR* SegmentTermVector::getField() { +return field; +} + +TCHAR* SegmentTermVector::toString() const{ +StringBuffer sb; +sb.appendChar('{'); +sb.append(field); +sb.append(_T(": ")); + +int32_t i=0; +while ( terms && terms[i] != NULL ){ + if (i>0) + sb.append(_T(", ")); + sb.append(terms[i]); + sb.appendChar('/'); + + sb.appendInt((*termFreqs)[i]); +} +sb.appendChar('}'); +return sb.toString(); +} + +int32_t SegmentTermVector::size() { +if ( terms == NULL ) + return 0; + +if ( termsLen == -1 ){ + termsLen=0; + while ( terms[termsLen] != 0 ) + termsLen++; +} +return termsLen; +} + +const TCHAR** SegmentTermVector::getTerms() { + return (const TCHAR**)terms; +} + +const Array<int32_t>* SegmentTermVector::getTermFrequencies() { + return termFreqs; +} + +int32_t SegmentTermVector::binarySearch(TCHAR** a, const int32_t arraylen, const TCHAR* key) const +{ + int32_t low = 0; + int32_t hi = arraylen - 1; + int32_t mid = 0; + while (low <= hi) + { + mid = (low + hi) >> 1; + + int32_t c = _tcscmp(a[mid],key); + if (c==0) + return mid; + else if (c > 0) + hi = mid - 1; + else // This gets the insertion point right on the last loop. + low = ++mid; + } + return -mid - 1; +} + +int32_t SegmentTermVector::indexOf(const TCHAR* termText) { + if(terms == NULL) + return -1; + int32_t res = binarySearch(terms, size(), termText); + return res >= 0 ? res : -1; +} + +void SegmentTermVector::indexesOf(const TCHAR** termNumbers, const int32_t start, const int32_t len, Array<int32_t>& ret) { + // TODO: there must be a more efficient way of doing this. + // At least, we could advance the lower bound of the terms array + // as we find valid indexes. Also, it might be possible to leverage + // this even more by starting in the middle of the termNumbers array + // and thus dividing the terms array maybe in half with each found index. + ret.length = len; + ret.values = _CL_NEWARRAY(int32_t,len); + for (int32_t i=0; i<len; ++i) { + ret.values[i] = indexOf(termNumbers[start+ i]); + } +} + + + + +SegmentTermPositionVector::SegmentTermPositionVector(const TCHAR* field, TCHAR** terms, Array<int32_t>* termFreqs, Array< Array<int32_t> >* positions, Array< Array<TermVectorOffsetInfo> >* offsets): + SegmentTermVector(field,terms,termFreqs) +{ + this->offsets = offsets; + this->positions = positions; +} + +SegmentTermPositionVector::~SegmentTermPositionVector(){ + if ( offsets ){ + for (size_t i=0;i<offsets->length;i++){ + if ( offsets->values != NULL ){ + Array<TermVectorOffsetInfo>& offs = offsets->values[i]; + for ( size_t j=0;j<offs.length;j++ ){ + _CLDELETE_ARRAY(offs.values); + } + } + } + _CLDELETE_ARRAY(offsets->values); + _CLDELETE(offsets); + } + if ( positions ){ + for (size_t i=0;i<positions->length;i++){ + if ( positions->values != NULL ){ + Array<int32_t>& pos = positions->values[i]; + for ( size_t j=0;j<pos.length;j++ ){ + _CLDELETE_ARRAY(pos.values); + } + } + } + _CLDELETE_ARRAY(positions->values); + _CLDELETE(positions); + } +} + +TermPositionVector* SegmentTermPositionVector::__asTermPositionVector(){ + return this; +} +/** +* Returns an array of TermVectorOffsetInfo in which the term is found. +* +* @param index The position in the array to get the offsets from +* @return An array of TermVectorOffsetInfo objects or the empty list +* @see org.apache.lucene.analysis.Token +*/ +Array<TermVectorOffsetInfo>* SegmentTermPositionVector::getOffsets(int32_t index) { + if(offsets == NULL) + return NULL; + if (index >=0 && index < offsets->length) + return &offsets->values[index]; + else + return &TermVectorOffsetInfo::EMPTY_OFFSET_INFO; +} + +/** +* Returns an array of positions in which the term is found. +* Terms are identified by the index at which its number appears in the +* term String array obtained from the <code>indexOf</code> method. +*/ +Array<int32_t>* SegmentTermPositionVector::getTermPositions(int32_t index) { + if(positions == NULL) + return NULL; + + if (index >=0 && index < positions->length) + return &positions->values[index]; + else + return &EMPTY_TERM_POS; +} +CL_NS_END + diff --git a/src/assistant/3rdparty/clucene/src/CLucene/index/Term.cpp b/src/assistant/3rdparty/clucene/src/CLucene/index/Term.cpp new file mode 100644 index 000000000..5ff7bb264 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/index/Term.cpp @@ -0,0 +1,182 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +* +* Changes are Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies). +------------------------------------------------------------------------------*/ + +#include "CLucene/StdHeader.h" +#include "Term.h" +#include "CLucene/util/StringIntern.h" + +CL_NS_USE(util) +CL_NS_DEF(index) + +Term::Term() +{ + init(); +} + +Term::Term(const TCHAR* fld, const TCHAR* txt, bool internField) +{ + init(); + set(fld, txt, internField); +} + +Term::Term(const Term* fieldTerm, const TCHAR* txt) +{ + init(); + set(fieldTerm, txt); +} + +Term::Term(const TCHAR* fld, const TCHAR* txt) +{ + init(); + set(fld, txt); +} + +Term::~Term() +{ + if (internF) + CLStringIntern::unintern(_field); + _field = NULL; + +#ifndef LUCENE_TERM_TEXT_LENGTH + if (_text != LUCENE_BLANK_STRING) + _CLDELETE_CARRAY(_text); +#endif +} + +const TCHAR* Term::field() const +{ + return _field; +} + +const TCHAR* Term::text() const +{ + return _text; +} + +void Term::set(const Term* term, const TCHAR* txt) +{ + set(term->field(), txt, false); +} + +void Term::set(const TCHAR* fld, const TCHAR* txt,bool internField) +{ + CND_PRECONDITION(fld != NULL, "fld contains NULL"); + CND_PRECONDITION(txt != NULL, "txt contains NULL"); + + //save field for unintern later + const TCHAR* oldField = _field; + cachedHashCode = 0; + textLen = _tcslen(txt); + + //Delete text if it is the owner +#ifdef LUCENE_TERM_TEXT_LENGTH + if (textLen > LUCENE_TERM_TEXT_LENGTH) + textLen = LUCENE_TERM_TEXT_LENGTH; + + _tcsncpy(_text,txt,textLen+1); + _text[textLen]=0; +#else + //if the term text buffer is bigger than what we have + if (_text && textLen > textLenBuf) { + if (_text != LUCENE_BLANK_STRING) { + _CLDELETE_ARRAY(_text); + } else { + _text = NULL; + } + textLenBuf = 0; + } + + if (_text == LUCENE_BLANK_STRING) { + _text = LUCENE_BLANK_STRING; + } else if (_text == NULL) { + if (txt[0] == 0) { + //if the string is blank and we aren't re-using the buffer... + _text = LUCENE_BLANK_STRING; + } else { + //duplicate the text + _text = stringDuplicate(txt); + textLenBuf = textLen; + } + } else { + //re-use the buffer + _tcscpy(_text,txt); + } +#endif + + //Set Term Field + if (internField) { + _field = CLStringIntern::intern(fld CL_FILELINE); + } else { + _field = fld; + } + + //unintern old field after interning new one, + if (internF) + CLStringIntern::unintern(oldField); + internF = internField; + + CND_PRECONDITION(_tcscmp(fld, _field) == 0, "field not equal"); +} + +bool Term::equals(const Term* other) const +{ + if (cachedHashCode != 0 && other->cachedHashCode != 0 + && other->cachedHashCode != cachedHashCode) + return false; + + if (_field == other->_field) { + if (textLen == other->textLen) + return (_tcscmp(_text, other->_text) == 0); + return false; + } + + return false; +} + +size_t Term::hashCode() +{ + if (cachedHashCode == 0) + cachedHashCode = Misc::thashCode(_field) + Misc::thashCode(_text, textLen); + + return cachedHashCode; +} + +int32_t Term::compareTo(const Term* other) const +{ + //Check ret to see if text needs to be compared + if (_field == other->_field) + return _tcscmp(_text, other->_text); + + int32_t ret = _tcscmp(_field, other->_field); + if (ret == 0) + ret = _tcscmp(_text, other->_text); + return ret; +} + +TCHAR* Term::toString() const +{ + return CL_NS(util)::Misc::join(_field, _T(":"), _text); +} + +void Term::init() +{ + textLen = 0; + internF = false; + cachedHashCode = 0; + _field = LUCENE_BLANK_STRING; + +#ifdef LUCENE_TERM_TEXT_LENGTH + _text[0] = 0; +#else + _text = LUCENE_BLANK_STRING; + textLenBuf = 0; +#endif +} + +CL_NS_END diff --git a/src/assistant/3rdparty/clucene/src/CLucene/index/Term.h b/src/assistant/3rdparty/clucene/src/CLucene/index/Term.h new file mode 100644 index 000000000..68eefd194 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/index/Term.h @@ -0,0 +1,146 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +* +* Changes are Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies). +------------------------------------------------------------------------------*/ +#ifndef _lucene_index_Term_ +#define _lucene_index_Term_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include "CLucene/util/Misc.h" +#include "CLucene/util/StringIntern.h" + +CL_NS_DEF(index) + +/* +A Term represents a word from text. This is the unit of search. It is +composed of two elements, the text of the word, as a string, and the name of +the field that the text occured in, an interned string. + +Note that terms may represent more than words from text fields, but also +things like dates, email addresses, urls, etc. + +IMPORTANT NOTE: +Term inherits from the template class LUCENE_REFBASE which tries to do +some garbage collection by counting the references an instance has. As a result +of this construction you MUST use _CLDECDELETE(obj) when you want to delete an +of Term! + +ABOUT intrn + +intrn indicates if field and text are interned or not. Interning of Strings +is the process of converting duplicated strings to shared ones. + +*/ +class Term : LUCENE_REFBASE +{ +private: + const TCHAR* _field; + bool internF; // Indicates if Term Field is interned(and therefore must be uninternd). + size_t cachedHashCode; + size_t textLen; // a cache of text len, this allows for a preliminary comparison of text lengths + +#ifdef LUCENE_TERM_TEXT_LENGTH + TCHAR _text[LUCENE_TERM_TEXT_LENGTH + 1]; +#else + TCHAR* _text; + size_t textLenBuf; //a cache of text len, this allows for a preliminary comparison of text lengths +#endif + + void init(); +public: + + //uses the specified fieldTerm's field. this saves on intern'ing time. + Term(const Term* fieldTerm, const TCHAR* txt); + + ///Constructs a blank term + Term(); + + // TODO: need to be private, a few other things need to be changed first... + Term(const TCHAR* fld, const TCHAR* txt, bool internField); + + /** + * Constructor. Constructs a Term with the given field and text. Field and + * text are not copied Field and text are deleted in destructor only if + * intern is false. + */ + Term(const TCHAR* fld, const TCHAR* txt); + + ///Destructor. + ~Term(); + + ///Returns the field of this term, an interned string. The field indicates + ///the part of a document which this term came from. + const TCHAR* field() const; ///<returns reference + + ///Returns the text of this term. In the case of words, this is simply the + ///text of the word. In the case of dates and other types, this is an + ///encoding of the object as a string. + const TCHAR* text() const; ///<returns reference + + ///Resets the field and text of a Term. + inline void set(const TCHAR* fld, const TCHAR* txt) + { + set(fld, txt, true); + } + + /** + * Optimized set of Term by reusing same field as this Term + * - avoids field.intern() overhead + * @param text The text of the new term + * (field is implicitly same as this Term instance) + */ + void set(const Term* term, const TCHAR* txt); + + void set(const TCHAR* fld, const TCHAR* txt, bool internField); + + /** Compares two terms, returning a negative integer if this + term belongs before the argument, zero if this term is equal to the + argument, and a positive integer if this term belongs after the argument. + + The ordering of terms is first by field, then by text.*/ + int32_t compareTo(const Term* other) const; + + bool equals(const Term* other) const; + + size_t textLength() const { return textLen; } + + ///Forms the contents of Field and term in some kind of tuple notation + ///<field:text> + TCHAR* toString() const; + + size_t hashCode(); + + class Equals:public CL_NS_STD(binary_function)<const Term*,const Term*,bool> + { + public: + bool operator()( const Term* val1, const Term* val2 ) const + { + return val1->equals(val2); + } + }; + + class Compare:LUCENE_BASE, public CL_NS(util)::Compare::_base //<Term*> + { + public: + bool operator()(Term* t1, Term* t2) const + { + return (t1->compareTo(t2) < 0); + } + + size_t operator()(Term* t) const + { + return t->hashCode(); + } + }; +}; + +CL_NS_END + +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/index/TermInfo.cpp b/src/assistant/3rdparty/clucene/src/CLucene/index/TermInfo.cpp new file mode 100644 index 000000000..ac1107317 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/index/TermInfo.cpp @@ -0,0 +1,53 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +* +* Changes are Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies). +------------------------------------------------------------------------------*/ + +#include "CLucene/StdHeader.h" +#include "TermInfo.h" + +CL_NS_DEF(index) + +TermInfo::TermInfo() +{ + set(0, 0, 0, 0); +} + +TermInfo::~TermInfo() +{ +} + +TermInfo::TermInfo(int32_t df, int64_t fp, int64_t pp) +{ + set(df, fp, pp, 0); +} + +TermInfo::TermInfo(const TermInfo* ti) +{ + if (ti) + set(ti); +} + +void TermInfo::set(const TermInfo* ti) +{ + if (ti) + set(ti->docFreq, ti->freqPointer, ti->proxPointer, ti->skipOffset); +} + +void TermInfo::set(int32_t df, int64_t fp, int64_t pp, int32_t so) +{ + CND_PRECONDITION(df >= 0, "df contains negative number"); + CND_PRECONDITION(fp >= 0, "fp contains negative number"); + CND_PRECONDITION(pp >= 0, "pp contains negative number"); + + docFreq = df; + freqPointer = fp; + proxPointer = pp; + skipOffset = so; +} + +CL_NS_END diff --git a/src/assistant/3rdparty/clucene/src/CLucene/index/TermInfo.h b/src/assistant/3rdparty/clucene/src/CLucene/index/TermInfo.h new file mode 100644 index 000000000..57b7a9a76 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/index/TermInfo.h @@ -0,0 +1,61 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +* +* Changes are Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies). +------------------------------------------------------------------------------*/ +#ifndef _lucene_index_TermInfo +#define _lucene_index_TermInfo + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +CL_NS_DEF(index) + +// A TermInfo is the record of information stored for a term. +class TermInfo : LUCENE_BASE +{ +public: + // The number of documents which contain the term. + int32_t docFreq; + + //A pointer into the TermFreqs file (.frq) + //The .frq file contains the lists of documents which contain each term, + //along with the frequency of the term in that document. + int64_t freqPointer; + + //A pointer into the TermPosition file (.prx). + //The .prx file contains the lists of positions that each term + //occurs at within documents. + int64_t proxPointer; + + int32_t skipOffset; + + //Constructor + TermInfo(); + + //Constructor + TermInfo(int32_t df, int64_t fp, int64_t pp); + + //Constructor + //Initialises this instance by copying the values of another TermInfo ti + TermInfo(const TermInfo* ti); + + //Destructor + ~TermInfo(); + + //Sets a new document frequency, a new freqPointer and a new proxPointer + void set(int32_t docFreq, int64_t freqPointer, int64_t proxPointer, + int32_t skipOffset); + + //Sets a new document frequency, a new freqPointer and a new proxPointer + //by copying these values from another instance of TermInfo + void set(const TermInfo* ti); +}; + +CL_NS_END + +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/index/TermInfosReader.cpp b/src/assistant/3rdparty/clucene/src/CLucene/index/TermInfosReader.cpp new file mode 100644 index 000000000..8f9e43dec --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/index/TermInfosReader.cpp @@ -0,0 +1,443 @@ +/* + * Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team + * + * Distributable under the terms of either the Apache License (Version 2.0) or + * the GNU Lesser General Public License, as specified in the COPYING file. + * + * Changes are Copyright(C) 2007, 2008 by Nokia Corporation and/or its subsidiary(-ies), all rights reserved. +*/ +#include "CLucene/StdHeader.h" +#include "TermInfosReader.h" + +#include "CLucene/store/Directory.h" +#include "CLucene/util/Misc.h" +#include "FieldInfos.h" +#include "Term.h" +#include "Terms.h" +#include "TermInfo.h" +#include "TermInfosWriter.h" + +CL_NS_USE(store) +CL_NS_USE(util) +CL_NS_DEF(index) + +TermInfosReader::TermInfosReader(Directory* dir, const QString& seg, + FieldInfos* fis) + : directory(dir) + , fieldInfos (fis) +{ + //Func - Constructor. + // Reads the TermInfos file (.tis) and eventually the Term Info Index file (.tii) + //Pre - dir is a reference to a valid Directory + // Fis contains a valid reference to an FieldInfos instance + // seg != NULL and contains the name of the segment + //Post - An instance has been created and the index named seg has been read. (Remember + // a segment is nothing more then an independently readable index) + + CND_PRECONDITION(!seg.isEmpty(), "seg is NULL"); + + //Initialize the name of the segment + segment = seg; + //There are no indexTerms yet + indexTerms = NULL; + //So there are no indexInfos + indexInfos = NULL; + //So there are no indexPointers + indexPointers = NULL; + //Create a filname fo a Term Info File + QString tisFile = Misc::segmentname(segment, QLatin1String(".tis")); + QString tiiFile = Misc::segmentname(segment, QLatin1String(".tii")); + + //Create an SegmentTermEnum for storing all the terms read of the segment + origEnum = _CLNEW SegmentTermEnum( directory->openInput( tisFile ), fieldInfos, false); + indexEnum = _CLNEW SegmentTermEnum( directory->openInput( tiiFile ), fieldInfos, true); + + //Check if enumerator points to a valid instance + CND_CONDITION(origEnum != NULL, "No memory could be allocated for orig enumerator"); + CND_CONDITION(indexEnum != NULL, "No memory could be allocated for index enumerator"); + + //Get the size of the enumeration and store it in size + _size = origEnum->size; +} + +TermInfosReader::~TermInfosReader() +{ + //Func - Destructor + //Pre - true + //Post - The instance has been destroyed + + //Close the TermInfosReader to be absolutly sure that enumerator has been closed + //and the arrays indexTerms, indexPointers and indexInfos and their elements + //have been destroyed + close(); +} + +void TermInfosReader::close() +{ + //Func - Close the enumeration of TermInfos + //Pre - true + //Post - The _enumeration has been closed and the arrays + + //Check if indexTerms and indexInfos exist + if (indexTerms && indexInfos){ + //Iterate through arrays indexTerms and indexPointer to + //destroy their elements +#ifdef _DEBUG + for (int32_t i = 0; i < indexTermsLength; ++i) { + if (indexTerms[i].__cl_refcount != 1) { + CND_PRECONDITION(indexTerms[i].__cl_refcount == 1, + "TermInfosReader term was references more than internally"); + } + // _CLDECDELETE(indexTerms[i]); + //_CLDELETE(indexInfos[i]); + } +#endif + //Delete the arrays + _CLDELETE_ARRAY(indexTerms); + _CLDELETE_ARRAY(indexInfos); + } + + //Delete the arrays + _CLDELETE_ARRAY(indexPointers); + + if (origEnum != NULL) { + origEnum->close(); + + //Get a pointer to IndexInput used by the enumeration but + //instantiated in the constructor by directory.open( tisFile ) + IndexInput *is = origEnum->input; + + //Delete the enumuration enumerator + _CLDELETE(origEnum); + + //Delete the IndexInput + _CLDELETE(is); + } + + if (indexEnum != NULL){ + indexEnum->close(); + + //Get a pointer to IndexInput used by the enumeration but + //instantiated in the constructor by directory.open( tiiFile ) + IndexInput *is = indexEnum->input; + + //Delete the enumuration enumerator + _CLDELETE(indexEnum); + + //Delete the IndexInput + _CLDELETE(is); + } +} + +int64_t TermInfosReader::size() const +{ + //Func - Return the size of the enumeration of TermInfos + //Pre - true + //Post - size has been returened + + return _size; +} + +Term* TermInfosReader::get(const int32_t position) +{ + //Func - Returns the nth term in the set + //Pre - position > = 0 + //Post - The n-th term in the set has been returned + + //Check if the size is 0 because then there are no terms + if (_size == 0) + return NULL; + + SegmentTermEnum* enumerator = getEnum(); + + if (enumerator != NULL //an enumeration exists + && enumerator->term(false) != NULL // term is at or past current + && position >= enumerator->position + && position < (enumerator->position + enumerator->indexInterval)) { + return scanEnum(position); // can avoid seek + } + + //random-access: must seek + seekEnum(position / enumerator->indexInterval); + + //Get the Term at position + return scanEnum(position); +} + +// TODO: currently there is no way of cleaning up a thread, if the thread ends. +// we are stuck with the terminfosreader of that thread. Hopefully this won't +// be too big a problem... solutions anyone? +SegmentTermEnum* TermInfosReader::getEnum() +{ + SegmentTermEnum* termEnum = enumerators.get(); + if (termEnum == NULL) { + termEnum = terms(); + enumerators.set(termEnum); + } + return termEnum; +} + +TermInfo* TermInfosReader::get(const Term* term) +{ + //Func - Returns a TermInfo for a term + //Pre - term holds a valid reference to term + //Post - if term can be found its TermInfo has been returned otherwise NULL + + //If the size of the enumeration is 0 then no Terms have been read + if (_size == 0) + return NULL; + + ensureIndexIsRead(); + + // optimize sequential access: first try scanning cached enum w/o seeking + SegmentTermEnum* enumerator = getEnum(); + + // optimize sequential access: first try scanning cached enumerator w/o seeking + // if the current term of the enumeration enumerator is not at the end + if (enumerator->term(false) != NULL + // AND there exists a previous current called prev and term is + // positioned after this prev + && ((enumerator->prev != NULL && term->compareTo(enumerator->prev) > 0) + // OR term is positioned at the same position as the current of + // enumerator or at a higher position + || term->compareTo(enumerator->term(false)) >= 0)) { + //Calculate the offset for the position + int32_t _enumOffset = (int32_t) + (enumerator->position / enumerator->indexInterval) + 1; + + // but before end of block the length of indexTerms (the number of + // terms in enumerator) equals _enum_offset + if (indexTermsLength == _enumOffset + // OR term is positioned in front of term found at _enumOffset in + // indexTerms + || term->compareTo(&indexTerms[_enumOffset]) < 0) { + //no need to seek, retrieve the TermInfo for term + return scanEnum(term); + } + } + + //Reposition current term in the enumeration + seekEnum(getIndexOffset(term)); + //Return the TermInfo for term + return scanEnum(term); +} + +int64_t TermInfosReader::getPosition(const Term* term) +{ + //Func - Returns the position of a Term in the set + //Pre - term holds a valid reference to a Term + // enumerator != NULL + //Post - If term was found then its position is returned otherwise -1 + + //if the enumeration is empty then return -1 + if (_size == 0) + return -1; + + ensureIndexIsRead(); + + //Retrieve the indexOffset for term + int32_t indexOffset = getIndexOffset(term); + seekEnum(indexOffset); + + SegmentTermEnum* enumerator = getEnum(); + + while(term->compareTo(enumerator->term(false)) > 0 && enumerator->next()) {} + + if (term->equals(enumerator->term(false))) + return enumerator->position; + + return -1; +} + +SegmentTermEnum* TermInfosReader::terms(const Term* term) +{ + //Func - Returns an enumeration of terms starting at or after the named term. + // If term is null then enumerator is set to the beginning + //Pre - term holds a valid reference to a Term + // enumerator != NULL + //Post - An enumeration of terms starting at or after the named term has been returned + + SegmentTermEnum* enumerator = NULL; + if (term != NULL) { + //Seek enumerator to term; delete the new TermInfo that's returned. + TermInfo* ti = get(term); + _CLDELETE(ti); + enumerator = getEnum(); + } else { + enumerator = origEnum; + } + //Clone the entire enumeration + SegmentTermEnum* cln = enumerator->clone(); + + //Check if cln points to a valid instance + CND_CONDITION(cln != NULL, "cln is NULL"); + + return cln; +} + +void TermInfosReader::ensureIndexIsRead() +{ + //Func - Reads the term info index file or .tti file. + // This file contains every IndexInterval-th entry from the .tis file, + // along with its location in the "tis" file. This is designed to be + // read entirely into memory and used to provide random access to the + // "tis" file. + //Pre - indexTerms = NULL + // indexInfos = NULL + // indexPointers = NULL + //Post - The term info index file has been read into memory + + SCOPED_LOCK_MUTEX(THIS_LOCK) + + if ( indexTerms != NULL ) + return; + + try { + indexTermsLength = (size_t)indexEnum->size; + + // Instantiate an block of Term's,so that each one doesn't have to be new'd + indexTerms = _CL_NEWARRAY(Term,indexTermsLength); + + // Check if is indexTerms is a valid array + CND_CONDITION(indexTerms != NULL, + "No memory could be allocated for indexTerms"); + + // Instantiate an big block of TermInfo's, so that each one doesn't + // have to be new'd + indexInfos = _CL_NEWARRAY(TermInfo,indexTermsLength); + + // Check if is indexInfos is a valid array + CND_CONDITION(indexInfos != NULL, + "No memory could be allocated for indexInfos"); + + // Instantiate an array indexPointers that contains pointers to the + // term info index file + indexPointers = _CL_NEWARRAY(int64_t,indexTermsLength); + + // Check if is indexPointers is a valid array + CND_CONDITION(indexPointers != NULL, + "No memory could be allocated for indexPointers"); + + //Iterate through the terms of indexEnum + for (int32_t i = 0; indexEnum->next(); ++i) { + indexTerms[i].set(indexEnum->term(false), indexEnum->term(false)->text()); + indexEnum->getTermInfo(&indexInfos[i]); + indexPointers[i] = indexEnum->indexPointer; + } + } _CLFINALLY ( + indexEnum->close(); + // Close and delete the IndexInput is. The close is done by the destructor. + _CLDELETE( indexEnum->input ); + _CLDELETE( indexEnum ); + ); +} + +int32_t TermInfosReader::getIndexOffset(const Term* term) +{ + //Func - Returns the offset of the greatest index entry which is less than + // or equal to term. + //Pre - term holds a reference to a valid term + // indexTerms != NULL + //Post - The new offset has been returned + + //Check if is indexTerms is a valid array + CND_PRECONDITION(indexTerms != NULL, "indexTerms is NULL"); + + int32_t lo = 0; + int32_t hi = indexTermsLength - 1; + int32_t mid; + int32_t delta; + + while (hi >= lo) { + //Start in the middle betwee hi and lo + mid = (lo + hi) >> 1; + + //Check if is indexTerms[mid] is a valid instance of Term + CND_PRECONDITION(&indexTerms[mid] != NULL, "indexTerms[mid] is NULL"); + CND_PRECONDITION(mid < indexTermsLength, "mid >= indexTermsLength"); + + //Determine if term is before mid or after mid + delta = term->compareTo(&indexTerms[mid]); + if (delta < 0) { + //Calculate the new hi + hi = mid - 1; + } else if (delta > 0) { + //Calculate the new lo + lo = mid + 1; + } else { + //term has been found so return its position + return mid; + } + } + // the new starting offset + return hi; +} + +void TermInfosReader::seekEnum(const int32_t indexOffset) +{ + //Func - Reposition the current Term and TermInfo to indexOffset + //Pre - indexOffset >= 0 + // indexTerms != NULL + // indexInfos != NULL + // indexPointers != NULL + //Post - The current Term and Terminfo have been repositioned to indexOffset + + CND_PRECONDITION(indexOffset >= 0, "indexOffset contains a negative number"); + CND_PRECONDITION(indexTerms != NULL, "indexTerms is NULL"); + CND_PRECONDITION(indexInfos != NULL, "indexInfos is NULL"); + CND_PRECONDITION(indexPointers != NULL, "indexPointers is NULL"); + + SegmentTermEnum* enumerator = getEnum(); + enumerator->seek(indexPointers[indexOffset], + (indexOffset * enumerator->indexInterval) - 1, + &indexTerms[indexOffset], &indexInfos[indexOffset]); +} + +TermInfo* TermInfosReader::scanEnum(const Term* term) +{ + //Func - Scans the Enumeration of terms for term and returns the + // corresponding TermInfo instance if found. The search is started + // from the current term. + //Pre - term contains a valid reference to a Term + // enumerator != NULL + //Post - if term has been found the corresponding TermInfo has been returned + // otherwise NULL has been returned + + SegmentTermEnum* enumerator = getEnum(); + enumerator->scanTo(term); + + //Check if the at the position the Term term can be found + if (enumerator->term(false) != NULL && term->equals(enumerator->term(false))) { + //Return the TermInfo instance about term + return enumerator->getTermInfo(); + } + + //term was not found so no TermInfo can be returned + return NULL; +} + +Term* TermInfosReader::scanEnum(const int32_t position) +{ + //Func - Scans the enumeration to the requested position and returns the + // Term located at that position + //Pre - position > = 0 + // enumerator != NULL + //Post - The Term at the requested position has been returned + + SegmentTermEnum* enumerator = getEnum(); + + // As long the position of the enumeration enumerator is smaller than the + // requested one + while(enumerator->position < position) { + //Move the current of enumerator to the next + if (!enumerator->next()) { + //If there is no next it means that the requested position was to big + return NULL; + } + } + + //Return the Term a the requested position + return enumerator->term(); +} + +CL_NS_END diff --git a/src/assistant/3rdparty/clucene/src/CLucene/index/TermInfosReader.h b/src/assistant/3rdparty/clucene/src/CLucene/index/TermInfosReader.h new file mode 100644 index 000000000..ed202e750 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/index/TermInfosReader.h @@ -0,0 +1,106 @@ +/* + * Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team + * + * Distributable under the terms of either the Apache License (Version 2.0) or + * the GNU Lesser General Public License, as specified in the COPYING file. + * + * Changes are Copyright(C) 2007, 2008 by Nokia Corporation and/or its subsidiary(-ies), all rights reserved. +*/ +#ifndef _lucene_index_TermInfosReader_ +#define _lucene_index_TermInfosReader_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include <QtCore/QString> + +#include "CLucene/store/Directory.h" +#include "CLucene/util/ThreadLocal.h" +#include "SegmentTermEnum.h" + +CL_NS_DEF(index) + +class FieldInfos; +class Term; +class TermInfo; +class TermInfos; +class TermInfosWriter; + +// PORT STATUS: 365707 (jlucene 1.9) +// This stores a monotonically increasing set of <Term, TermInfo> pairs in a +// Directory. Pairs are accessed either by Term or by ordinal position the set. +class TermInfosReader : LUCENE_BASE +{ +private: + CL_NS(store)::Directory* directory; + QString segment; + FieldInfos* fieldInfos; + + CL_NS(util)::ThreadLocal<SegmentTermEnum*, + CL_NS(util)::Deletor::Object<SegmentTermEnum> > enumerators; + + SegmentTermEnum* getEnum(); + SegmentTermEnum* origEnum; + SegmentTermEnum* indexEnum; + int64_t _size; + + Term* indexTerms; + int32_t indexTermsLength; + TermInfo* indexInfos; + int64_t* indexPointers; + + DEFINE_MUTEX(THIS_LOCK) + +public: + // Reads the TermInfos file(.tis) and eventually the Term Info Index(.tii) + TermInfosReader(CL_NS(store)::Directory* dir, const QString& segment, + FieldInfos* fis); + ~TermInfosReader(); + + //Close the enumeration of TermInfos + void close(); + + //Return the size of the enumeration of TermInfos + int64_t size() const; + + int32_t getSkipInterval() { + return origEnum->skipInterval; } + + // Returns an enumeration of terms starting at or after the named term. + // If no term is specified, an enumeration of all the Terms + // and TermInfos in the set is returned. + SegmentTermEnum* terms(const Term* term = NULL); + + // Returns the TermInfo for a Term in the set + // synchronized + TermInfo* get(const Term* term); + +private: + // Reads the term info index file or .tti file. + void ensureIndexIsRead(); + + // Returns the offset of the greatest index entry which is less than term. + int32_t getIndexOffset(const Term* term); + + // Reposition the current Term and TermInfo to indexOffset + void seekEnum(const int32_t indexOffset); + + // Scans the Enumeration of terms for term and returns the corresponding + // TermInfo instance if found. The search is started from the current term. + TermInfo* scanEnum(const Term* term); + + // Scans the enumeration to the requested position and returns the Term + // located at that position + Term* scanEnum(const int32_t position); + + // Returns the position of a Term in the set. synchronized + int64_t getPosition(const Term* term); + + // Returns the nth term in the set. synchronized + Term* get(const int32_t position); +}; + +CL_NS_END + +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/index/TermInfosWriter.cpp b/src/assistant/3rdparty/clucene/src/CLucene/index/TermInfosWriter.cpp new file mode 100644 index 000000000..c5b5340c3 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/index/TermInfosWriter.cpp @@ -0,0 +1,185 @@ +/* + * Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team + * + * Distributable under the terms of either the Apache License (Version 2.0) or + * the GNU Lesser General Public License, as specified in the COPYING file. + * + * Changes are Copyright(C) 2007, 2008 by Nokia Corporation and/or its subsidiary(-ies), all rights reserved. +*/ +#include "CLucene/StdHeader.h" +#include "TermInfosWriter.h" + +#include "CLucene/store/Directory.h" +#include "CLucene/util/Misc.h" +#include "FieldInfos.h" +#include "Term.h" +#include "TermInfo.h" +#include "IndexWriter.h" + +CL_NS_USE(util) +CL_NS_USE(store) +CL_NS_DEF(index) + +TermInfosWriter::TermInfosWriter(Directory* directory, const QString& segment, + FieldInfos* fis, int32_t interval) + : fieldInfos(fis) +{ + //Func - Constructor + //Pre - directory contains a valid reference to a Directory + // segment != NULL + // fis contains a valid reference to a reference FieldInfos + //Post - The instance has been created + + CND_PRECONDITION(!segment.isEmpty(), "segment is NULL"); + //Initialize instance + initialise(directory, segment, interval, false); + + other = _CLNEW TermInfosWriter(directory, segment, fieldInfos, interval, true); + + CND_CONDITION(other != NULL, "other is NULL"); + + other->other = this; +} + +TermInfosWriter::TermInfosWriter(Directory* directory, const QString& segment, + FieldInfos* fis, int32_t interval, bool isIndex) + : fieldInfos(fis) +{ + //Func - Constructor + //Pre - directory contains a valid reference to a Directory + // segment != NULL + // fis contains a valid reference to a reference FieldInfos + // isIndex is true or false + //Post - The instance has been created + + CND_PRECONDITION(!segment.isEmpty(), "segment is NULL"); + initialise(directory, segment, interval, isIndex); +} + +void TermInfosWriter::initialise(Directory* directory, const QString& segment, + int32_t interval, bool IsIndex) +{ + //Func - Helps constructors to initialize Instance + //Pre - directory contains a valid reference to a Directory + // segment != NULL + // fis contains a valid reference to a reference FieldInfos + //Post - The instance has been initialized + + lastTerm = _CLNEW Term; + + CND_CONDITION(lastTerm != NULL, "Could not allocate memory for lastTerm"); + + lastTi = _CLNEW TermInfo(); + + CND_CONDITION(lastTi != NULL, "Could not allocate memory for lastTi"); + + lastIndexPointer = 0; + size = 0; + isIndex = IsIndex; + indexInterval = interval; + skipInterval = LUCENE_DEFAULT_TERMDOCS_SKIP_INTERVAL; + + QString buf = Misc::segmentname(segment, QLatin1String(isIndex ? ".tii" : ".tis")); + output = directory->createOutput(buf); + + output->writeInt(FORMAT); // write format + output->writeLong(0); // leave space for size + output->writeInt(indexInterval);// write indexInterval + output->writeInt(skipInterval); // write skipInterval + + //Set other to NULL by Default + other = NULL; +} + +TermInfosWriter::~TermInfosWriter() +{ + //Func - Destructor + //Pre - true + //Post - de instance has been destroyed + + close(); +} + +void TermInfosWriter::add(Term* term, const TermInfo* ti) +{ + //Func - Writes a Term and TermInfo to the outputstream + //Pre - Term must be lexicographically greater than all previous Terms added. + // Pointers of TermInfo ti (freqPointer and proxPointer) must be + // positive and greater than all previous. + + CND_PRECONDITION(isIndex || (!isIndex && term->compareTo(lastTerm) > 0),"term out of order"); + CND_PRECONDITION(ti->freqPointer >= lastTi->freqPointer,"freqPointer out of order"); + CND_PRECONDITION(ti->proxPointer >= lastTi->proxPointer,"proxPointer out of order"); + + if (!isIndex && size % indexInterval == 0) { + //add an index term + other->add(lastTerm, lastTi); + } + + //write term + writeTerm(term); + // write doc freq + output->writeVInt(ti->docFreq); + //write pointers + output->writeVLong(ti->freqPointer - lastTi->freqPointer); + output->writeVLong(ti->proxPointer - lastTi->proxPointer); + if (ti->docFreq >= skipInterval) { + output->writeVInt(ti->skipOffset); + } + + if (isIndex){ + output->writeVLong(other->output->getFilePointer() - lastIndexPointer); + lastIndexPointer = other->output->getFilePointer(); // write pointer + } + + lastTi->set(ti); + size++; +} + +void TermInfosWriter::close() { + //Func - Closes the TermInfosWriter + //Pre - true + //Post - The TermInfosWriter has been closed + + if (output){ + //write size at start + output->seek(4); // write size after format + output->writeLong(size); + output->close(); + _CLDELETE(output); + + if (!isIndex){ + if(other){ + other->close(); + _CLDELETE( other ); + } + } + _CLDECDELETE(lastTerm); + + _CLDELETE(lastTi); + } +} + +void TermInfosWriter::writeTerm(Term* term) +{ + int32_t start = Misc::stringDifference(lastTerm->text(),lastTerm->textLength(), + term->text(),term->textLength()); + int32_t length = term->textLength() - start; + + output->writeVInt(start); // write shared prefix length + output->writeVInt(length); // write delta length + output->writeChars(term->text(), start, length); // write delta chars + + int32_t fieldnum = fieldInfos->fieldNumber(term->field()); + CND_PRECONDITION(fieldnum>=-1&&fieldnum<fieldInfos->size(),"Fieldnum is out of range"); + output->writeVInt(fieldnum); // write field num + + if ( lastTerm->__cl_refcount == 1 ){ + lastTerm->set(term,term->text()); + }else{ + _CLDECDELETE(lastTerm); + lastTerm = _CL_POINTER(term); + } +} + +CL_NS_END diff --git a/src/assistant/3rdparty/clucene/src/CLucene/index/TermInfosWriter.h b/src/assistant/3rdparty/clucene/src/CLucene/index/TermInfosWriter.h new file mode 100644 index 000000000..7e3c68699 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/index/TermInfosWriter.h @@ -0,0 +1,89 @@ +/* + * Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team + * + * Distributable under the terms of either the Apache License (Version 2.0) or + * the GNU Lesser General Public License, as specified in the COPYING file. + * + * Changes are Copyright(C) 2007, 2008 by Nokia Corporation and/or its subsidiary(-ies), all rights reserved. +*/ +#ifndef _lucene_index_TermInfosWriter_ +#define _lucene_index_TermInfosWriter_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include <QtCore/QString> + +#include "CLucene/store/Directory.h" +#include "FieldInfos.h" +#include "TermInfo.h" +#include "Term.h" + +CL_NS_DEF(index) + + +// This stores a monotonically increasing set of <Term, TermInfo> pairs in a +// Directory. A TermInfos can be written once, in order. +class TermInfosWriter : LUCENE_BASE +{ +private: + FieldInfos* fieldInfos; + CL_NS(store)::IndexOutput* output; + Term* lastTerm; + TermInfo* lastTi; + int64_t size; + int64_t lastIndexPointer; + bool isIndex; + TermInfosWriter* other; + + //inititalize + TermInfosWriter(CL_NS(store)::Directory* directory, + const QString& segment, FieldInfos* fis, int32_t interval, bool isIndex); +public: + /** The file format version, a negative number. */ + LUCENE_STATIC_CONSTANT(int32_t,FORMAT=-2); + + /** + * Expert: The fraction of terms in the "dictionary" which should be stored + * in RAM. Smaller values use more memory, but make searching slightly + * faster, while larger values use less memory and make searching slightly + * slower. Searching is typically not dominated by dictionary lookup, so + * tweaking this is rarely useful. + */ + int32_t indexInterval;// = 128 + + /** + * Expert: The fraction of {@link TermDocs} entries stored in skip tables, + * used to accellerate {@link TermDocs#SkipTo(int32_t)}. Larger values result in + * smaller indexes, greater acceleration, but fewer accelerable cases, while + * smaller values result in bigger indexes, less acceleration and more + * accelerable cases. More detailed experiments would be useful here. + */ + int32_t skipInterval;// = 16 + + TermInfosWriter(CL_NS(store)::Directory* directory, + const QString& segment, FieldInfos* fis, int32_t interval); + + ~TermInfosWriter(); + + /** + * Adds a new <Term, TermInfo> pair to the set. + * Term must be lexicographically greater than all previous Terms added. + * TermInfo pointers must be positive and greater than all previous. + */ + void add(Term* term, const TermInfo* ti); + + /** Called to complete TermInfos creation. */ + void close(); + +private: + /** Helps constructors to initialize instances */ + void initialise(CL_NS(store)::Directory* directory, + const QString& segment, int32_t interval, bool IsIndex); + void writeTerm(Term* term); +}; + +CL_NS_END + +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/index/TermVector.h b/src/assistant/3rdparty/clucene/src/CLucene/index/TermVector.h new file mode 100644 index 000000000..8601fbf53 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/index/TermVector.h @@ -0,0 +1,418 @@ +/* + * Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team + * + * Distributable under the terms of either the Apache License (Version 2.0) or + * the GNU Lesser General Public License, as specified in the COPYING file. + * + * Changes are Copyright(C) 2007, 2008 by Nokia Corporation and/or its subsidiary(-ies), all rights reserved. +*/ +#ifndef _lucene_index_termvector_h +#define _lucene_index_termvector_h + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include <QtCore/QString> + +#include "CLucene/store/Directory.h" +#include "CLucene/store/IndexOutput.h" +#include "FieldInfos.h" + +CL_NS_DEF(index) + +struct TermVectorOffsetInfo; +class TermPositionVector; + +// Provides access to stored term vector of a document field. +class TermFreqVector : LUCENE_BASE +{ +public: + virtual ~TermFreqVector() {} + + // @return The field this vector is associated with. + virtual const TCHAR* getField() = 0; + + // @return The number of terms in the term vector. + virtual int32_t size() = 0; + + // @return An Array of term texts in ascending order. + virtual const TCHAR** getTerms() = 0; + + + /* Array of term frequencies. Locations of the array correspond one to one + * to the terms in the array obtained from <code>getTerms</code> + * method. Each location in the array contains the number of times this + * term occurs in the document or the document field. + * + * The size of the returned array is size() + * @memory Returning a pointer to internal data. Do not delete. + */ + virtual const Array<int32_t>* getTermFrequencies() = 0; + + + /* Return an index in the term numbers array returned from + * <code>getTerms</code> at which the term with the specified + * <code>term</code> appears. If this term does not appear in the array, + * return -1. + */ + virtual int32_t indexOf(const TCHAR* term) = 0; + + + /* Just like <code>indexOf(int32_t)</code> but searches for a number of terms + * at the same time. Returns an array that has the same size as the number + * of terms searched for, each slot containing the result of searching for + * that term number. + * + * @param terms array containing terms to look for + * @param start index in the array where the list of terms starts + * @param len the number of terms in the list + */ + virtual void indexesOf(const TCHAR** terms, const int32_t start, + const int32_t len, Array<int32_t>& ret) = 0; + + // Solve the diamond inheritence problem by providing a reinterpret function. + // No dynamic casting is required and no RTTI data is needed to do this + virtual TermPositionVector* __asTermPositionVector() = 0; +}; + + +/** +* Writer works by opening a document and then opening the fields within the document and then +* writing out the vectors for each field. +* +* Rough usage: +* +<CODE> +for each document +{ +writer.openDocument(); +for each field on the document +{ +writer.openField(field); +for all of the terms +{ +writer.addTerm(...) +} +writer.closeField +} +writer.closeDocument() +} +</CODE> +*/ +class TermVectorsWriter : LUCENE_BASE +{ +private: + class TVField : LUCENE_BASE + { + public: + int32_t number; + int64_t tvfPointer; + int32_t length; // number of distinct term positions + bool storePositions; + bool storeOffsets; + + TVField(int32_t number, bool storePos, bool storeOff) + : tvfPointer(0) + , length(0) + { + this->number = number; + this->storePositions = storePos; + this->storeOffsets = storeOff; + } + ~TVField() {} + }; + + class TVTerm : LUCENE_BASE + { + const TCHAR* termText; + int32_t termTextLen; //textlen cache + + public: + TVTerm(); + ~TVTerm(); + + int32_t freq; + Array<int32_t>* positions; + Array<TermVectorOffsetInfo>* offsets; + + const TCHAR* getTermText() const; + size_t getTermTextLen(); + void setTermText(const TCHAR* val); + }; + + CL_NS(store)::IndexOutput* tvx, *tvd, *tvf; + CL_NS(util)::CLVector<TVField*,CL_NS(util)::Deletor::Object<TVField> > fields; + CL_NS(util)::CLVector<TVTerm*,CL_NS(util)::Deletor::Object<TVTerm> > terms; + FieldInfos* fieldInfos; + + TVField* currentField; + int64_t currentDocPointer; + + void addTermInternal(const TCHAR* termText, const int32_t freq, + Array<int32_t>* positions, Array<TermVectorOffsetInfo>* offsets); + + void writeField(); + void writeDoc(); + + void openField(int32_t fieldNumber, bool storePositionWithTermVector, + bool storeOffsetWithTermVector); + +public: + LUCENE_STATIC_CONSTANT(int32_t, FORMAT_VERSION = 2); + + // The size in bytes that the FORMAT_VERSION will take up at the beginning + // of each file + LUCENE_STATIC_CONSTANT(int32_t, FORMAT_SIZE = 4); + + LUCENE_STATIC_CONSTANT(uint8_t, STORE_POSITIONS_WITH_TERMVECTOR = 0x1); + LUCENE_STATIC_CONSTANT(uint8_t, STORE_OFFSET_WITH_TERMVECTOR = 0x2); + + static const QLatin1String LUCENE_TVX_EXTENSION; + static const QLatin1String LUCENE_TVD_EXTENSION; + static const QLatin1String LUCENE_TVF_EXTENSION; + + TermVectorsWriter(CL_NS(store)::Directory* directory, const QString& segment, + FieldInfos* fieldInfos); + + ~TermVectorsWriter(); + void openDocument(); + void closeDocument(); + + /** Close all streams. */ + void close(); + bool isDocumentOpen() const; + + /** Start processing a field. This can be followed by a number of calls to + * addTerm, and a final call to closeField to indicate the end of + * processing of this field. If a field was previously open, it is + * closed automatically. + */ + void openField(const TCHAR* field); + + /** Finished processing current field. This should be followed by a call to + * openField before future calls to addTerm. + */ + void closeField(); + + /** Return true if a field is currently open. */ + bool isFieldOpen() const; + + /** + * Add a complete document specified by all its term vectors. If document has no + * term vectors, add value for tvx. + * + * @param vectors + * @throws IOException + */ + void addAllDocVectors(Array<TermFreqVector*>& vectors); + + /** Add term to the field's term vector. Field must already be open. + * Terms should be added in + * increasing order of terms, one call per unique termNum. ProxPointer + * is a pointer into the TermPosition file (prx). Freq is the number of + * times this term appears in this field, in this document. + * @throws IllegalStateException if document or field is not open + */ + void addTerm(const TCHAR* termText, int32_t freq, + Array<int32_t>* positions = NULL, Array<TermVectorOffsetInfo>* offsets = NULL); +}; + +class SegmentTermVector : public virtual TermFreqVector +{ +private: + const TCHAR* field; + TCHAR** terms; + int32_t termsLen; //cache + Array<int32_t>* termFreqs; + + int32_t binarySearch(TCHAR** a, const int32_t arraylen, const TCHAR* key) const; +public: + //note: termFreqs must be the same length as terms + SegmentTermVector(const TCHAR* field, TCHAR** terms, Array<int32_t>* termFreqs); + virtual ~SegmentTermVector(); + + /** + * + * @return The number of the field this vector is associated with + */ + const TCHAR* getField(); + TCHAR* toString() const; + int32_t size(); + const TCHAR** getTerms(); + const Array<int32_t>* getTermFrequencies(); + int32_t indexOf(const TCHAR* termText); + void indexesOf(const TCHAR** termNumbers, const int32_t start, const int32_t len, Array<int32_t>& ret); + + virtual TermPositionVector* __asTermPositionVector(); +}; + +class TermVectorsReader : LUCENE_BASE +{ +private: + FieldInfos* fieldInfos; + + CL_NS(store)::IndexInput* tvx; + CL_NS(store)::IndexInput* tvd; + CL_NS(store)::IndexInput* tvf; + int64_t _size; + + int32_t tvdFormat; + int32_t tvfFormat; + + + int32_t checkValidFormat(CL_NS(store)::IndexInput* in); + + void readTermVectors(const TCHAR** fields, const int64_t* tvfPointers, + const int32_t len, Array<TermFreqVector*>& _return); + + /** + * + * @param field The field to read in + * @param tvfPointer The pointer within the tvf file where we should start reading + * @return The TermVector located at that position + * @throws IOException + */ + SegmentTermVector* readTermVector(const TCHAR* field, const int64_t tvfPointer); + + int64_t size(); + + + DEFINE_MUTEX(THIS_LOCK) + TermVectorsReader(const TermVectorsReader& copy); +public: + TermVectorsReader(CL_NS(store)::Directory* d, const QString& segment, + FieldInfos* fieldInfos); + ~TermVectorsReader(); + + void close(); + TermVectorsReader* clone() const; + + /** + * Retrieve the term vector for the given document and field + * @param docNum The document number to retrieve the vector for + * @param field The field within the document to retrieve + * @return The TermFreqVector for the document and field or null if there is no termVector for this field. + * @throws IOException if there is an error reading the term vector files + */ + TermFreqVector* get(const int32_t docNum, const TCHAR* field); + + + /** + * Return all term vectors stored for this document or null if the could not be read in. + * + * @param docNum The document number to retrieve the vector for + * @return All term frequency vectors + * @throws IOException if there is an error reading the term vector files + */ + bool get(int32_t docNum, Array<TermFreqVector*>& result); +}; + + +struct TermVectorOffsetInfo +{ + int startOffset; + int endOffset; + +public: + static Array<TermVectorOffsetInfo> EMPTY_OFFSET_INFO; + TermVectorOffsetInfo(); + ~TermVectorOffsetInfo(); + TermVectorOffsetInfo(int32_t startOffset, int32_t endOffset); + int32_t getEndOffset() const; + void setEndOffset(int32_t endOffset); + int32_t getStartOffset() const; + void setStartOffset(int32_t startOffset); + bool equals(TermVectorOffsetInfo* o); + size_t hashCode() const; +}; + + +/* Extends <code>TermFreqVector</code> to provide additional information about + * positions in which each of the terms is found. A TermPositionVector not + * necessarily contains both positions and offsets, but at least one of these + * arrays exists. +*/ +class TermPositionVector : public virtual TermFreqVector +{ +public: + + /** Returns an array of positions in which the term is found. + * Terms are identified by the index at which its number appears in the + * term String array obtained from the <code>indexOf</code> method. + * May return null if positions have not been stored. + */ + virtual Array<int32_t>* getTermPositions(int32_t index) = 0; + + /** + * Returns an array of TermVectorOffsetInfo in which the term is found. + * May return null if offsets have not been stored. + * + * @see org.apache.lucene.analysis.Token + * + * @param index The position in the array to get the offsets from + * @return An array of TermVectorOffsetInfo objects or the empty list + */ + virtual Array<TermVectorOffsetInfo>* getOffsets(int32_t index) = 0; + + virtual ~TermPositionVector(){ + } +}; + + +class SegmentTermPositionVector: public SegmentTermVector, public TermPositionVector +{ +protected: + Array< Array<int32_t> >* positions; + Array< Array<TermVectorOffsetInfo> >* offsets; + static Array<int32_t> EMPTY_TERM_POS; +public: + SegmentTermPositionVector(const TCHAR* field, TCHAR** terms, + Array<int32_t>* termFreqs, Array< Array<int32_t> >* positions, + Array< Array<TermVectorOffsetInfo> >* offsets); + ~SegmentTermPositionVector(); + + /** + * Returns an array of TermVectorOffsetInfo in which the term is found. + * + * @param index The position in the array to get the offsets from + * @return An array of TermVectorOffsetInfo objects or the empty list + * @see org.apache.lucene.analysis.Token + */ + Array<TermVectorOffsetInfo>* getOffsets(int32_t index); + + /** + * Returns an array of positions in which the term is found. + * Terms are identified by the index at which its number appears in the + * term String array obtained from the <code>indexOf</code> method. + */ + Array<int32_t>* getTermPositions(int32_t index); + + const TCHAR* getField() { + return SegmentTermVector::getField(); } + + TCHAR* toString() const { + return SegmentTermVector::toString(); } + + int32_t size() { + return SegmentTermVector::size(); } + + const TCHAR** getTerms() { + return SegmentTermVector::getTerms(); } + + const Array<int32_t>* getTermFrequencies() { + return SegmentTermVector::getTermFrequencies(); } + + int32_t indexOf(const TCHAR* termText) { + return SegmentTermVector::indexOf(termText); } + + void indexesOf(const TCHAR** termNumbers, const int32_t start, + const int32_t len, Array<int32_t>& ret) { + SegmentTermVector::indexesOf(termNumbers, start, len, ret); } + + virtual TermPositionVector* __asTermPositionVector(); +}; + +CL_NS_END + +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/index/TermVectorReader.cpp b/src/assistant/3rdparty/clucene/src/CLucene/index/TermVectorReader.cpp new file mode 100644 index 000000000..53d909b29 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/index/TermVectorReader.cpp @@ -0,0 +1,393 @@ +/* + * Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team + * + * Distributable under the terms of either the Apache License (Version 2.0) or + * the GNU Lesser General Public License, as specified in the COPYING file. + * + * Changes are Copyright(C) 2007, 2008 by Nokia Corporation and/or its subsidiary(-ies), all rights reserved. +*/ +#include "CLucene/StdHeader.h" +#include "TermVector.h" +#include "CLucene/util/StringBuffer.h" + +CL_NS_USE(util) +CL_NS_DEF(index) + +TermVectorsReader::TermVectorsReader(CL_NS(store)::Directory* d, + const QString& segment, FieldInfos* fieldInfos) +{ + if (d->fileExists(segment + TermVectorsWriter::LUCENE_TVX_EXTENSION)) { + tvx = d->openInput(segment + TermVectorsWriter::LUCENE_TVX_EXTENSION); + checkValidFormat(tvx); + + tvd = d->openInput(segment + TermVectorsWriter::LUCENE_TVD_EXTENSION); + tvdFormat = checkValidFormat(tvd); + + tvf = d->openInput(segment + TermVectorsWriter::LUCENE_TVF_EXTENSION); + tvfFormat = checkValidFormat(tvf); + + _size = tvx->length() / 8; + }else{ + tvx = NULL; + tvd = NULL; + tvf = NULL; + _size = 0; + } + + this->fieldInfos = fieldInfos; +} + +TermVectorsReader::TermVectorsReader(const TermVectorsReader& copy) +{ + tvx = copy.tvx->clone(); + tvd = copy.tvd->clone(); + tvf = copy.tvf->clone(); + + tvdFormat = copy.tvdFormat; + tvfFormat = copy.tvfFormat; + _size = copy._size; + fieldInfos = copy.fieldInfos; +} +TermVectorsReader* TermVectorsReader::clone() const{ + if (tvx == NULL || tvd == NULL || tvf == NULL) + return NULL; + return _CLNEW TermVectorsReader(*this); +} + +TermVectorsReader::~TermVectorsReader(){ + close(); +} + +void TermVectorsReader::close(){ + // why don't we trap the exception and at least make sure that + // all streams that we can close are closed? + CLuceneError keep(0,"",false); + bool thrown = false; + + if (tvx != NULL){ + try{ + tvx->close(); + }catch(CLuceneError& err){ + if ( err.number() == CL_ERR_IO ){ + keep = err; + thrown = true; + }else + throw err; + } + _CLDELETE(tvx);//delete even if error thrown + } + if (tvd != NULL){ + try{ + tvd->close(); + }catch(CLuceneError& err){ + if ( err.number() == CL_ERR_IO ){ + keep = err; + thrown = true; + }else + throw err; + } + _CLDELETE(tvd); + } + if (tvf != NULL){ + try{ + tvf->close(); + }catch(CLuceneError& err){ + if ( err.number() == CL_ERR_IO ){ + keep = err; + thrown = true; + }else + throw err; + } + _CLDELETE(tvf); + } + + if ( thrown ) + throw keep; +} + +TermFreqVector* TermVectorsReader::get(const int32_t docNum, const TCHAR* field){ + // Check if no term vectors are available for this segment at all + int32_t fieldNumber = fieldInfos->fieldNumber(field); + TermFreqVector* result = NULL; + if (tvx != NULL) { + //We need to account for the FORMAT_SIZE at when seeking in the tvx + //We don't need to do this in other seeks because we already have the + // file pointer + //that was written in another file + tvx->seek((docNum * 8L) + TermVectorsWriter::FORMAT_SIZE); + int64_t position = tvx->readLong(); + + tvd->seek(position); + int32_t fieldCount = tvd->readVInt(); + // There are only a few fields per document. We opt for a full scan + // rather then requiring that they be ordered. We need to read through + // all of the fields anyway to get to the tvf pointers. + int32_t number = 0; + int32_t found = -1; + for (int32_t i = 0; i < fieldCount; ++i) { + if(tvdFormat == TermVectorsWriter::FORMAT_VERSION) + number = tvd->readVInt(); + else + number += tvd->readVInt(); + if (number == fieldNumber) + found = i; + } + + // This field, although valid in the segment, was not found in this + // document + if (found != -1) { + // Compute position in the tvf file + position = 0; + for (int32_t i = 0; i <= found; ++i) + position += tvd->readVLong(); + result = readTermVector(field, position); + } + } + return result; +} + + +bool TermVectorsReader::get(int32_t docNum, Array<TermFreqVector*>& result){ + // Check if no term vectors are available for this segment at all + if (tvx != NULL) { + //We need to offset by + tvx->seek((docNum * 8L) + TermVectorsWriter::FORMAT_SIZE); + int64_t position = tvx->readLong(); + + tvd->seek(position); + int32_t fieldCount = tvd->readVInt(); + + // No fields are vectorized for this document + if (fieldCount != 0) { + int32_t number = 0; + const TCHAR** fields = _CL_NEWARRAY(const TCHAR*,fieldCount+1); + + { //msvc6 scope fix + for (int32_t i = 0; i < fieldCount; ++i) { + if(tvdFormat == TermVectorsWriter::FORMAT_VERSION) + number = tvd->readVInt(); + else + number += tvd->readVInt(); + fields[i] = fieldInfos->fieldName(number); + } + } + fields[fieldCount]=NULL; + + // Compute position in the tvf file + position = 0; + int64_t* tvfPointers = _CL_NEWARRAY(int64_t,fieldCount); + { //msvc6 scope fix + for (int32_t i = 0; i < fieldCount; ++i) { + position += tvd->readVLong(); + tvfPointers[i] = position; + } + } + + readTermVectors(fields, tvfPointers, fieldCount, result); + _CLDELETE_ARRAY(tvfPointers); + _CLDELETE_ARRAY(fields); + } + return true; + } + return false; +} + + +int32_t TermVectorsReader::checkValidFormat(CL_NS(store)::IndexInput* in) +{ + int32_t format = in->readInt(); + if (format > TermVectorsWriter::FORMAT_VERSION) + { + CL_NS(util)::StringBuffer err; + err.append(_T("Incompatible format version: ")); + err.appendInt(format); + err.append(_T(" expected ")); + err.appendInt(TermVectorsWriter::FORMAT_VERSION); + err.append(_T(" or less")); + _CLTHROWT(CL_ERR_Runtime,err.getBuffer()); + } + return format; +} + +void TermVectorsReader::readTermVectors(const TCHAR** fields, + const int64_t* tvfPointers, const int32_t len, Array<TermFreqVector*>& result) +{ + result.length = len; + result.values = _CL_NEWARRAY(TermFreqVector*,len); + for (int32_t i = 0; i < len; ++i) { + result.values[i] = readTermVector(fields[i], tvfPointers[i]); + } +} + +SegmentTermVector* TermVectorsReader::readTermVector(const TCHAR* field, + const int64_t tvfPointer) +{ + // Now read the data from specified position. We don't need to offset by + // the FORMAT here since the pointer already includes the offset + tvf->seek(tvfPointer); + + int32_t numTerms = tvf->readVInt(); + // If no terms - return a constant empty termvector. However, this should never occur! + if (numTerms == 0) + return _CLNEW SegmentTermVector(field, NULL, NULL); + + bool storePositions; + bool storeOffsets; + + if(tvfFormat == TermVectorsWriter::FORMAT_VERSION){ + uint8_t bits = tvf->readByte(); + storePositions = (bits & TermVectorsWriter::STORE_POSITIONS_WITH_TERMVECTOR) != 0; + storeOffsets = (bits & TermVectorsWriter::STORE_OFFSET_WITH_TERMVECTOR) != 0; + } + else{ + tvf->readVInt(); + storePositions = false; + storeOffsets = false; + } + + TCHAR** terms = _CL_NEWARRAY(TCHAR*,numTerms+1); + Array<int32_t>* termFreqs = _CLNEW Array<int32_t>(numTerms); + + // we may not need these, but declare them + Array< Array<int32_t> >* positions = NULL; + Array< Array<TermVectorOffsetInfo> >* offsets = NULL; + if(storePositions){ + Array<int32_t>* tmp = _CL_NEWARRAY(Array<int32_t>,numTerms); + positions = _CLNEW Array< Array<int32_t> >(tmp, numTerms); + } + if(storeOffsets){ + Array<TermVectorOffsetInfo>* tmp = _CL_NEWARRAY(Array<TermVectorOffsetInfo>,numTerms); + offsets = _CLNEW Array< Array<TermVectorOffsetInfo> >(tmp, numTerms); + } + + int32_t start = 0; + int32_t deltaLength = 0; + int32_t totalLength = 0; + int32_t bufferLen=10; // init the buffer with a length of 10 character + TCHAR* buffer = (TCHAR*)malloc(bufferLen * sizeof(TCHAR)); + + for (int32_t i = 0; i < numTerms; ++i) { + start = tvf->readVInt(); + deltaLength = tvf->readVInt(); + totalLength = start + deltaLength; + if (bufferLen < totalLength) // increase buffer + { + buffer=(TCHAR*)realloc(buffer,totalLength * sizeof(TCHAR)); + bufferLen = totalLength; + } + + //read the term + tvf->readChars(buffer, start, deltaLength); + terms[i] = _CL_NEWARRAY(TCHAR,totalLength+1); + _tcsncpy(terms[i],buffer,totalLength); + terms[i][totalLength] = '\0'; //null terminate term + + //read the frequency + int32_t freq = tvf->readVInt(); + termFreqs->values[i] = freq; + + if (storePositions) { //read in the positions + Array<int32_t>& pos = positions->values[i]; + pos.length = freq; + pos.values = _CL_NEWARRAY(int32_t,freq); + + int32_t prevPosition = 0; + for (int32_t j = 0; j < freq; ++j) + { + pos.values[j] = prevPosition + tvf->readVInt(); + prevPosition = pos.values[j]; + } + } + + if (storeOffsets) { + Array<TermVectorOffsetInfo>& offs = offsets->values[i]; + offs.length = freq; + offs.values = _CL_NEWARRAY(TermVectorOffsetInfo,freq); + + int32_t prevOffset = 0; + for (int32_t j = 0; j < freq; ++j) { + int32_t startOffset = prevOffset + tvf->readVInt(); + int32_t endOffset = startOffset + tvf->readVInt(); + offs.values[j].setStartOffset(startOffset); + offs.values[j].setEndOffset(endOffset); + prevOffset = endOffset; + } + } + } + free(buffer); + terms[numTerms]=NULL; //null terminate terms array + + SegmentTermVector* tv = NULL; + if (storePositions || storeOffsets){ + return _CLNEW SegmentTermPositionVector(field, terms, termFreqs, positions, offsets); + }else { + return _CLNEW SegmentTermVector(field, terms, termFreqs); + } +} + +int64_t TermVectorsReader::size() +{ + return _size; +} + + + + +Array<TermVectorOffsetInfo> TermVectorOffsetInfo::EMPTY_OFFSET_INFO; + +TermVectorOffsetInfo::TermVectorOffsetInfo() +{ + startOffset = 0; + endOffset=0; +} + +TermVectorOffsetInfo::~TermVectorOffsetInfo() +{ +} + +TermVectorOffsetInfo::TermVectorOffsetInfo(int32_t startOffset, int32_t endOffset) +{ + this->endOffset = endOffset; + this->startOffset = startOffset; +} + +int32_t TermVectorOffsetInfo::getEndOffset() const +{ + return endOffset; +} + +void TermVectorOffsetInfo::setEndOffset(int32_t endOffset) +{ + this->endOffset = endOffset; +} + +int32_t TermVectorOffsetInfo::getStartOffset() const +{ + return startOffset; +} + +void TermVectorOffsetInfo::setStartOffset(int32_t startOffset) +{ + this->startOffset = startOffset; +} + +bool TermVectorOffsetInfo::equals(TermVectorOffsetInfo* termVectorOffsetInfo) +{ + if (this == termVectorOffsetInfo) + return true; + + if (endOffset != termVectorOffsetInfo->endOffset) return false; + if (startOffset != termVectorOffsetInfo->startOffset) return false; + + return true; +} + +size_t TermVectorOffsetInfo::hashCode() const +{ + size_t result; + result = startOffset; + result = 29 * result + endOffset; + return result; +} + +CL_NS_END diff --git a/src/assistant/3rdparty/clucene/src/CLucene/index/TermVectorWriter.cpp b/src/assistant/3rdparty/clucene/src/CLucene/index/TermVectorWriter.cpp new file mode 100644 index 000000000..276b1bbd0 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/index/TermVectorWriter.cpp @@ -0,0 +1,349 @@ +/* + * Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team + * + * Distributable under the terms of either the Apache License (Version 2.0) or + * the GNU Lesser General Public License, as specified in the COPYING file. + * + * Changes are Copyright(C) 2007, 2008 by Nokia Corporation and/or its subsidiary(-ies), all rights reserved. +*/ +#include "CLucene/StdHeader.h" +#include "TermVector.h" +#include "CLucene/util/Misc.h" + +CL_NS_USE(util) +CL_NS_DEF(index) + +const QLatin1String TermVectorsWriter::LUCENE_TVX_EXTENSION(".tvx"); +const QLatin1String TermVectorsWriter::LUCENE_TVD_EXTENSION(".tvd"); +const QLatin1String TermVectorsWriter::LUCENE_TVF_EXTENSION(".tvf"); + +TermVectorsWriter::TermVectorsWriter(CL_NS(store)::Directory* directory, + const QString& segment,FieldInfos* fieldInfos) +{ + // Open files for TermVector storage + tvx = directory->createOutput(segment + LUCENE_TVX_EXTENSION); + tvx->writeInt(FORMAT_VERSION); + + tvd = directory->createOutput(segment + LUCENE_TVD_EXTENSION); + tvd->writeInt(FORMAT_VERSION); + + tvf = directory->createOutput(segment + LUCENE_TVF_EXTENSION); + tvf->writeInt(FORMAT_VERSION); + + this->fieldInfos = fieldInfos; + + currentField = NULL; + currentDocPointer = -1; +} + +TermVectorsWriter::~TermVectorsWriter() +{ + if (tvx != NULL) { + tvx->close(); + _CLDELETE(tvx); + } + + if (tvd != NULL) { + tvd->close(); + _CLDELETE(tvd); + } + + if (tvf != NULL){ + tvf->close(); + _CLDELETE(tvf); + } +} + +void TermVectorsWriter::openDocument() +{ + closeDocument(); + currentDocPointer = tvd->getFilePointer(); +} + +void TermVectorsWriter::closeDocument() +{ + if (isDocumentOpen()) { + closeField(); + writeDoc(); + fields.clear(); + currentDocPointer = -1; + } +} + +bool TermVectorsWriter::isDocumentOpen() const +{ + return currentDocPointer != -1; +} + + +void TermVectorsWriter::openField(int32_t fieldNumber, + bool storePositionWithTermVector, bool storeOffsetWithTermVector) +{ + if (!isDocumentOpen()) + _CLTHROWA(CL_ERR_InvalidState,"Cannot open field when no document is open."); + + closeField(); + currentField = _CLNEW TVField(fieldNumber, storePositionWithTermVector, + storeOffsetWithTermVector); +} + +void TermVectorsWriter::openField(const TCHAR* field) +{ + FieldInfo* fieldInfo = fieldInfos->fieldInfo(field); + openField(fieldInfo->number, fieldInfo->storePositionWithTermVector, + fieldInfo->storeOffsetWithTermVector); +} + +void TermVectorsWriter::closeField() +{ + if (isFieldOpen()) { + /* DEBUG */ + //System.out.println("closeField()"); + /* DEBUG */ + + // save field and terms + writeField(); + fields.push_back(currentField); + terms.clear(); + currentField = NULL; + } +} + +bool TermVectorsWriter::isFieldOpen() const +{ + return currentField != NULL; +} + +void TermVectorsWriter::addTerm(const TCHAR* termText, int32_t freq, + Array<int32_t>* positions, Array<TermVectorOffsetInfo>* offsets) +{ + if (!isDocumentOpen()) + _CLTHROWA(CL_ERR_InvalidState, "Cannot add terms when document is not open"); + + if (!isFieldOpen()) + _CLTHROWA(CL_ERR_InvalidState, "Cannot add terms when field is not open"); + + addTermInternal(termText, freq, positions, offsets); +} + +void TermVectorsWriter::addTermInternal(const TCHAR* termText, int32_t freq, + Array<int32_t>* positions, Array<TermVectorOffsetInfo>* offsets) +{ + TVTerm* term = _CLNEW TVTerm(); + term->setTermText(termText); + term->freq = freq; + term->positions = positions; + term->offsets = offsets; + terms.push_back(term); +} + +void TermVectorsWriter::addAllDocVectors(Array<TermFreqVector*>& vectors) +{ + openDocument(); + + for (int32_t i = 0; i < vectors.length; ++i) { + bool storePositionWithTermVector = false; + bool storeOffsetWithTermVector = false; + + if ( vectors[i]->__asTermPositionVector() != NULL ) { + TermPositionVector* tpVector = vectors[i]->__asTermPositionVector(); + + if (tpVector->size() > 0 && tpVector->getTermPositions(0) != NULL) + storePositionWithTermVector = true; + if (tpVector->size() > 0 && tpVector->getOffsets(0) != NULL) + storeOffsetWithTermVector = true; + + FieldInfo* fieldInfo = fieldInfos->fieldInfo(tpVector->getField()); + openField(fieldInfo->number, storePositionWithTermVector, storeOffsetWithTermVector); + + for (int32_t j = 0; j < tpVector->size(); ++j) + addTermInternal(tpVector->getTerms()[j], + (*tpVector->getTermFrequencies())[j], + tpVector->getTermPositions(j), + tpVector->getOffsets(j)); + + closeField(); + + } else { + TermFreqVector* tfVector = vectors[i]; + + FieldInfo* fieldInfo = fieldInfos->fieldInfo(tfVector->getField()); + openField(fieldInfo->number, storePositionWithTermVector, storeOffsetWithTermVector); + + for (int32_t j = 0; j < tfVector->size(); ++j) + addTermInternal(tfVector->getTerms()[j], + (*tfVector->getTermFrequencies())[j], NULL, NULL); + + closeField(); + } + } + + closeDocument(); +} + + +void TermVectorsWriter::close() +{ + try { + closeDocument(); + + // make an effort to close all streams we can but remember and re-throw + // the first exception encountered in this process +#define _DOTVWCLOSE(x) \ + if (x != NULL) { \ + try { \ + x->close(); \ + _CLDELETE(x) \ + } catch (CLuceneError& e) { \ + if ( e.number() != CL_ERR_IO ) \ + throw e; \ + if (ikeep == 0) \ + ikeep = e.number(); \ + if (keep[0] == 0) \ + strcpy(keep,e.what()); \ + } catch (...) { \ + if (keep[0] == 0) \ + strcpy(keep, "Unknown error while closing " #x);\ + } \ + } + } _CLFINALLY ( \ + char keep[200]; + int32_t ikeep = 0; + keep[0] = 0; + _DOTVWCLOSE(tvx); + _DOTVWCLOSE(tvd); + _DOTVWCLOSE(tvf); + if (keep[0] != 0) + _CLTHROWA(ikeep, keep); + ); +} + +void TermVectorsWriter::writeField() +{ + // remember where this field is written + currentField->tvfPointer = tvf->getFilePointer(); + //System.out.println("Field Pointer: " + currentField.tvfPointer); + int32_t size = terms.size(); + + tvf->writeVInt(size); + + bool storePositions = currentField->storePositions; + bool storeOffsets = currentField->storeOffsets; + uint8_t bits = 0x0; + if (storePositions) + bits |= STORE_POSITIONS_WITH_TERMVECTOR; + if (storeOffsets) + bits |= STORE_OFFSET_WITH_TERMVECTOR; + tvf->writeByte(bits); + + const TCHAR* lastTermText = LUCENE_BLANK_STRING; + int32_t lastTermTextLen = 0; + + for (int32_t i = 0; i < size; ++i) { + TVTerm* term = terms[i]; + int32_t start = CL_NS(util)::Misc::stringDifference(lastTermText, + lastTermTextLen, term->getTermText(),term->getTermTextLen()); + int32_t length = term->getTermTextLen() - start; + tvf->writeVInt(start); // write shared prefix length + tvf->writeVInt(length); // write delta length + tvf->writeChars(term->getTermText(), start, length); // write delta chars + tvf->writeVInt(term->freq); + + lastTermText = term->getTermText(); + lastTermTextLen = term->getTermTextLen(); + + if (storePositions) { + if(term->positions == NULL) + _CLTHROWA(CL_ERR_IllegalState, "Trying to write positions that are NULL!"); + + // use delta encoding for positions + int32_t position = 0; + for (int32_t j = 0; j < term->freq; ++j){ + tvf->writeVInt((*term->positions)[j] - position); + position = (*term->positions)[j]; + } + } + + if (storeOffsets) { + if(term->offsets == NULL) + _CLTHROWA(CL_ERR_IllegalState, "Trying to write offsets that are NULL!"); + + // use delta encoding for offsets + int32_t position = 0; + for (int32_t j = 0; j < term->freq; ++j) { + tvf->writeVInt((*term->offsets)[j].getStartOffset() - position); + //Save the diff between the two. + tvf->writeVInt((*term->offsets)[j].getEndOffset() - + (*term->offsets)[j].getStartOffset()); + position = (*term->offsets)[j].getEndOffset(); + } + } + } +} + +void TermVectorsWriter::writeDoc() +{ + if (isFieldOpen()) { + _CLTHROWA(CL_ERR_InvalidState, + "Field is still open while writing document"); + } + + // write document index record + tvx->writeLong(currentDocPointer); + + // write document data record + int32_t size = fields.size(); + + // write the number of fields + tvd->writeVInt(size); + + // write field numbers + for (int32_t j = 0; j < size; ++j) { + tvd->writeVInt(fields[j]->number); + } + + // write field pointers + int64_t lastFieldPointer = 0; + for (int32_t i = 0; i < size; ++i) { + TVField* field = (TVField*) fields[i]; + tvd->writeVLong(field->tvfPointer - lastFieldPointer); + + lastFieldPointer = field->tvfPointer; + } +} + +const TCHAR* TermVectorsWriter::TVTerm::getTermText() const +{ + return termText; +} + +size_t TermVectorsWriter::TVTerm::getTermTextLen() +{ + if (termTextLen==-1) + termTextLen = _tcslen(termText); + return termTextLen; +} + +void TermVectorsWriter::TVTerm::setTermText(const TCHAR* val) +{ + _CLDELETE_CARRAY(termText); + termText = STRDUP_TtoT(val); + termTextLen = -1; + +} + +TermVectorsWriter::TVTerm::TVTerm() + : freq(0) + , positions(NULL) + , offsets(NULL) +{ + termText=NULL; + termTextLen=-1; +} + +TermVectorsWriter::TVTerm::~TVTerm() +{ + _CLDELETE_CARRAY(termText) +} + +CL_NS_END diff --git a/src/assistant/3rdparty/clucene/src/CLucene/index/Terms.h b/src/assistant/3rdparty/clucene/src/CLucene/index/Terms.h new file mode 100644 index 000000000..806441876 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/index/Terms.h @@ -0,0 +1,174 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _lucene_index_Terms_ +#define _lucene_index_Terms_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include "Term.h" +CL_NS_DEF(index) + +class TermEnum; //predefine +class TermPositions; + +/** TermDocs provides an interface for enumerating <document, frequency> + pairs for a term. <p> The document portion names each document containing + the term. Documents are indicated by number. The frequency portion gives + the number of times the term occurred in each document. <p> The pairs are + ordered by document number. + + @see IndexReader#termDocs() + */ +class TermDocs: LUCENE_BASE { +public: + virtual ~TermDocs(){ + } + + // Sets this to the data for a term. + // The enumeration is reset to the start of the data for this term. + virtual void seek(Term* term)=0; + + /** Sets this to the data for the current term in a {@link TermEnum}. + * This may be optimized in some implementations. + */ + virtual void seek(TermEnum* termEnum)=0; + + // Returns the current document number. <p> This is invalid until {@link + // #next()} is called for the first time. + virtual int32_t doc() const=0; + + // Returns the frequency of the term within the current document. <p> This + // is invalid until {@link #next()} is called for the first time. + virtual int32_t freq() const=0; + + // Moves to the next pair in the enumeration. <p> Returns true iff there is + // such a next pair in the enumeration. + virtual bool next() =0; + + // Attempts to read multiple entries from the enumeration, up to length of + // <i>docs</i>. Document numbers are stored in <i>docs</i>, and term + // frequencies are stored in <i>freqs</i>. The <i>freqs</i> array must be as + // int64_t as the <i>docs</i> array. + // + // <p>Returns the number of entries read. Zero is only returned when the + // stream has been exhausted. + virtual int32_t read(int32_t* docs, int32_t* freqs, int32_t length)=0; + + // Skips entries to the first beyond the current whose document number is + // greater than or equal to <i>target</i>. <p>Returns true iff there is such + // an entry. <p>Behaves as if written: <pre> + // bool skipTo(int32_t target) { + // do { + // if (!next()) + // return false; + // } while (target > doc()); + // return true; + // } + // </pre> + // Some implementations are considerably more efficient than that. + virtual bool skipTo(const int32_t target)=0; + + // Frees associated resources. + virtual void close() = 0; + + + /** Solve the diamond inheritence problem by providing a reinterpret function. + * No dynamic casting is required and no RTTI data is needed to do this + */ + virtual TermPositions* __asTermPositions()=0; +}; + + +// Abstract class for enumerating terms. +// +//<p>Term enumerations are always ordered by Term.compareTo(). Each term in +//the enumeration is greater than all that precede it. +class TermEnum: LUCENE_BASE { +public: + // Increments the enumeration to the next element. True if one exists. + virtual bool next()=0; + + // Returns a pointer to the current Term in the enumeration. + virtual Term* term()=0; + + // Returns the current Term in the enumeration. + virtual Term* term(bool pointer){ + Term* ret = term(); + if ( !pointer ) + ret->__cl_decref(); + return ret; + } + + // Returns the docFreq of the current Term in the enumeration. + virtual int32_t docFreq() const=0; + + // Closes the enumeration to further activity, freeing resources. + virtual void close() =0; + + virtual ~TermEnum(){ + } + + // Term Vector support + /** Skips terms to the first beyond the current whose value is + * greater or equal to <i>target</i>. <p>Returns true iff there is such + * an entry. <p>Behaves as if written: <pre> + * public boolean skipTo(Term target) { + * do { + * if (!next()) + * return false; + * } while (target > term()); + * return true; + * } + * </pre> + * Some implementations are considerably more efficient than that. + */ + virtual bool skipTo(Term* target){ + do { + if (!next()) + return false; + } while (target->compareTo(term(false)) > 0); + return true; + } + + /** + * Because we need to know how to cast the object, we need the objects name. + */ + virtual const char* getObjectName() = 0; +}; + + + +/** + * TermPositions provides an interface for enumerating the <document, + * frequency, <position>* > tuples for a term. <p> The document and + * frequency are the same as for a TermDocs. The positions portion lists the ordinal + * positions of each occurrence of a term in a document. + * + * @see IndexReader#termPositions() + */ +class TermPositions: public virtual TermDocs { +public: + // Returns next position in the current document. It is an error to call + // this more than {@link #freq()} times + // without calling {@link #next()}<p> This is + // invalid until {@link #next()} is called for + // the first time. + virtual int32_t nextPosition() = 0; + + virtual ~TermPositions(){ + } + + /** Solve the diamond inheritence problem by providing a reinterpret function. + * No dynamic casting is required and no RTTI data is needed to do this + */ + virtual TermDocs* __asTermDocs()=0; + virtual TermPositions* __asTermPositions()=0; +}; +CL_NS_END +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/queryParser/Lexer.cpp b/src/assistant/3rdparty/clucene/src/CLucene/queryParser/Lexer.cpp new file mode 100644 index 000000000..861c5d3cb --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/queryParser/Lexer.cpp @@ -0,0 +1,371 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#include "CLucene/StdHeader.h" +#include "Lexer.h" + +#include "CLucene/util/FastCharStream.h" +#include "CLucene/util/Reader.h" +#include "CLucene/util/StringBuffer.h" +#include "TokenList.h" +#include "QueryToken.h" +#include "QueryParserBase.h" + +CL_NS_USE(util) + +CL_NS_DEF(queryParser) +Lexer::Lexer(QueryParserBase* queryparser, const TCHAR* query) { + //Func - Constructor + //Pre - query != NULL and contains the query string + //Post - An instance of Lexer has been created + + this->queryparser = queryparser; + + CND_PRECONDITION(query != NULL, "query is NULL"); + + //The InputStream of Reader must be destroyed in the destructor + delSR = true; + + StringReader *r = _CLNEW StringReader(query); + + //Check to see if r has been created properly + CND_CONDITION(r != NULL, "Could not allocate memory for StringReader r"); + + //Instantie a FastCharStream instance using r and assign it to reader + reader = _CLNEW FastCharStream(r); + + //Check to see if reader has been created properly + CND_CONDITION(reader != NULL, "Could not allocate memory for FastCharStream reader"); + + //The InputStream of Reader must be destroyed in the destructor + delSR = true; + +} + + +Lexer::Lexer(QueryParserBase* queryparser, Reader* source) { + //Func - Constructor + // Initializes a new instance of the Lexer class with the specified + // TextReader to lex. + //Pre - Source contains a valid reference to a Reader + //Post - An instance of Lexer has been created using source as the reader + + this->queryparser = queryparser; + + //Instantie a FastCharStream instance using r and assign it to reader + reader = _CLNEW FastCharStream(source); + + //Check to see if reader has been created properly + CND_CONDITION(reader != NULL, "Could not allocate memory for FastCharStream reader"); + + //The InputStream of Reader must not be destroyed in the destructor + delSR = false; +} + + +Lexer::~Lexer() { + //Func - Destructor + //Pre - true + //Post - if delSR was true the InputStream input of reader has been deleted + // The instance of Lexer has been destroyed + + if (delSR) { + _CLDELETE(reader->input); + } + + _CLDELETE(reader); +} + + +void Lexer::Lex(TokenList *tokenList) { + //Func - Breaks the input stream onto the tokens list tokens + //Pre - tokens != NULL and contains a TokenList in which the tokens can be stored + //Post - The tokens have been added to the TokenList tokens + + CND_PRECONDITION(tokenList != NULL, "tokens is NULL"); + + //Get all the tokens + while(true) { + //Add the token to the tokens list + + //Get the next token + QueryToken* token = _CLNEW QueryToken; + if ( !GetNextToken(token) ){ + _CLDELETE(token); + break; + } + tokenList->add(token); + } + + //The end has been reached so create an EOF_ token + //Add the final token to the TokenList _tokens + tokenList->add(_CLNEW QueryToken( QueryToken::EOF_)); +} + + +bool Lexer::GetNextToken(QueryToken* token) { + while(!reader->Eos()) { + int ch = reader->GetNext(); + + if ( ch == -1 ) + break; + + // skipping whitespaces + if( _istspace(ch)!=0 ) { + continue; + } + TCHAR buf[2] = {ch,'\0'}; + switch(ch) { + case '+': + token->set(buf, QueryToken::PLUS); + return true; + case '-': + token->set(buf, QueryToken::MINUS); + return true; + case '(': + token->set(buf, QueryToken::LPAREN); + return true; + case ')': + token->set(buf, QueryToken::RPAREN); + return true; + case ':': + token->set(buf, QueryToken::COLON); + return true; + case '!': + token->set(buf, QueryToken::NOT); + return true; + case '^': + token->set(buf, QueryToken::CARAT); + return true; + case '~': + if( _istdigit( reader->Peek() )!=0 ) { + TCHAR number[LUCENE_MAX_FIELD_LEN]; + ReadIntegerNumber(ch, number,LUCENE_MAX_FIELD_LEN); + token->set(number, QueryToken::SLOP); + return true; + }else{ + token->set(buf, QueryToken::FUZZY); + return true; + } + break; + case '"': + return ReadQuoted(ch, token); + case '[': + return ReadInclusiveRange(ch, token); + case '{': + return ReadExclusiveRange(ch, token); + case ']': + case '}': + case '*': + queryparser->throwParserException( _T("Unrecognized TCHAR %d at %d::%d."), + ch, reader->Column(), reader->Line() ); + return false; + default: + return ReadTerm(ch, token); + + // end of swith + } + + } + return false; +} + + +void Lexer::ReadIntegerNumber(const TCHAR ch, TCHAR* buf, int buflen) { + int bp=0; + buf[bp++] = ch; + + int c = reader->Peek(); + while( c!=-1 && _istdigit(c)!=0 && bp<buflen-1 ) { + buf[bp++] = reader->GetNext(); + c = reader->Peek(); + } + buf[bp++] = 0; +} + + +bool Lexer::ReadInclusiveRange(const TCHAR prev, QueryToken* token) { + int ch = prev; + StringBuffer range; + range.appendChar(ch); + + while(!reader->Eos()) { + ch = reader->GetNext(); + if ( ch == -1 ) + break; + range.appendChar(ch); + + if(ch == ']'){ + token->set(range.getBuffer(), QueryToken::RANGEIN); + return true; + } + } + queryparser->throwParserException(_T("Unterminated inclusive range! %d %d::%d"),' ', + reader->Column(),reader->Column()); + return false; +} + + +bool Lexer::ReadExclusiveRange(const TCHAR prev, QueryToken* token) { + int ch = prev; + StringBuffer range; + range.appendChar(ch); + + while(!reader->Eos()) { + ch = reader->GetNext(); + + if (ch==-1) + break; + range.appendChar(ch); + + if(ch == '}'){ + token->set(range.getBuffer(), QueryToken::RANGEEX); + return true; + } + } + queryparser->throwParserException(_T("Unterminated exclusive range! %d %d::%d"),' ', + reader->Column(),reader->Column() ); + return false; +} + +bool Lexer::ReadQuoted(const TCHAR prev, QueryToken* token) { + int ch = prev; + StringBuffer quoted; + quoted.appendChar(ch); + + while(!reader->Eos()) { + ch = reader->GetNext(); + + if (ch==-1) + break; + + quoted.appendChar(ch); + + if(ch == '"'){ + token->set(quoted.getBuffer(), QueryToken::QUOTED); + return true; + } + } + queryparser->throwParserException(_T("Unterminated string! %d %d::%d"),' ', + reader->Column(),reader->Column()); + return false; +} + + +bool Lexer::ReadTerm(const TCHAR prev, QueryToken* token) { + int ch = prev; + bool completed = false; + int32_t asteriskCount = 0; + bool hasQuestion = false; + + StringBuffer val; + TCHAR buf[3]; //used for readescaped + + while(true) { + switch(ch) { + case -1: + break; + case '\\': + { + if ( ReadEscape(ch, buf) ) + val.append( buf ); + else + return false; + } + break; + + case LUCENE_WILDCARDTERMENUM_WILDCARD_STRING: + asteriskCount++; + val.appendChar(ch); + break; + case LUCENE_WILDCARDTERMENUM_WILDCARD_CHAR: + hasQuestion = true; + val.appendChar(ch); + break; + case '\n': + case '\t': + case ' ': + case '+': + case '-': + case '!': + case '(': + case ')': + case ':': + case '^': + case '[': + case ']': + case '{': + case '}': + case '~': + case '"': + // create new QueryToken + reader->UnGet(); + completed = true; + break; + default: + val.appendChar(ch); + break; + // end of switch + } + + if(completed || ch==-1 || reader->Eos() ) + break; + else + ch = reader->GetNext(); + } + + // create new QueryToken + if(hasQuestion) + token->set(val.getBuffer(), QueryToken::WILDTERM); + else if(asteriskCount == 1 && val.getBuffer()[val.length() - 1] == '*') + token->set(val.getBuffer(), QueryToken::PREFIXTERM); + else if(asteriskCount > 0) + token->set(val.getBuffer(), QueryToken::WILDTERM); + else if( _tcsicmp(val.getBuffer(), _T("AND"))==0 || _tcscmp(val.getBuffer(), _T("&&"))==0 ) + token->set(val.getBuffer(), QueryToken::AND_); + else if( _tcsicmp(val.getBuffer(), _T("OR"))==0 || _tcscmp(val.getBuffer(), _T("||"))==0) + token->set(val.getBuffer(), QueryToken::OR); + else if( _tcsicmp(val.getBuffer(), _T("NOT"))==0 ) + token->set(val.getBuffer(), QueryToken::NOT); + else { + bool isnum = true; + int32_t nlen=val.length(); + for (int32_t i=0;i<nlen;++i) { + TCHAR ch=val.getBuffer()[i]; + if ( _istalpha(ch) ) { + isnum=false; + break; + } + } + + if ( isnum ) + token->set(val.getBuffer(), QueryToken::NUMBER); + else + token->set(val.getBuffer(), QueryToken::TERM); + } + return true; +} + + +bool Lexer::ReadEscape(TCHAR prev, TCHAR* buf) { + TCHAR ch = prev; + int bp=0; + buf[bp++] = ch; + + ch = reader->GetNext(); + int32_t idx = _tcscspn( buf, _T("\\+-!():^[]{}\"~*") ); + if(idx == 0) { + buf[bp++] = ch; + buf[bp++]=0; + return true; + } + queryparser->throwParserException(_T("Unrecognized escape sequence at %d %d::%d"), ' ', + reader->Column(),reader->Line()); + return false; +} + + +CL_NS_END diff --git a/src/assistant/3rdparty/clucene/src/CLucene/queryParser/Lexer.h b/src/assistant/3rdparty/clucene/src/CLucene/queryParser/Lexer.h new file mode 100644 index 000000000..b3b55523e --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/queryParser/Lexer.h @@ -0,0 +1,67 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +* +* Changes are Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies). +------------------------------------------------------------------------------*/ +#ifndef _lucene_queryParser_Lexer_ +#define _lucene_queryParser_Lexer_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include "CLucene/util/FastCharStream.h" +#include "CLucene/util/Reader.h" +#include "CLucene/util/StringBuffer.h" + +#include "TokenList.h" + +CL_NS_DEF(queryParser) +class QueryParserBase; + // A simple Lexer that is used by QueryParser. + class Lexer:LUCENE_BASE + { + private: + CL_NS(util)::FastCharStream* reader; + QueryParserBase* queryparser; //holds the queryparser so that we can do callbacks + bool delSR; //Indicates if the reader must be deleted or not + + public: + // Initializes a new instance of the Lexer class with the specified + // query to lex. + Lexer(QueryParserBase* queryparser, const TCHAR* query); + + // Initializes a new instance of the Lexer class with the specified + // TextReader to lex. + Lexer(QueryParserBase* queryparser, CL_NS(util)::Reader* source); + + //Breaks the input stream onto the tokens list tokens + void Lex(TokenList *tokens); + + ~Lexer(); + + private: + bool GetNextToken(QueryToken* token); + + // Reads an integer number. buf should quite large, probably as large as a field should ever be + void ReadIntegerNumber(const TCHAR ch, TCHAR* buf, int buflen); + + // Reads an inclusive range like [some words] + bool ReadInclusiveRange(const TCHAR prev, QueryToken* token); + + // Reads an exclusive range like {some words} + bool ReadExclusiveRange(const TCHAR prev, QueryToken* token); + + // Reads quoted string like "something else" + bool ReadQuoted(const TCHAR prev, QueryToken* token); + + bool ReadTerm(const TCHAR prev, QueryToken* token); + + //reads an escaped character into the buf. Buf requires at least 3 characters + bool ReadEscape(const TCHAR prev, TCHAR* buf); + }; +CL_NS_END +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/queryParser/MultiFieldQueryParser.cpp b/src/assistant/3rdparty/clucene/src/CLucene/queryParser/MultiFieldQueryParser.cpp new file mode 100644 index 000000000..b57896b66 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/queryParser/MultiFieldQueryParser.cpp @@ -0,0 +1,215 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +* +* Changes are Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies). +------------------------------------------------------------------------------*/ +#include "CLucene/StdHeader.h" +#include "MultiFieldQueryParser.h" +#include "CLucene/analysis/AnalysisHeader.h" +#include "CLucene/search/BooleanQuery.h" +#include "CLucene/search/PhraseQuery.h" +#include "CLucene/search/SearchHeader.h" +#include "QueryParser.h" + +CL_NS_USE(index) +CL_NS_USE(util) +CL_NS_USE(search) +CL_NS_USE(analysis) + +CL_NS_DEF(queryParser) + +MultiFieldQueryParser::MultiFieldQueryParser(const TCHAR** fields, + CL_NS(analysis)::Analyzer* analyzer, BoostMap* boosts) + : QueryParser(NULL, analyzer) +{ + this->fields = fields; + this->boosts = boosts; +} + +MultiFieldQueryParser::~MultiFieldQueryParser() +{ +} + +//static +Query* MultiFieldQueryParser::parse(const TCHAR* query, const TCHAR** fields, + Analyzer* analyzer) +{ + BooleanQuery* bQuery = _CLNEW BooleanQuery(); + int32_t i = 0; + while (fields[i] != NULL){ + Query* q = QueryParser::parse(query, fields[i], analyzer); + if (q && (q->getQueryName() != _T("BooleanQuery") + || ((BooleanQuery*)q)->getClauseCount() > 0)) { + bQuery->add(q , true, false, false); + } else { + _CLDELETE(q); + } + i++; + } + return bQuery; +} + +//static +Query* MultiFieldQueryParser::parse(const TCHAR* query, const TCHAR** fields, + const uint8_t* flags, Analyzer* analyzer) +{ + BooleanQuery* bQuery = _CLNEW BooleanQuery(); + int32_t i = 0; + while ( fields[i] != NULL ) { + Query* q = QueryParser::parse(query, fields[i], analyzer); + if (q && (q->getQueryName() != _T("BooleanQuery") + || ((BooleanQuery*)q)->getClauseCount() > 0)) { + uint8_t flag = flags[i]; + switch (flag) { + case MultiFieldQueryParser::REQUIRED_FIELD: + bQuery->add(q, true, true, false); + break; + case MultiFieldQueryParser::PROHIBITED_FIELD: + bQuery->add(q, true, false, true); + break; + default: + bQuery->add(q, true, false, false); + break; + } + } else { + _CLDELETE(q); + } + i++; + } + return bQuery; +} + + +Query* MultiFieldQueryParser::GetFieldQuery(const TCHAR* field, TCHAR* queryText, int32_t slop){ + if (field == NULL) { + CL_NS_STD(vector)<BooleanClause*> clauses; + for (int i = 0; fields[i]!=NULL; ++i) { + Query* q = QueryParser::GetFieldQuery(fields[i], queryText); + if (q != NULL) { + //If the user passes a map of boosts + if (boosts != NULL) { + //Get the boost from the map and apply them + BoostMap::const_iterator itr = boosts->find(fields[i]); + if (itr != boosts->end()) { + q->setBoost(itr->second); + } + } + if (q->getQueryName() == PhraseQuery::getClassName()) { + ((PhraseQuery*)q)->setSlop(slop); + } + //if (q instanceof MultiPhraseQuery) { + // ((MultiPhraseQuery) q).setSlop(slop); + //} + q = QueryAddedCallback(fields[i], q); + if ( q ) + clauses.push_back(_CLNEW BooleanClause(q, true, false,false)); + } + } + if (clauses.size() == 0) // happens for stopwords + return NULL; + Query* q = QueryParser::GetBooleanQuery(clauses); + return q; + }else{ + Query* q = QueryParser::GetFieldQuery(field, queryText); + if ( q ) + q = QueryAddedCallback(field,q); + return q; + } +} + + +Query* MultiFieldQueryParser::GetFieldQuery(const TCHAR* field, TCHAR* queryText){ + return GetFieldQuery(field, queryText, 0); +} + + +CL_NS(search)::Query* MultiFieldQueryParser::GetFuzzyQuery(const TCHAR* field, TCHAR* termStr){ + if (field == NULL) { + CL_NS_STD(vector)<BooleanClause*> clauses; + for (int i = 0; fields[i]!=NULL; ++i) { + Query* q = QueryParser::GetFuzzyQuery(fields[i], termStr); //todo: , minSimilarity + if ( q ){ + q = QueryAddedCallback(fields[i], q); + if ( q ){ + clauses.push_back(_CLNEW BooleanClause(q,true,false,false) ); + } + } + } + return QueryParser::GetBooleanQuery(clauses); + }else{ + Query* q = QueryParser::GetFuzzyQuery(field, termStr);//todo: , minSimilarity + if ( q ) + q = QueryAddedCallback(field,q); + return q; + } +} + +Query* MultiFieldQueryParser::GetPrefixQuery(const TCHAR* field, TCHAR* termStr){ + if (field == NULL) { + CL_NS_STD(vector)<BooleanClause*> clauses; + for (int i = 0; fields[i]!=NULL; ++i) { + Query* q = QueryParser::GetPrefixQuery(fields[i], termStr); + if ( q ){ + q = QueryAddedCallback(fields[i],q); + if ( q ){ + clauses.push_back(_CLNEW BooleanClause(q,true,false,false)); + } + } + } + return QueryParser::GetBooleanQuery(clauses); + }else{ + Query* q = QueryParser::GetPrefixQuery(field, termStr); + if ( q ) + q = QueryAddedCallback(field,q); + return q; + } +} + +Query* MultiFieldQueryParser::GetWildcardQuery(const TCHAR* field, TCHAR* termStr){ + if (field == NULL) { + CL_NS_STD(vector)<BooleanClause*> clauses; + for (int i = 0; fields[i]!=NULL; ++i) { + Query* q = QueryParser::GetWildcardQuery(fields[i], termStr); + if ( q ){ + q = QueryAddedCallback(fields[i],q); + if ( q ){ + clauses.push_back(_CLNEW BooleanClause(q,true,false,false)); + } + } + } + return QueryParser::GetBooleanQuery(clauses); + }else{ + Query* q = QueryParser::GetWildcardQuery(field, termStr); + if ( q ) + q = QueryAddedCallback(field,q); + return q; + } +} + + +Query* MultiFieldQueryParser::GetRangeQuery(const TCHAR* field, TCHAR* part1, TCHAR* part2, bool inclusive){ + if (field == NULL) { + CL_NS_STD(vector)<BooleanClause*> clauses; + for (int i = 0; fields[i]!=NULL; ++i) { + Query* q = QueryParser::GetRangeQuery(fields[i], part1, part2, inclusive); + if ( q ){ + q = QueryAddedCallback(fields[i],q); + if ( q ){ + clauses.push_back(_CLNEW BooleanClause(q,true,false,false)); + } + } + } + return QueryParser::GetBooleanQuery(clauses); + }else{ + Query* q = QueryParser::GetRangeQuery(field, part1, part2, inclusive); + if ( q ) + q = QueryAddedCallback(field,q); + return q; + } +} + + +CL_NS_END diff --git a/src/assistant/3rdparty/clucene/src/CLucene/queryParser/MultiFieldQueryParser.h b/src/assistant/3rdparty/clucene/src/CLucene/queryParser/MultiFieldQueryParser.h new file mode 100644 index 000000000..bf7d652a7 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/queryParser/MultiFieldQueryParser.h @@ -0,0 +1,136 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef MultiFieldQueryParser_H +#define MultiFieldQueryParser_H + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include "CLucene/analysis/AnalysisHeader.h" +#include "CLucene/search/SearchHeader.h" +#include "QueryParser.h" + + +CL_NS_DEF(queryParser) + +typedef CL_NS(util)::CLHashMap<const TCHAR*, + qreal, + CL_NS(util)::Compare::TChar, + CL_NS(util)::Equals::TChar, + CL_NS(util)::Deletor::tcArray, + CL_NS(util)::Deletor::DummyFloat + > BoostMap; + + /** + * A QueryParser which constructs queries to search multiple fields. + * + */ + class MultiFieldQueryParser: public QueryParser + { + protected: + const TCHAR** fields; + BoostMap* boosts; + public: + LUCENE_STATIC_CONSTANT(uint8_t, NORMAL_FIELD=0); + LUCENE_STATIC_CONSTANT(uint8_t, REQUIRED_FIELD=1); + LUCENE_STATIC_CONSTANT(uint8_t, PROHIBITED_FIELD=2); + + /** + * Creates a MultiFieldQueryParser. + * + * <p>It will, when parse(String query) + * is called, construct a query like this (assuming the query consists of + * two terms and you specify the two fields <code>title</code> and <code>body</code>):</p> + * + * <code> + * (title:term1 body:term1) (title:term2 body:term2) + * </code> + * + * <p>When setDefaultOperator(AND_OPERATOR) is set, the result will be:</p> + * + * <code> + * +(title:term1 body:term1) +(title:term2 body:term2) + * </code> + * + * <p>In other words, all the query's terms must appear, but it doesn't matter in + * what fields they appear.</p> + */ + MultiFieldQueryParser(const TCHAR** fields, CL_NS(analysis)::Analyzer* a, BoostMap* boosts = NULL); + virtual ~MultiFieldQueryParser(); + + /** + * <p> + * Parses a query which searches on the fields specified. + * <p> + * If x fields are specified, this effectively constructs: + * <pre> + * <code> + * (field1:query) (field2:query) (field3:query)...(fieldx:query) + * </code> + * </pre> + * + * @param query Query string to parse + * @param fields Fields to search on + * @param analyzer Analyzer to use + * @throws ParserException if query parsing fails + * @throws TokenMgrError if query parsing fails + */ + static CL_NS(search)::Query* parse(const TCHAR* query, const TCHAR** fields, CL_NS(analysis)::Analyzer* analyzer); + + /** + * <p> + * Parses a query, searching on the fields specified. + * Use this if you need to specify certain fields as required, + * and others as prohibited. + * <p><pre> + * Usage: + * <code> + * String[] fields = {"filename", "contents", "description"}; + * int32_t[] flags = {MultiFieldQueryParser.NORMAL FIELD, + * MultiFieldQueryParser.REQUIRED FIELD, + * MultiFieldQueryParser.PROHIBITED FIELD,}; + * parse(query, fields, flags, analyzer); + * </code> + * </pre> + *<p> + * The code above would construct a query: + * <pre> + * <code> + * (filename:query) +(contents:query) -(description:query) + * </code> + * </pre> + * + * @param query Query string to parse + * @param fields Fields to search on + * @param flags Flags describing the fields + * @param analyzer Analyzer to use + * @throws ParserException if query parsing fails + * @throws TokenMgrError if query parsing fails + */ + static CL_NS(search)::Query* parse(const TCHAR* query, const TCHAR** fields, const uint8_t* flags, CL_NS(analysis)::Analyzer* analyzer); + + + + protected: + CL_NS(search)::Query* GetFieldQuery(const TCHAR* field, TCHAR* queryText); + CL_NS(search)::Query* GetFieldQuery(const TCHAR* field, TCHAR* queryText, int32_t slop); + CL_NS(search)::Query* GetFuzzyQuery(const TCHAR* field, TCHAR* termStr); + CL_NS(search)::Query* GetRangeQuery(const TCHAR* field, TCHAR* part1, TCHAR* part2, bool inclusive); + CL_NS(search)::Query* GetPrefixQuery(const TCHAR* field, TCHAR* termStr); + CL_NS(search)::Query* GetWildcardQuery(const TCHAR* field, TCHAR* termStr); + + /** + * A special virtual function for the MultiFieldQueryParser which can be used + * to clean up queries. Once the field name is known and the query has been + * created, its passed to this function. + * An example of this usage is to set boosts. + */ + virtual CL_NS(search)::Query* QueryAddedCallback(const TCHAR* field, CL_NS(search)::Query* query){ return query; } + }; +CL_NS_END +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/queryParser/QueryParser.cpp b/src/assistant/3rdparty/clucene/src/CLucene/queryParser/QueryParser.cpp new file mode 100644 index 000000000..b11eec0bb --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/queryParser/QueryParser.cpp @@ -0,0 +1,509 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +* +* Changes are Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies). +------------------------------------------------------------------------------*/ +#include "CLucene/StdHeader.h" +#include "QueryParser.h" + +#include "CLucene/analysis/AnalysisHeader.h" +#include "CLucene/util/Reader.h" +#include "CLucene/search/SearchHeader.h" +#include "CLucene/index/Term.h" + +#include "TokenList.h" +#include "QueryToken.h" +#include "QueryParserBase.h" +#include "Lexer.h" + +CL_NS_USE(util) +CL_NS_USE(index) +CL_NS_USE(analysis) +CL_NS_USE(search) + +CL_NS_DEF(queryParser) + + QueryParser::QueryParser(const TCHAR* _field, Analyzer* _analyzer) : QueryParserBase(_analyzer){ + //Func - Constructor. + // Instantiates a QueryParser for the named field _field + //Pre - _field != NULL + //Post - An instance has been created + + if ( _field ) + field = STRDUP_TtoT(_field); + else + field = NULL; + tokens = NULL; + lowercaseExpandedTerms = true; + } + + QueryParser::~QueryParser() { + //Func - Destructor + //Pre - true + //Post - The instance has been destroyed + + _CLDELETE_CARRAY(field); + } + + //static + Query* QueryParser::parse(const TCHAR* query, const TCHAR* field, Analyzer* analyzer){ + //Func - Returns a new instance of the Query class with a specified query, field and + // analyzer values. + //Pre - query != NULL and holds the query to parse + // field != NULL and holds the default field for query terms + // analyzer holds a valid reference to an Analyzer and is used to + // find terms in the query text + //Post - query has been parsed and an instance of Query has been returned + + CND_PRECONDITION(query != NULL, "query is NULL"); + CND_PRECONDITION(field != NULL, "field is NULL"); + + QueryParser parser(field, analyzer); + return parser.parse(query); + } + + Query* QueryParser::parse(const TCHAR* query){ + //Func - Returns a parsed Query instance + //Pre - query != NULL and contains the query value to be parsed + //Post - Returns a parsed Query Instance + + CND_PRECONDITION(query != NULL, "query is NULL"); + + //Instantie a Stringer that can read the query string + Reader* r = _CLNEW StringReader(query); + + //Check to see if r has been created properly + CND_CONDITION(r != NULL, "Could not allocate memory for StringReader r"); + + //Pointer for the return value + Query* ret = NULL; + + try{ + //Parse the query managed by the StringReader R and return a parsed Query instance + //into ret + ret = parse(r); + }_CLFINALLY ( + _CLDELETE(r); + ); + + return ret; + } + + Query* QueryParser::parse(Reader* reader){ + //Func - Returns a parsed Query instance + //Pre - reader contains a valid reference to a Reader and manages the query string + //Post - A parsed Query instance has been returned or + + //instantiate the TokenList tokens + TokenList _tokens; + this->tokens = &_tokens; + + //Instantiate a lexer + Lexer lexer(this, reader); + + //tokens = lexer.Lex(); + //Lex the tokens + lexer.Lex(tokens); + + //Peek to the first token and check if is an EOF + if (tokens->peek()->Type == QueryToken::EOF_){ + // The query string failed to yield any tokens. We discard the + // TokenList tokens and raise an exceptioin. + QueryToken* token = this->tokens->extract(); + _CLDELETE(token); + _CLTHROWA(CL_ERR_Parse, "No query given."); + } + + //Return the parsed Query instance + Query* ret = MatchQuery(field); + this->tokens = NULL; + return ret; + } + + int32_t QueryParser::MatchConjunction(){ + //Func - matches for CONJUNCTION + // CONJUNCTION ::= <AND> | <OR> + //Pre - tokens != NULL + //Post - if the first token is an AND or an OR then + // the token is extracted and deleted and CONJ_AND or CONJ_OR is returned + // otherwise CONJ_NONE is returned + + CND_PRECONDITION(tokens != NULL, "tokens is NULL"); + + switch(tokens->peek()->Type){ + case QueryToken::AND_ : + //Delete the first token of tokenlist + ExtractAndDeleteToken(); + return CONJ_AND; + case QueryToken::OR : + //Delete the first token of tokenlist + ExtractAndDeleteToken(); + return CONJ_OR; + default : + return CONJ_NONE; + } + } + + int32_t QueryParser::MatchModifier(){ + //Func - matches for MODIFIER + // MODIFIER ::= <PLUS> | <MINUS> | <NOT> + //Pre - tokens != NULL + //Post - if the first token is a PLUS the token is extracted and deleted and MOD_REQ is returned + // if the first token is a MINUS or NOT the token is extracted and deleted and MOD_NOT is returned + // otherwise MOD_NONE is returned + CND_PRECONDITION(tokens != NULL, "tokens is NULL"); + + switch(tokens->peek()->Type){ + case QueryToken::PLUS : + //Delete the first token of tokenlist + ExtractAndDeleteToken(); + return MOD_REQ; + case QueryToken::MINUS : + case QueryToken::NOT : + //Delete the first token of tokenlist + ExtractAndDeleteToken(); + return MOD_NOT; + default : + return MOD_NONE; + } + } + + Query* QueryParser::MatchQuery(const TCHAR* field){ + //Func - matches for QUERY + // QUERY ::= [MODIFIER] QueryParser::CLAUSE (<CONJUNCTION> [MODIFIER] CLAUSE)* + //Pre - field != NULL + //Post - + + CND_PRECONDITION(tokens != NULL, "tokens is NULL"); + + CL_NS_STD(vector)<BooleanClause*> clauses; + + Query* q = NULL; + + int32_t mods = MOD_NONE; + int32_t conj = CONJ_NONE; + + //match for MODIFIER + mods = MatchModifier(); + + //match for CLAUSE + q = MatchClause(field); + AddClause(clauses, CONJ_NONE, mods, q); + + // match for CLAUSE* + while(true){ + QueryToken* p = tokens->peek(); + if(p->Type == QueryToken::EOF_){ + QueryToken* qt = MatchQueryToken(QueryToken::EOF_); + _CLDELETE(qt); + break; + } + + if(p->Type == QueryToken::RPAREN){ + //MatchQueryToken(QueryToken::RPAREN); + break; + } + + //match for a conjuction (AND OR NOT) + conj = MatchConjunction(); + //match for a modifier + mods = MatchModifier(); + + q = MatchClause(field); + if ( q != NULL ) + AddClause(clauses, conj, mods, q); + } + + // finalize query + if(clauses.size() == 1){ //bvk: removed this && firstQuery != NULL + BooleanClause* c = clauses[0]; + Query* q = c->query; + + //Condition check to be sure clauses[0] is valid + CND_CONDITION(c != NULL, "c is NULL"); + + //Tell the boolean clause not to delete its query + c->deleteQuery=false; + //Clear the clauses list + clauses.clear(); + _CLDELETE(c); + + return q; + }else{ + return GetBooleanQuery(clauses); + } + } + + Query* QueryParser::MatchClause(const TCHAR* field){ + //Func - matches for CLAUSE + // CLAUSE ::= [TERM <COLONQueryParser::>] ( TERM | (<LPAREN> QUERY <RPAREN>)) + //Pre - field != NULL + //Post - + + Query* q = NULL; + const TCHAR* sfield = field; + bool delField = false; + + QueryToken *DelToken = NULL; + + //match for [TERM <COLON>] + QueryToken* term = tokens->extract(); + if(term->Type == QueryToken::TERM && tokens->peek()->Type == QueryToken::COLON){ + DelToken = MatchQueryToken(QueryToken::COLON); + + CND_CONDITION(DelToken != NULL,"DelToken is NULL"); + _CLDELETE(DelToken); + + TCHAR* tmp = STRDUP_TtoT(term->Value); + discardEscapeChar(tmp); + delField = true; + sfield = tmp; + _CLDELETE(term); + }else{ + tokens->push(term); + term = NULL; + } + + // match for + // TERM | (<LPAREN> QUERY <RPAREN>) + if(tokens->peek()->Type == QueryToken::LPAREN){ + DelToken = MatchQueryToken(QueryToken::LPAREN); + + CND_CONDITION(DelToken != NULL,"DelToken is NULL"); + _CLDELETE(DelToken); + + q = MatchQuery(sfield); + //DSR:2004.11.01: + //If exception is thrown while trying to match trailing parenthesis, + //need to prevent q from leaking. + + try{ + DelToken = MatchQueryToken(QueryToken::RPAREN); + + CND_CONDITION(DelToken != NULL,"DelToken is NULL"); + _CLDELETE(DelToken); + + }catch(...) { + _CLDELETE(q); + throw; + } + }else{ + q = MatchTerm(sfield); + } + + if ( delField ) + _CLDELETE_CARRAY(sfield); + return q; + } + + + Query* QueryParser::MatchTerm(const TCHAR* field){ + //Func - matches for TERM + // TERM ::= TERM | PREFIXTERM | WILDTERM | NUMBER + // [ <FUZZY> ] [ <CARAT> <NUMBER> [<FUZZY>]] + // | (<RANGEIN> | <RANGEEX>) [<CARAT> <NUMBER>] + // | <QUOTED> [SLOP] [<CARAT> <NUMBER>] + //Pre - field != NULL + //Post - + + QueryToken* term = NULL; + QueryToken* slop = NULL; + QueryToken* boost = NULL; + + bool prefix = false; + bool wildcard = false; + bool fuzzy = false; + bool rangein = false; + Query* q = NULL; + + term = tokens->extract(); + QueryToken* DelToken = NULL; //Token that is about to be deleted + + switch(term->Type){ + case QueryToken::TERM: + case QueryToken::NUMBER: + case QueryToken::PREFIXTERM: + case QueryToken::WILDTERM: + { //start case + //Check if type of QueryToken term is a prefix term + if(term->Type == QueryToken::PREFIXTERM){ + prefix = true; + } + //Check if type of QueryToken term is a wildcard term + if(term->Type == QueryToken::WILDTERM){ + wildcard = true; + } + //Peek to see if the type of the next token is fuzzy term + if(tokens->peek()->Type == QueryToken::FUZZY){ + DelToken = MatchQueryToken(QueryToken::FUZZY); + + CND_CONDITION(DelToken !=NULL, "DelToken is NULL"); + _CLDELETE(DelToken); + + fuzzy = true; + } + if(tokens->peek()->Type == QueryToken::CARAT){ + DelToken = MatchQueryToken(QueryToken::CARAT); + + CND_CONDITION(DelToken !=NULL, "DelToken is NULL"); + _CLDELETE(DelToken); + + boost = MatchQueryToken(QueryToken::NUMBER); + + if(tokens->peek()->Type == QueryToken::FUZZY){ + DelToken = MatchQueryToken(QueryToken::FUZZY); + + CND_CONDITION(DelToken !=NULL, "DelToken is NULL"); + _CLDELETE(DelToken); + + fuzzy = true; + } + } //end if type==CARAT + + discardEscapeChar(term->Value); //clean up + if(wildcard){ + q = GetWildcardQuery(field,term->Value); + break; + }else if(prefix){ + //Create a PrefixQuery + term->Value[_tcslen(term->Value)-1] = 0; //discard the * + q = GetPrefixQuery(field,term->Value); + break; + }else if(fuzzy){ + //Create a FuzzyQuery + + //Check if the last char is a ~ + if(term->Value[_tcslen(term->Value)-1] == '~'){ + //remove the ~ + term->Value[_tcslen(term->Value)-1] = '\0'; + } + + q = GetFuzzyQuery(field,term->Value); + break; + }else{ + q = GetFieldQuery(field, term->Value); + break; + } + } + + + case QueryToken::RANGEIN: + case QueryToken::RANGEEX:{ + if(term->Type == QueryToken::RANGEIN){ + rangein = true; + } + + if(tokens->peek()->Type == QueryToken::CARAT){ + DelToken = MatchQueryToken(QueryToken::CARAT); + + CND_CONDITION(DelToken !=NULL, "DelToken is NULL"); + _CLDELETE(DelToken); + + boost = MatchQueryToken(QueryToken::NUMBER); + } + + TCHAR* noBrackets = term->Value + 1; + noBrackets[_tcslen(noBrackets)-1] = 0; + q = ParseRangeQuery(field, noBrackets, rangein); + break; + } + + + case QueryToken::QUOTED:{ + if(tokens->peek()->Type == QueryToken::SLOP){ + slop = MatchQueryToken(QueryToken::SLOP); + } + + if(tokens->peek()->Type == QueryToken::CARAT){ + DelToken = MatchQueryToken(QueryToken::CARAT); + + CND_CONDITION(DelToken !=NULL, "DelToken is NULL"); + _CLDELETE(DelToken); + + boost = MatchQueryToken(QueryToken::NUMBER); + } + + //remove the quotes + TCHAR* quotedValue = term->Value+1; + quotedValue[_tcslen(quotedValue)-1] = '\0'; + + int32_t islop = phraseSlop; + if(slop != NULL ){ + try { + TCHAR* end; //todo: should parse using float... + islop = (int32_t)_tcstoi64(slop->Value+1, &end, 10); + }catch(...){ + //ignored + } + } + + q = GetFieldQuery(field, quotedValue, islop); + _CLDELETE(slop); + } + } // end of switch + + _CLDELETE(term); + + + if( q!=NULL && boost != NULL ){ + qreal f = 1.0F; + try { + TCHAR* tmp; + f = _tcstod(boost->Value, &tmp); + }catch(...){ + //ignored + } + _CLDELETE(boost); + + q->setBoost( f); + } + + return q; + } + + QueryToken* QueryParser::MatchQueryToken(QueryToken::Types expectedType){ + //Func - matches for QueryToken of the specified type and returns it + // otherwise Exception throws + //Pre - tokens != NULL + //Post - + + CND_PRECONDITION(tokens != NULL,"tokens is NULL"); + + if(tokens->count() == 0){ + throwParserException(_T("Error: Unexpected end of program"),' ',0,0); + } + + //Extract a token form the TokenList tokens + QueryToken* t = tokens->extract(); + //Check if the type of the token t matches the expectedType + if (expectedType != t->Type){ + TCHAR buf[200]; + _sntprintf(buf,200,_T("Error: Unexpected QueryToken: %d, expected: %d"),t->Type,expectedType); + _CLDELETE(t); + throwParserException(buf,' ',0,0); + } + + //Return the matched token + return t; + } + + void QueryParser::ExtractAndDeleteToken(void){ + //Func - Extracts the first token from the Tokenlist tokenlist + // and destroys it + //Pre - true + //Post - The first token has been extracted and destroyed + + CND_PRECONDITION(tokens != NULL, "tokens is NULL"); + + //Extract the token from the TokenList tokens + QueryToken* t = tokens->extract(); + //Condition Check Token may not be NULL + CND_CONDITION(t != NULL, "Token is NULL"); + //Delete Token + _CLDELETE(t); + } + +CL_NS_END diff --git a/src/assistant/3rdparty/clucene/src/CLucene/queryParser/QueryParser.h b/src/assistant/3rdparty/clucene/src/CLucene/queryParser/QueryParser.h new file mode 100644 index 000000000..a2fc85c89 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/queryParser/QueryParser.h @@ -0,0 +1,165 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _lucene_queryParser_QueryParser_ +#define _lucene_queryParser_QueryParser_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include "CLucene/analysis/AnalysisHeader.h" +#include "CLucene/util/Reader.h" +#include "CLucene/search/SearchHeader.h" +#include "CLucene/index/Term.h" + +#include "TokenList.h" +#include "QueryToken.h" +#include "QueryParserBase.h" +#include "Lexer.h" + +CL_NS_DEF(queryParser) + +/** +* @brief CLucene's default query parser. +* +* <p>It's a query parser. +* The only method that clients should need to call is Parse(). +* The syntax for query const TCHAR*s is as follows: +* A Query is a series of clauses. A clause may be prefixed by:</p> +* <ul> +* <li>a plus (+) or a minus (-) sign, indicating that the +* clause is required or prohibited respectively; or</li> +* <li>a term followed by a colon, indicating the field to be searched. +* This enables one to construct queries which search multiple fields.</li> +* </ul> +* <p> +* A clause may be either:</p> +* <ul> +* <li>a term, indicating all the documents that contain this term; or</li> +* <li>a nested query, enclosed in parentheses. Note that this may be +* used with a +/- prefix to require any of a set of terms.</li> +* </ul> +* <p> +* Thus, in BNF, the query grammar is:</p> +* <code> +* Query ::= ( Clause )* +* Clause ::= ["+", "-"] [<TERM> ":"] ( <TERM> | "(" Query ")" ) +* </code> +* <p> +* Examples of appropriately formatted queries can be found in the test cases. +* </p> +*/ +class QueryParser : public QueryParserBase +{ +private: + const TCHAR* field; + TokenList* tokens; +public: + /** + * Initializes a new instance of the QueryParser class with a specified field and + * analyzer values. + */ + QueryParser(const TCHAR* field, CL_NS(analysis)::Analyzer* analyzer); + ~QueryParser(); + + /** + * Returns a parsed Query instance. + * Note: this call is not threadsafe, either use a seperate QueryParser for each thread, or use a thread lock + * <param name="query">The query value to be parsed.</param> + * <returns>A parsed Query instance.</returns> + */ + virtual CL_NS(search)::Query* parse(const TCHAR* query); + + /** + * Returns a parsed Query instance. + * Note: this call is not threadsafe, either use a seperate QueryParser for each thread, or use a thread lock + * <param name="reader">The TextReader value to be parsed.</param> + * <returns>A parsed Query instance.</returns> + */ + virtual CL_NS(search)::Query* parse(CL_NS(util)::Reader* reader); + + /** + * Returns a new instance of the Query class with a specified query, field and + * analyzer values. + */ + static CL_NS(search)::Query* parse(const TCHAR* query, const TCHAR* field, CL_NS(analysis)::Analyzer* analyzer); + + CL_NS(analysis)::Analyzer* getAnalyzer() { return analyzer; } + + /** + * @return Returns the field. + */ + const TCHAR* getField() { return field; } + + //deprecated functions + _CL_DEPRECATED( setLowercaseExpandedTerms ) void setLowercaseWildcardTerms(bool lowercaseWildcardTerms){ setLowercaseExpandedTerms(lowercaseWildcardTerms); } + _CL_DEPRECATED( getLowercaseExpandedTerms ) bool getLowercaseWildcardTerms() const { return getLowercaseExpandedTerms(); } +protected: + //these functions may be defined under certain compilation conditions. + //note that this functionality is deprecated, you should create your own queryparser + //if you want to remove this functionality...it will be removed... be warned! +#ifdef NO_PREFIX_QUERY + virtual CL_NS(search)::Query* GetPrefixQuery(const TCHAR* field,const TCHAR* termStr){ return NULL; } +#endif +#ifdef NO_FUZZY_QUERY + virtual CL_NS(search)::Query* GetFuzzyQuery(const TCHAR* field,const TCHAR* termStr){ return NULL; } +#endif +#ifdef NO_RANGE_QUERY + virtual CL_NS(search)::Query* GetRangeQuery(const TCHAR* field, const TCHAR* part1, const TCHAR* part2, bool inclusive) { return NULL; } +#endif +#ifdef NO_WILDCARD_QUERY + virtual CL_NS(search)::Query* GetWildcardQuery(const TCHAR* field, TCHAR* termStr) { return NULL; } +#endif +private: + /** + * matches for CONJUNCTION + * CONJUNCTION ::= <AND> | <OR> + */ + int32_t MatchConjunction(); + + /** + * matches for MODIFIER + * MODIFIER ::= <PLUS> | <MINUS> | <NOT> + */ + int32_t MatchModifier(); + + /** + * matches for QUERY + * QUERY ::= [MODIFIER] CLAUSE (<CONJUNCTION> [MODIFIER] CLAUSE)* + */ + CL_NS(search)::Query* MatchQuery(const TCHAR* field); + + /** + * matches for CLAUSE + * CLAUSE ::= [TERM <COLON>] ( TERM | (<LPAREN> QUERY <RPAREN>)) + */ + CL_NS(search)::Query* MatchClause(const TCHAR* field); + + /** + * matches for TERM + * TERM ::= TERM | PREFIXTERM | WILDTERM | NUMBER + * [ <FUZZY> ] [ <CARAT> <NUMBER> [<FUZZY>]] + * + * | (<RANGEIN> | <RANGEEX>) [<CARAT> <NUMBER>] + * | <QUOTED> [SLOP] [<CARAT> <NUMBER>] + */ + CL_NS(search)::Query* MatchTerm(const TCHAR* field); + + /** + * matches for QueryToken of the specified type and returns it + * otherwise Exception throws + */ + QueryToken* MatchQueryToken(QueryToken::Types expectedType); + + /** + * Extracts the first token from the Tokenlist tokenlist + * and destroys it + */ + void ExtractAndDeleteToken(void); +}; +CL_NS_END +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/queryParser/QueryParserBase.cpp b/src/assistant/3rdparty/clucene/src/CLucene/queryParser/QueryParserBase.cpp new file mode 100644 index 000000000..7b95b30f9 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/queryParser/QueryParserBase.cpp @@ -0,0 +1,369 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +* +* Changes are Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies). +------------------------------------------------------------------------------*/ +#include "CLucene/StdHeader.h" +#include "QueryParserBase.h" + +#include "CLucene/search/TermQuery.h" +#include "CLucene/search/PhraseQuery.h" +#include "CLucene/search/RangeQuery.h" +#include "CLucene/search/FuzzyQuery.h" +#include "CLucene/search/WildcardQuery.h" +#include "CLucene/search/PrefixQuery.h" + + +CL_NS_USE(search) +CL_NS_USE(util) +CL_NS_USE(analysis) +CL_NS_USE(index) + +CL_NS_DEF(queryParser) + +QueryParserBase::QueryParserBase(Analyzer* analyzer){ +//Func - Constructor +//Pre - true +//Post - instance has been created with PhraseSlop = 0 + this->analyzer = analyzer; + this->defaultOperator = OR_OPERATOR; + this->phraseSlop = 0; + this->lowercaseExpandedTerms = true; +} + +QueryParserBase::~QueryParserBase(){ +//Func - Destructor +//Pre - true +//Post - The instance has been destroyed +} + + +void QueryParserBase::discardEscapeChar(TCHAR* source) const{ + int len = _tcslen(source); + int j = 0; + for (int i = 0; i < len; i++) { + if (source[i] == '\\' && source[i+1] != '\0' ) { + _tcscpy(source+i,source+i+1); + len--; + } + } +} + +void QueryParserBase::AddClause(CL_NS_STD(vector)<BooleanClause*>& clauses, int32_t conj, int32_t mods, Query* q){ +//Func - Adds the next parsed clause. +//Pre - +//Post - + + bool required, prohibited; + + // If this term is introduced by AND, make the preceding term required, + // unless it's already prohibited. + const uint32_t nPreviousClauses = clauses.size(); + if (nPreviousClauses > 0 && conj == CONJ_AND) { + BooleanClause* c = clauses[nPreviousClauses-1]; + if (!c->prohibited) + c->required = true; + } + + if (nPreviousClauses > 0 && defaultOperator == AND_OPERATOR && conj == CONJ_OR) { + // If this term is introduced by OR, make the preceding term optional, + // unless it's prohibited (that means we leave -a OR b but +a OR b-->a OR b) + // notice if the input is a OR b, first term is parsed as required; without + // this modification a OR b would parse as +a OR b + BooleanClause* c = clauses[nPreviousClauses-1]; + if (!c->prohibited){ + c->required = false; + c->prohibited = false; + } + } + + // We might have been passed a NULL query; the term might have been + // filtered away by the analyzer. + if (q == NULL) + return; + + if (defaultOperator == OR_OPERATOR) { + // We set REQUIRED if we're introduced by AND or +; PROHIBITED if + // introduced by NOT or -; make sure not to set both. + prohibited = (mods == MOD_NOT); + required = (mods == MOD_REQ); + if (conj == CONJ_AND && !prohibited) { + required = true; + } + } else { + // We set PROHIBITED if we're introduced by NOT or -; We set REQUIRED + // if not PROHIBITED and not introduced by OR + prohibited = (mods == MOD_NOT); + required = (!prohibited && conj != CONJ_OR); + } + + if ( required && prohibited ) + throwParserException( _T("Clause cannot be both required and prohibited"), ' ',0,0); + clauses.push_back(_CLNEW BooleanClause(q,true, required, prohibited)); +} + +void QueryParserBase::throwParserException(const TCHAR* message, TCHAR ch, int32_t col, int32_t line ) +{ + TCHAR msg[1024]; + _sntprintf(msg,1024,message,ch,col,line); + _CLTHROWT (CL_ERR_Parse, msg ); +} + + +Query* QueryParserBase::GetFieldQuery(const TCHAR* field, TCHAR* queryText, int32_t slop){ + Query* ret = GetFieldQuery(field,queryText); + if ( ret && ret->getQueryName() == PhraseQuery::getClassName() ) + ((PhraseQuery*)ret)->setSlop(slop); + + return ret; +} + +Query* QueryParserBase::GetFieldQuery(const TCHAR* field, TCHAR* queryText){ +//Func - Returns a query for the specified field. +// Use the analyzer to get all the tokens, and then build a TermQuery, +// PhraseQuery, or nothing based on the term count +//Pre - field != NULL +// analyzer contains a valid reference to an Analyzer +// queryText != NULL and contains the query +//Post - A query instance has been returned for the specified field + + CND_PRECONDITION(field != NULL, "field is NULL"); + CND_PRECONDITION(queryText != NULL, "queryText is NULL"); + + //Instantiate a stringReader for queryText + StringReader reader(queryText); + TokenStream* source = analyzer->tokenStream(field, &reader); + CND_CONDITION(source != NULL,"source is NULL"); + + StringArrayConstWithDeletor v; + + Token t; + int positionCount = 0; + bool severalTokensAtSamePosition = false; + + //Get the tokens from the source + try{ + while (source->next(&t)){ + v.push_back(STRDUP_TtoT(t.termText())); + + if (t.getPositionIncrement() != 0) + positionCount += t.getPositionIncrement(); + else + severalTokensAtSamePosition = true; + } + }catch(CLuceneError& err){ + if ( err.number() != CL_ERR_IO ) + throw err; + } + _CLDELETE(source); + + //Check if there are any tokens retrieved + if (v.size() == 0){ + return NULL; + }else{ + if (v.size() == 1){ + Term* t = _CLNEW Term(field, v[0]); + Query* ret = _CLNEW TermQuery( t ); + _CLDECDELETE(t); + return ret; + }else{ + if (severalTokensAtSamePosition) { + if (positionCount == 1) { + // no phrase query: + BooleanQuery* q = _CLNEW BooleanQuery; //todo: disableCoord=true here, but not implemented in BooleanQuery + StringArrayConst::iterator itr = v.begin(); + while ( itr != v.end() ){ + Term* t = _CLNEW Term(field, *itr); + q->add(_CLNEW TermQuery(t),true, false,false);//should occur... + _CLDECDELETE(t); + ++itr; + } + return q; + }else { + _CLTHROWA(CL_ERR_UnsupportedOperation, "MultiPhraseQuery NOT Implemented"); + } + }else{ + PhraseQuery* q = _CLNEW PhraseQuery; + q->setSlop(phraseSlop); + + StringArrayConst::iterator itr = v.begin(); + while ( itr != v.end() ){ + const TCHAR* data = *itr; + Term* t = _CLNEW Term(field, data); + q->add(t); + _CLDECDELETE(t); + ++itr; + } + return q; + } + } + } +} + +void QueryParserBase::setLowercaseExpandedTerms(bool lowercaseExpandedTerms){ + this->lowercaseExpandedTerms = lowercaseExpandedTerms; +} +bool QueryParserBase::getLowercaseExpandedTerms() const { + return lowercaseExpandedTerms; +} +void QueryParserBase::setDefaultOperator(int oper){ + this->defaultOperator=oper; +} +int QueryParserBase::getDefaultOperator() const{ + return defaultOperator; +} + + +Query* QueryParserBase::ParseRangeQuery(const TCHAR* field, TCHAR* queryText, bool inclusive) +{ + //todo: this must be fixed, [-1--5] (-1 to -5) should yield a result, but won't parse properly + //because it uses an analyser, should split it up differently... + + // Use the analyzer to get all the tokens. There should be 1 or 2. + StringReader reader(queryText); + TokenStream* source = analyzer->tokenStream(field, &reader); + + TCHAR* terms[2]; + terms[0]=NULL;terms[1]=NULL; + Token t; + bool tret=true; + bool from=true; + while(tret) + { + try{ + tret = source->next(&t); + }catch (CLuceneError& err){ + if ( err.number() == CL_ERR_IO ) + tret=false; + else + throw err; + } + if (tret) + { + if ( !from && _tcscmp(t.termText(),_T("TO"))==0 ) + continue; + + + TCHAR* tmp = STRDUP_TtoT(t.termText()); + discardEscapeChar(tmp); + terms[from? 0 : 1] = tmp; + + if (from) + from = false; + else + break; + } + } + Query* ret = GetRangeQuery(field, terms[0], terms[1],inclusive); + _CLDELETE_CARRAY(terms[0]); + _CLDELETE_CARRAY(terms[1]); + _CLDELETE(source); + + return ret; +} + +Query* QueryParserBase::GetPrefixQuery(const TCHAR* field, TCHAR* termStr){ +//Pre - field != NULL and field contains the name of the field that the query will use +// termStr != NULL and is the token to use for building term for the query +// (WITH or WITHOUT a trailing '*' character!) +//Post - A PrefixQuery instance has been returned + + CND_PRECONDITION(field != NULL,"field is NULL"); + CND_PRECONDITION(termStr != NULL,"termStr is NULL"); + + if ( lowercaseExpandedTerms ) + _tcslwr(termStr); + + Term* t = _CLNEW Term(field, termStr); + CND_CONDITION(t != NULL,"Could not allocate memory for term t"); + + Query *q = _CLNEW PrefixQuery(t); + CND_CONDITION(q != NULL,"Could not allocate memory for PrefixQuery q"); + + _CLDECDELETE(t); + return q; +} + +Query* QueryParserBase::GetFuzzyQuery(const TCHAR* field, TCHAR* termStr){ +//Func - Factory method for generating a query (similar to getPrefixQuery}). Called when parser parses +// an input term token that has the fuzzy suffix (~) appended. +//Pre - field != NULL and field contains the name of the field that the query will use +// termStr != NULL and is the token to use for building term for the query +// (WITH or WITHOUT a trailing '*' character!) +//Post - A FuzzyQuery instance has been returned + + CND_PRECONDITION(field != NULL,"field is NULL"); + CND_PRECONDITION(termStr != NULL,"termStr is NULL"); + + if ( lowercaseExpandedTerms ) + _tcslwr(termStr); + + Term* t = _CLNEW Term(field, termStr); + CND_CONDITION(t != NULL,"Could not allocate memory for term t"); + + Query *q = _CLNEW FuzzyQuery(t); + CND_CONDITION(q != NULL,"Could not allocate memory for FuzzyQuery q"); + + _CLDECDELETE(t); + return q; +} + + +Query* QueryParserBase::GetWildcardQuery(const TCHAR* field, TCHAR* termStr){ + CND_PRECONDITION(field != NULL,"field is NULL"); + CND_PRECONDITION(termStr != NULL,"termStr is NULL"); + + if ( lowercaseExpandedTerms ) + _tcslwr(termStr); + + Term* t = _CLNEW Term(field, termStr); + CND_CONDITION(t != NULL,"Could not allocate memory for term t"); + Query* q = _CLNEW WildcardQuery(t); + _CLDECDELETE(t); + + return q; +} + +Query* QueryParserBase::GetBooleanQuery(CL_NS_STD(vector)<CL_NS(search)::BooleanClause*>& clauses){ + if ( clauses.size() == 0 ) + return NULL; + + BooleanQuery* query = _CLNEW BooleanQuery(); + //Condition check to see if query has been allocated properly + CND_CONDITION(query != NULL, "No memory could be allocated for query"); + + //iterate through all the clauses + for( uint32_t i=0;i<clauses.size();i++ ){ + //Condition check to see if clauses[i] is valdid + CND_CONDITION(clauses[i] != NULL, "clauses[i] is NULL"); + //Add it to query + query->add(clauses[i]); + } + return query; +} + + +CL_NS(search)::Query* QueryParserBase::GetRangeQuery(const TCHAR* field, TCHAR* part1, TCHAR* part2, bool inclusive){ + //todo: does jlucene handle rangequeries differntly? if we are using + //a certain type of analyser, the terms may be filtered out, which + //is not necessarily what we want. + if (lowercaseExpandedTerms) { + _tcslwr(part1); + _tcslwr(part2); + } + //todo: should see if we can parse the strings as dates... currently we leave that up to the end-developer... + Term* t1 = _CLNEW Term(field,part1); + Term* t2 = _CLNEW Term(field,part2); + Query* ret = _CLNEW RangeQuery(t1, t2, inclusive); + _CLDECDELETE(t1); + _CLDECDELETE(t2); + + return ret; +} + + + +CL_NS_END diff --git a/src/assistant/3rdparty/clucene/src/CLucene/queryParser/QueryParserBase.h b/src/assistant/3rdparty/clucene/src/CLucene/queryParser/QueryParserBase.h new file mode 100644 index 000000000..261e587b0 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/queryParser/QueryParserBase.h @@ -0,0 +1,204 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _lucene_queryParser_QueryParserBase_ +#define _lucene_queryParser_QueryParserBase_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include "CLucene/util/VoidList.h" +#include "CLucene/search/BooleanClause.h" +#include "CLucene/analysis/Analyzers.h" +#include "QueryToken.h" + +CL_NS_DEF(queryParser) + + /** + * Contains default implementations used by QueryParser. + * You can override any of these to provide a customised QueryParser. + */ + class QueryParserBase:LUCENE_BASE + { + protected: + /* The actual operator the parser uses to combine query terms */ + int defaultOperator; + int32_t phraseSlop; + + bool lowercaseExpandedTerms; + + LUCENE_STATIC_CONSTANT(int, CONJ_NONE=0); + LUCENE_STATIC_CONSTANT(int, CONJ_AND=1); + LUCENE_STATIC_CONSTANT(int, CONJ_OR=2); + + LUCENE_STATIC_CONSTANT(int, MOD_NONE=0); + LUCENE_STATIC_CONSTANT(int, MOD_NOT=10); + LUCENE_STATIC_CONSTANT(int, MOD_REQ=11); + + CL_NS(analysis)::Analyzer* analyzer; + + public: + QueryParserBase(CL_NS(analysis)::Analyzer* analyzer); + ~QueryParserBase(); + + /** + * Whether terms of wildcard, prefix, fuzzy and range queries are to be automatically + * lower-cased or not. Default is <code>true</code>. + */ + void setLowercaseExpandedTerms(bool lowercaseExpandedTerms); + + /** + * @see #setLowercaseExpandedTerms(boolean) + */ + bool getLowercaseExpandedTerms() const; + + //values used for setOperator + LUCENE_STATIC_CONSTANT(int, OR_OPERATOR=0); + LUCENE_STATIC_CONSTANT(int, AND_OPERATOR=1); + + /** + * Sets the boolean operator of the QueryParser. + * In default mode (<code>OR_OPERATOR</code>) terms without any modifiers + * are considered optional: for example <code>capital of Hungary</code> is equal to + * <code>capital OR of OR Hungary</code>.<br/> + * In <code>AND_OPERATOR</code> mode terms are considered to be in conjuction: the + * above mentioned query is parsed as <code>capital AND of AND Hungary</code> + */ + void setDefaultOperator(int oper); + /** + * Gets implicit operator setting, which will be either AND_OPERATOR + * or OR_OPERATOR. + */ + int getDefaultOperator() const; + + //public so that the lexer can call this + virtual void throwParserException(const TCHAR* message, TCHAR ch, int32_t col, int32_t line ); + + /** + * Sets the default slop for phrases. If zero, then exact phrase matches + * are required. Default value is zero. + */ + void setPhraseSlop(int phraseSlop) { this->phraseSlop = phraseSlop; } + + /** + * Gets the default slop for phrases. + */ + int getPhraseSlop() { return phraseSlop; } + + protected: + + /** + * Removes the escaped characters + */ + void discardEscapeChar(TCHAR* token) const; + + //Analyzes the expanded term termStr with the StandardFilter and the LowerCaseFilter. + TCHAR* AnalyzeExpandedTerm(const TCHAR* field, TCHAR* termStr); + + // Adds the next parsed clause. + virtual void AddClause(std::vector<CL_NS(search)::BooleanClause*>& clauses, int32_t conj, int32_t mods, CL_NS(search)::Query* q); + + /** + * Returns a termquery, phrasequery for the specified field. + * Note: this is only a partial implementation, since MultiPhraseQuery is not implemented yet + * return NULL to disallow + */ + virtual CL_NS(search)::Query* GetFieldQuery(const TCHAR* field, TCHAR* queryText); + + /** + * Delegates to GetFieldQuery(string, string), and adds slop onto phrasequery. + * Can be used to remove slop functionality + */ + virtual CL_NS(search)::Query* GetFieldQuery(const TCHAR* field, TCHAR* queryText, int32_t slop); + + /** + * Factory method for generating a query (similar to + * {@link #GetWildcardQuery}). Called when parser parses an input term + * token that uses prefix notation; that is, contains a single '*' wildcard + * character as its last character. Since this is a special case + * of generic wildcard term, and such a query can be optimized easily, + * this usually results in a different query object. + *<p> + * Depending on settings, a prefix term may be lower-cased + * automatically. It will not go through the default Analyzer, + * however, since normal Analyzers are unlikely to work properly + * with wildcard templates. + *<p> + * Can be overridden by extending classes, to provide custom handling for + * wild card queries, which may be necessary due to missing analyzer calls. + * + * @param field Name of the field query will use. + * @param termStr Term token to use for building term for the query + * (<b>without</b> trailing '*' character!) + * + * @return Resulting {@link Query} built for the term + * return NULL to disallow + */ + virtual CL_NS(search)::Query* GetPrefixQuery(const TCHAR* field, TCHAR* termStr); + + /** + * Factory method for generating a query. Called when parser + * parses an input term token that contains one or more wildcard + * characters (? and *), but is not a prefix term token (one + * that has just a single * character at the end) + *<p> + * Depending on settings, prefix term may be lower-cased + * automatically. It will not go through the default Analyzer, + * however, since normal Analyzers are unlikely to work properly + * with wildcard templates. + *<p> + * Can be overridden by extending classes, to provide custom handling for + * wildcard queries, which may be necessary due to missing analyzer calls. + * + * @param field Name of the field query will use. + * @param termStr Term token that contains one or more wild card + * characters (? or *), but is not simple prefix term + * + * @return Resulting {@link Query} built for the term + * return NULL to disallow + */ + virtual CL_NS(search)::Query* GetWildcardQuery(const TCHAR* field, TCHAR* termStr); + + /** + * Factory method for generating a query (similar to + * {@link #GetWildcardQuery}). Called when parser parses + * an input term token that has the fuzzy suffix (~) appended. + * + * @param field Name of the field query will use. + * @param termStr Term token to use for building term for the query + * + * @return Resulting {@link Query} built for the term + * return NULL to disallow + */ + virtual CL_NS(search)::Query* GetFuzzyQuery(const TCHAR* field, TCHAR* termStr); + + /** + * Factory method for generating query, given a set of clauses. + * By default creates a boolean query composed of clauses passed in. + * + * Can be overridden by extending classes, to modify query being + * returned. + * + * @param clauses Vector that contains {@link BooleanClause} instances + * to join. + * + * @return Resulting {@link Query} object. + * return NULL to disallow + * + * Memory: clauses must all be cleaned up by this function. + */ + virtual CL_NS(search)::Query* GetBooleanQuery(std::vector<CL_NS(search)::BooleanClause*>& clauses); + + /** + * return NULL to disallow + */ + virtual CL_NS(search)::Query* GetRangeQuery(const TCHAR* field, TCHAR* part1, TCHAR* part2, bool inclusive); + virtual CL_NS(search)::Query* ParseRangeQuery(const TCHAR* field, TCHAR* str, bool inclusive); + + }; +CL_NS_END +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/queryParser/QueryToken.cpp b/src/assistant/3rdparty/clucene/src/CLucene/queryParser/QueryToken.cpp new file mode 100644 index 000000000..ee88a3cb6 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/queryParser/QueryToken.cpp @@ -0,0 +1,73 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#include "CLucene/StdHeader.h" +#include "QueryToken.h" + +CL_NS_DEF(queryParser) + + +QueryToken::QueryToken(): + Value(NULL) +{ + set(UNKNOWN_); +} +QueryToken::QueryToken(TCHAR* value, const int32_t start, const int32_t end, const QueryToken::Types type): + Value(NULL) +{ + set(value,start,end,type); +} + +QueryToken::~QueryToken(){ +//Func - Destructor +//Pre - true +//Post - Instance has been destroyed + + #ifndef LUCENE_TOKEN_WORD_LENGTH + _CLDELETE_CARRAY( Value ); + #endif +} + +// Initializes a new instance of the Token class LUCENE_EXPORT. +// +QueryToken::QueryToken(TCHAR* value, const QueryToken::Types type): + Value(NULL) +{ + set(value,type); +} + +// Initializes a new instance of the Token class LUCENE_EXPORT. +// +QueryToken::QueryToken(QueryToken::Types type): + Value(NULL) +{ + set(type); +} + + +void QueryToken::set(TCHAR* value, const Types type){ + set(value,0,-1,type); +} +void QueryToken::set(TCHAR* value, const int32_t start, const int32_t end, const Types type){ + #ifndef LUCENE_TOKEN_WORD_LENGTH + _CLDELETE_CARRAY(Value); + Value = STRDUP_TtoT(value); + #else + _tcsncpy(Value,value,LUCENE_TOKEN_WORD_LENGTH); + Value[LUCENE_TOKEN_WORD_LENGTH]; + #endif + this->Start = start; + this->End = end; + this->Type = type; + + if ( this->End < 0 ) + this->End = _tcslen(Value); +} +void QueryToken::set(Types type){ + set(LUCENE_BLANK_STRING,0,0,type); +} + +CL_NS_END diff --git a/src/assistant/3rdparty/clucene/src/CLucene/queryParser/QueryToken.h b/src/assistant/3rdparty/clucene/src/CLucene/queryParser/QueryToken.h new file mode 100644 index 000000000..739a667ba --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/queryParser/QueryToken.h @@ -0,0 +1,76 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _lucene_queryParser_QueryToken_ +#define _lucene_queryParser_QueryToken_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include "QueryParserBase.h" + +CL_NS_DEF(queryParser) + + // Token class that used by QueryParser. + class QueryToken:LUCENE_BASE + { + public: + enum Types + { + AND_, + OR, + NOT, + PLUS, + MINUS, + LPAREN, + RPAREN, + COLON, + CARAT, + QUOTED, + TERM, + SLOP, + FUZZY, + PREFIXTERM, + WILDTERM, + RANGEIN, + RANGEEX, + NUMBER, + EOF_, + UNKNOWN_ + }; + + + #ifdef LUCENE_TOKEN_WORD_LENGTH + TCHAR Value[LUCENE_TOKEN_WORD_LENGTH+1]; + #else + TCHAR* Value; + #endif + + int32_t Start; + int32_t End; + QueryToken::Types Type; + + // Initializes a new instance of the Token class. + QueryToken(TCHAR* value, const int32_t start, const int32_t end, const Types type); + + // Initializes a new instance of the Token class. + QueryToken(TCHAR* value, const Types type); + + // Initializes a new instance of the Token class. + QueryToken(Types type); + + // Initializes an empty instance of the Token class. + QueryToken(); + + ~QueryToken(); + + void set(TCHAR* value, const int32_t start, const int32_t end, const Types type); + void set(TCHAR* value, const Types type); + void set(Types type); + }; +CL_NS_END +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/queryParser/TokenList.cpp b/src/assistant/3rdparty/clucene/src/CLucene/queryParser/TokenList.cpp new file mode 100644 index 000000000..7d30b931f --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/queryParser/TokenList.cpp @@ -0,0 +1,79 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#include "CLucene/StdHeader.h" +#include "TokenList.h" + +#include "CLucene/util/VoidMap.h" +#include "CLucene/util/VoidList.h" +#include "QueryToken.h" + +CL_NS_DEF(queryParser) + + TokenList::TokenList(){ + //Func - Constructor + //Pre - true + //Post - Instance has been created + } + + TokenList::~TokenList(){ + //Func - Destructor + //Pre - true + //Post - The tokenlist has been destroyed + + tokens.clear(); + } + + void TokenList::add(QueryToken* token){ + //Func - Adds a QueryToken token to the TokenList + //Pre - token != NULL + //Post - token has been added to the token list + + CND_PRECONDITION(token != NULL, "token != NULL"); + + tokens.insert(tokens.begin(),token); + } + + void TokenList::push(QueryToken* token){ + //Func - + //Pre - token != NULL + //Post - + + CND_PRECONDITION(token != NULL, "token is NULL"); + + tokens.push_back(token); + } + + QueryToken* TokenList::peek() { + /* DSR:2004.11.01: Reverted my previous (circa April 2004) fix (which + ** raised an exception if Peek was called when there were no tokens) in + ** favor of returning the EOF token. This solution is much better + ** integrated with the rest of the code in the queryParser subsystem. */ + size_t nTokens = tokens.size(); + if (nTokens == 0) { + push(_CLNEW QueryToken(QueryToken::EOF_)); + nTokens++; + } + return tokens[nTokens-1]; + } + + QueryToken* TokenList::extract(){ + //Func - Extract token from the TokenList + //Pre - true + //Post - Retracted token has been returned + + QueryToken* token = peek(); + //Retract the current peeked token + tokens.delete_back(); + + return token; + } + + int32_t TokenList::count() const + { + return tokens.size(); + } +CL_NS_END diff --git a/src/assistant/3rdparty/clucene/src/CLucene/queryParser/TokenList.h b/src/assistant/3rdparty/clucene/src/CLucene/queryParser/TokenList.h new file mode 100644 index 000000000..3166bba78 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/queryParser/TokenList.h @@ -0,0 +1,38 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _lucene_queryParser_TokenList_ +#define _lucene_queryParser_TokenList_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include "CLucene/util/VoidList.h" +#include "QueryToken.h" +CL_NS_DEF(queryParser) + + // Represents a list of the tokens. + class TokenList:LUCENE_BASE + { + private: + CL_NS(util)::CLVector<QueryToken*> tokens; //todo:,CL_NS(util)::Deletor::Object<QueryToken> + public: + TokenList(); + ~TokenList(); + + void add(QueryToken* token); + + void push(QueryToken* token); + + QueryToken* peek(); + + QueryToken* extract(); + + int32_t count() const; + }; +CL_NS_END +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/search/BooleanClause.h b/src/assistant/3rdparty/clucene/src/CLucene/search/BooleanClause.h new file mode 100644 index 000000000..b89cb31d7 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/search/BooleanClause.h @@ -0,0 +1,90 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _lucene_search_BooleanClause_ +#define _lucene_search_BooleanClause_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif +#include "SearchHeader.h" + +CL_NS_DEF(search) + // A clause in a BooleanQuery. + class BooleanClause:LUCENE_BASE { + public: + class Compare:public CL_NS_STD(binary_function)<const BooleanClause*,const BooleanClause*,bool> + { + public: + bool operator()( const BooleanClause* val1, const BooleanClause* val2 ) const{ + return val1->equals(val2); + } + }; + + // The query whose matching documents are combined by the boolean query. + Query* query; + + int32_t getClauseCount(); + + // If true, documents documents which <i>do not</i> + // match this sub-query will <i>not</i> match the boolean query. + bool required; + + // If true, documents documents which <i>do</i> + // match this sub-query will <i>not</i> match the boolean query. + bool prohibited; + + bool deleteQuery; + + // Constructs a BooleanClause with query <code>q</code>, required + // <code>r</code> and prohibited <code>p</code>. + BooleanClause(Query* q, const bool DeleteQuery,const bool req, const bool p): + query(q), + required(req), + prohibited(p), + deleteQuery(DeleteQuery) + { + } + + BooleanClause(const BooleanClause& clone): +#if defined(LUCENE_ENABLE_MEMLEAKTRACKING) +#elif defined(LUCENE_ENABLE_REFCOUNT) +#else + LuceneVoidBase(), +#endif + query(clone.query->clone()), + required(clone.required), + prohibited(clone.prohibited), + deleteQuery(true) + { + } + + BooleanClause* clone() const{ + BooleanClause* ret = _CLNEW BooleanClause(*this); + return ret; + } + + ~BooleanClause(){ + if ( deleteQuery ) + _CLDELETE( query ); + } + + /** Returns true iff <code>o</code> is equal to this. */ + bool equals(const BooleanClause* other) const { + return this->query->equals(other->query) + && (this->required == other->required) + && (this->prohibited == other->prohibited); + } + + size_t hashCode() const{ + return query->hashCode() ^ (this->required?1:0) ^ (this->prohibited?2:0); + } + }; + + +CL_NS_END +#endif + diff --git a/src/assistant/3rdparty/clucene/src/CLucene/search/BooleanQuery.cpp b/src/assistant/3rdparty/clucene/src/CLucene/search/BooleanQuery.cpp new file mode 100644 index 000000000..3fd36d847 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/search/BooleanQuery.cpp @@ -0,0 +1,363 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#include "CLucene/StdHeader.h" +#include "BooleanQuery.h" + +#include "BooleanClause.h" +#include "CLucene/index/IndexReader.h" +#include "CLucene/util/StringBuffer.h" +#include "CLucene/util/Arrays.h" +#include "SearchHeader.h" +#include "BooleanScorer.h" +#include "Scorer.h" + +CL_NS_USE(index) +CL_NS_USE(util) +CL_NS_DEF(search) + + BooleanQuery::BooleanQuery(): + clauses(true) + { + } + + BooleanQuery::BooleanQuery(const BooleanQuery& clone): + Query(clone) + { + for ( uint32_t i=0;i<clone.clauses.size();i++ ){ + BooleanClause* clause = clone.clauses[i]->clone(); + clause->deleteQuery=true; + add(clause); + } + } + + BooleanQuery::~BooleanQuery(){ + clauses.clear(); + } + + size_t BooleanQuery::hashCode() const { + //todo: do cachedHashCode, and invalidate on add/remove clause + size_t ret = 0; + for (uint32_t i = 0 ; i < clauses.size(); i++) { + BooleanClause* c = clauses[i]; + ret = 31 * ret + c->hashCode(); + } + ret = ret ^ Similarity::floatToByte(getBoost()); + return ret; + } + + const TCHAR* BooleanQuery::getQueryName() const{ + return getClassName(); + } + const TCHAR* BooleanQuery::getClassName(){ + return _T("BooleanQuery"); + } + + /** + * Default value is 1024. Use <code>org.apache.lucene.maxClauseCount</code> + * system property to override. + */ + size_t BooleanQuery::maxClauseCount = LUCENE_BOOLEANQUERY_MAXCLAUSECOUNT; + size_t BooleanQuery::getMaxClauseCount(){ + return maxClauseCount; + } + + void BooleanQuery::setMaxClauseCount(size_t maxClauseCount){ + BooleanQuery::maxClauseCount = maxClauseCount; + } + + void BooleanQuery::add(Query* query, const bool deleteQuery, const bool required, const bool prohibited) { + BooleanClause* bc = _CLNEW BooleanClause(query,deleteQuery,required, prohibited); + try{ + add(bc); + }catch(...){ + _CLDELETE(bc); + throw; + } + } + + void BooleanQuery::add(BooleanClause* clause) { + if (clauses.size() >= getMaxClauseCount()) + _CLTHROWA(CL_ERR_TooManyClauses,"Too Many Clauses"); + + clauses.push_back(clause); + } + + + size_t BooleanQuery::getClauseCount() const { + return (int32_t) clauses.size(); + } + + TCHAR* BooleanQuery::toString(const TCHAR* field) const{ + StringBuffer buffer; + if (getBoost() != 1.0) { + buffer.append(_T("(")); + } + + for (uint32_t i = 0 ; i < clauses.size(); i++) { + BooleanClause* c = clauses[i]; + if (c->prohibited) + buffer.append(_T("-")); + else if (c->required) + buffer.append(_T("+")); + + if ( c->query->instanceOf(BooleanQuery::getClassName()) ) { // wrap sub-bools in parens + buffer.append(_T("(")); + + TCHAR* buf = c->query->toString(field); + buffer.append(buf); + _CLDELETE_CARRAY( buf ); + + buffer.append(_T(")")); + } else { + TCHAR* buf = c->query->toString(field); + buffer.append(buf); + _CLDELETE_CARRAY( buf ); + } + if (i != clauses.size()-1) + buffer.append(_T(" ")); + + if (getBoost() != 1.0) { + buffer.append(_T(")^")); + buffer.appendFloat(getBoost(),1); + } + } + return buffer.toString(); + } + + + + + BooleanClause** BooleanQuery::getClauses() const + { + CND_MESSAGE(false, "Warning: BooleanQuery::getClauses() is deprecated") + BooleanClause** ret = _CL_NEWARRAY(BooleanClause*, clauses.size()+1); + getClauses(ret); + return ret; + } + + void BooleanQuery::getClauses(BooleanClause** ret) const + { + size_t size=clauses.size(); + for ( uint32_t i=0;i<size;i++ ) + ret[i] = clauses[i]; + } + Query* BooleanQuery::rewrite(IndexReader* reader) { + if (clauses.size() == 1) { // optimize 1-clause queries + BooleanClause* c = clauses[0]; + if (!c->prohibited) { // just return clause + Query* query = c->query->rewrite(reader); // rewrite first + + //if the query doesn't actually get re-written, + //then return a clone (because the BooleanQuery + //will register different to the returned query. + if ( query == c->query ) + query = query->clone(); + + if (getBoost() != 1.0f) { // incorporate boost + query->setBoost(getBoost() * query->getBoost()); + } + + return query; + } + } + + BooleanQuery* clone = NULL; // recursively rewrite + for (uint32_t i = 0 ; i < clauses.size(); i++) { + BooleanClause* c = clauses[i]; + Query* query = c->query->rewrite(reader); + if (query != c->query) { // clause rewrote: must clone + if (clone == NULL) + clone = (BooleanQuery*)this->clone(); + //todo: check if delete query should be on... + //in fact we should try and get rid of these + //for compatibility sake + clone->clauses.set (i, _CLNEW BooleanClause(query, true, c->required, c->prohibited)); + } + } + if (clone != NULL) { + return clone; // some clauses rewrote + } else + return this; // no clauses rewrote + } + + + Query* BooleanQuery::clone() const{ + BooleanQuery* clone = _CLNEW BooleanQuery(*this); + return clone; + } + + /** Returns true iff <code>o</code> is equal to this. */ + bool BooleanQuery::equals(Query* o)const { + if (!(o->instanceOf(BooleanQuery::getClassName()))) + return false; + const BooleanQuery* other = (BooleanQuery*)o; + + bool ret = (this->getBoost() == other->getBoost()); + if ( ret ){ + CLListEquals<BooleanClause,BooleanClause::Compare, const ClausesType, const ClausesType> comp; + ret = comp.equals(&this->clauses,&other->clauses); + } + return ret; + } + + qreal BooleanQuery::BooleanWeight::getValue() { return parentQuery->getBoost(); } + Query* BooleanQuery::BooleanWeight::getQuery() { return (Query*)parentQuery; } + + + + + + BooleanQuery::BooleanWeight::BooleanWeight(Searcher* searcher, + CLVector<BooleanClause*,Deletor::Object<BooleanClause> >* clauses, BooleanQuery* parentQuery) + { + this->searcher = searcher; + this->parentQuery = parentQuery; + this->clauses = clauses; + for (uint32_t i = 0 ; i < clauses->size(); i++) { + weights.push_back((*clauses)[i]->query->_createWeight(searcher)); + } + } + BooleanQuery::BooleanWeight::~BooleanWeight(){ + this->weights.clear(); + } + + qreal BooleanQuery::BooleanWeight::sumOfSquaredWeights() { + qreal sum = 0.0f; + for (uint32_t i = 0 ; i < weights.size(); i++) { + BooleanClause* c = (*clauses)[i]; + Weight* w = weights[i]; + if (!c->prohibited) + sum += w->sumOfSquaredWeights(); // sum sub weights + } + sum *= parentQuery->getBoost() * parentQuery->getBoost(); // boost each sub-weight + return sum ; + } + + void BooleanQuery::BooleanWeight::normalize(qreal norm) { + norm *= parentQuery->getBoost(); // incorporate boost + for (uint32_t i = 0 ; i < weights.size(); i++) { + BooleanClause* c = (*clauses)[i]; + Weight* w = weights[i]; + if (!c->prohibited) + w->normalize(norm); + } + } + + Scorer* BooleanQuery::BooleanWeight::scorer(IndexReader* reader){ + // First see if the (faster) ConjunctionScorer will work. This can be + // used when all clauses are required. Also, at this point a + // BooleanScorer cannot be embedded in a ConjunctionScorer, as the hits + // from a BooleanScorer are not always sorted by document number (sigh) + // and hence BooleanScorer cannot implement skipTo() correctly, which is + // required by ConjunctionScorer. + bool allRequired = true; + bool noneBoolean = true; + { //msvc6 scope fix + for (uint32_t i = 0 ; i < weights.size(); i++) { + BooleanClause* c = (*clauses)[i]; + if (!c->required) + allRequired = false; + if (c->query->instanceOf(BooleanQuery::getClassName())) + noneBoolean = false; + } + } + + if (allRequired && noneBoolean) { // ConjunctionScorer is okay + ConjunctionScorer* result = + _CLNEW ConjunctionScorer(parentQuery->getSimilarity(searcher)); + for (uint32_t i = 0 ; i < weights.size(); i++) { + Weight* w = weights[i]; + Scorer* subScorer = w->scorer(reader); + if (subScorer == NULL) + return NULL; + result->add(subScorer); + } + return result; + } + + // Use good-old BooleanScorer instead. + BooleanScorer* result = _CLNEW BooleanScorer(parentQuery->getSimilarity(searcher)); + + { //msvc6 scope fix + for (uint32_t i = 0 ; i < weights.size(); i++) { + BooleanClause* c = (*clauses)[i]; + Weight* w = weights[i]; + Scorer* subScorer = w->scorer(reader); + if (subScorer != NULL) + result->add(subScorer, c->required, c->prohibited); + else if (c->required) + return NULL; + } + } + + return result; + } + + void BooleanQuery::BooleanWeight::explain(IndexReader* reader, int32_t doc, Explanation* result){ + int32_t coord = 0; + int32_t maxCoord = 0; + qreal sum = 0.0f; + Explanation* sumExpl = _CLNEW Explanation; + for (uint32_t i = 0 ; i < weights.size(); i++) { + BooleanClause* c = (*clauses)[i]; + Weight* w = weights[i]; + Explanation* e = _CLNEW Explanation; + w->explain(reader, doc, e); + if (!c->prohibited) + maxCoord++; + if (e->getValue() > 0) { + if (!c->prohibited) { + sumExpl->addDetail(e); + sum += e->getValue(); + coord++; + e = NULL; //prevent e from being deleted + } else { + //we want to return something else... + _CLDELETE(sumExpl); + result->setValue(0.0f); + result->setDescription(_T("match prohibited")); + return; + } + } else if (c->required) { + _CLDELETE(sumExpl); + result->setValue(0.0f); + result->setDescription(_T("match prohibited")); + return; + } + + _CLDELETE(e); + } + sumExpl->setValue(sum); + + if (coord == 1){ // only one clause matched + Explanation* tmp = sumExpl; + sumExpl = sumExpl->getDetail(0)->clone(); // eliminate wrapper + _CLDELETE(tmp); + } + + sumExpl->setDescription(_T("sum of:")); + qreal coordFactor = parentQuery->getSimilarity(searcher)->coord(coord, maxCoord); + if (coordFactor == 1.0f){ // coord is no-op + result->set(*sumExpl); // eliminate wrapper + _CLDELETE(sumExpl); + } else { + result->setDescription( _T("product of:")); + result->addDetail(sumExpl); + + StringBuffer explbuf; + explbuf.append(_T("coord(")); + explbuf.appendInt(coord); + explbuf.append(_T("/")); + explbuf.appendInt(maxCoord); + explbuf.append(_T(")")); + result->addDetail(_CLNEW Explanation(coordFactor, explbuf.getBuffer())); + result->setValue(sum*coordFactor); + } + } + + +CL_NS_END diff --git a/src/assistant/3rdparty/clucene/src/CLucene/search/BooleanQuery.h b/src/assistant/3rdparty/clucene/src/CLucene/search/BooleanQuery.h new file mode 100644 index 000000000..27b67d1e5 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/search/BooleanQuery.h @@ -0,0 +1,126 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _lucene_search_BooleanQuery_ +#define _lucene_search_BooleanQuery_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include "ConjunctionScorer.h" +#include "CLucene/index/IndexReader.h" +#include "CLucene/util/StringBuffer.h" +#include "SearchHeader.h" +#include "BooleanClause.h" +#include "BooleanScorer.h" +#include "Scorer.h" + +CL_NS_DEF(search) + + + // A Query that matches documents matching boolean combinations of other + // queries, typically {@link TermQuery}s or {@link PhraseQuery}s. + class BooleanQuery:public Query { + public: + typedef CL_NS(util)::CLVector<BooleanClause*,CL_NS(util)::Deletor::Object<BooleanClause> > ClausesType; + private: + BooleanQuery::ClausesType clauses; + static size_t maxClauseCount; + + class BooleanWeight: public Weight { + private: + Searcher* searcher; + CL_NS(util)::CLVector<Weight*,CL_NS(util)::Deletor::Object<Weight> > weights; + ClausesType* clauses; + BooleanQuery* parentQuery; + public: + BooleanWeight(Searcher* searcher, + CL_NS(util)::CLVector<BooleanClause*,CL_NS(util)::Deletor::Object<BooleanClause> >* clauses, + BooleanQuery* parentQuery); + ~BooleanWeight(); + Query* getQuery(); + qreal getValue(); + qreal sumOfSquaredWeights(); + void normalize(qreal norm); + Scorer* scorer(CL_NS(index)::IndexReader* reader); + void explain(CL_NS(index)::IndexReader* reader, int32_t doc, Explanation* ret); + };//booleanweight + + protected: + Weight* _createWeight(Searcher* searcher) { + return _CLNEW BooleanWeight(searcher,&clauses,this); + } + BooleanQuery(const BooleanQuery& clone); + public: + /** Constructs an empty boolean query. */ + BooleanQuery(); + + ~BooleanQuery(); + + const TCHAR* getQueryName() const; + static const TCHAR* getClassName(); + + /** Return the maximum number of clauses permitted, 1024 by default. + * Attempts to add more than the permitted number of clauses cause {@link + * TooManyClauses} to be thrown.*/ + static size_t getMaxClauseCount(); + + /** Set the maximum number of clauses permitted. */ + static void setMaxClauseCount(size_t maxClauseCount); + + /** Adds a clause to a boolean query. Clauses may be: + * <ul> + * <li><code>required</code> which means that documents which <i>do not</i> + * match this sub-query will <i>not</i> match the boolean query; + * <li><code>prohibited</code> which means that documents which <i>do</i> + * match this sub-query will <i>not</i> match the boolean query; or + * <li>neither, in which case matched documents are neither prohibited from + * nor required to match the sub-query. However, a document must match at + * least 1 sub-query to match the boolean query. + * </ul> + * It is an error to specify a clause as both <code>required</code> and + * <code>prohibited</code>. + * + * @see #getMaxClauseCount() + */ + void add(Query* query, const bool required, const bool prohibited){ + add(query,false,required,prohibited); + } + void add(Query* query, const bool deleteQuery, const bool required, const bool prohibited); + + /** Copies the clauses of this query into the array. + * The array must be at least as long as getClauseCount() + * If you want to use the clauses, make sure you null terminate it. + */ + void getClauses(BooleanClause** clauses) const; + + ///@deprecated + _CL_DEPRECATED( getClauses(clauses) ) BooleanClause** getClauses() const; + + /** + * Give client code access to clauses.size() so we know how + * large the array returned by getClauses is. + */ + size_t getClauseCount() const; + + /** Adds a clause to a boolean query. + * @see #getMaxClauseCount() + */ + void add(BooleanClause* clause); + + Query* rewrite(CL_NS(index)::IndexReader* reader); + Query* clone() const; + bool equals(Query* o) const; + + /** Prints a user-readable version of this query. */ + TCHAR* toString(const TCHAR* field) const; + /** Returns a hash code value for this object.*/ + size_t hashCode() const; + }; + +CL_NS_END +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/search/BooleanScorer.cpp b/src/assistant/3rdparty/clucene/src/CLucene/search/BooleanScorer.cpp new file mode 100644 index 000000000..ae7ee40d6 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/search/BooleanScorer.cpp @@ -0,0 +1,248 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#include "CLucene/StdHeader.h" +#include "BooleanScorer.h" + +#include "Scorer.h" +#include "Similarity.h" + +CL_NS_USE(util) +CL_NS_DEF(search) + + BooleanScorer::BooleanScorer(Similarity* similarity): + Scorer(similarity), + scorers(NULL), + maxCoord (1), + nextMask (1), + end(0), + current(NULL), + requiredMask (0), + prohibitedMask (0), + coordFactors (NULL) + { + bucketTable = _CLNEW BucketTable(this); + } + + BooleanScorer::~BooleanScorer(){ + //Func - Destructor + //Pre - true + //Post - The instance has been destroyed + + _CLDELETE(bucketTable); + _CLDELETE_ARRAY(coordFactors); + _CLDELETE(scorers); + } + + + bool BooleanScorer::next() { + bool more; + do { + while (bucketTable->first != NULL) { // more queued + current = bucketTable->first; + bucketTable->first = current->next; // pop the queue + + // check prohibited & required + if ((current->bits & prohibitedMask) == 0 && + (current->bits & requiredMask) == requiredMask) { + return true; + } + } + + // refill the queue + more = false; + end += BooleanScorer::BucketTable_SIZE; + for (SubScorer* sub = scorers; sub != NULL; sub = sub->next) { + Scorer* scorer = sub->scorer; + int32_t doc; + while (!sub->done && (doc=scorer->doc()) < end) { + sub->collector->collect(doc, scorer->score()); + sub->done = !scorer->next(); + } + if (!sub->done) { + more = true; + } + } + } while (bucketTable->first != NULL || more); + + return false; + } + + qreal BooleanScorer::score(){ + if (coordFactors == NULL) + computeCoordFactors(); + return current->score * coordFactors[current->coord]; + } + + bool BooleanScorer::skipTo(int32_t target) { + _CLTHROWA(CL_ERR_UnsupportedOperation,"UnsupportedOperationException: BooleanScorer::skipTo"); + } + + void BooleanScorer::explain(int32_t doc, Explanation* ret) { + _CLTHROWA(CL_ERR_UnsupportedOperation,"UnsupportedOperationException: BooleanScorer::explain"); + } + + TCHAR* BooleanScorer::toString() { + CL_NS(util)::StringBuffer buffer; + buffer.append(_T("boolean(")); + for (SubScorer* sub = scorers; sub != NULL; sub = sub->next) { + buffer.append(sub->scorer->toString()); + buffer.append(_T(" ")); + } + buffer.appendChar(')'); + return buffer.toString(); + } + + void BooleanScorer::add(Scorer* scorer, const bool required, const bool prohibited) { + int32_t mask = 0; + if (required || prohibited) { + if (nextMask == 0) + _CLTHROWA(CL_ERR_IndexOutOfBounds, "More than 32 required/prohibited clauses in query."); + mask = nextMask; + nextMask = ( nextMask << 1 ); + } else + mask = 0; + + if (!prohibited) + maxCoord++; + + if (prohibited) + prohibitedMask |= mask; // update prohibited mask + else if (required) + requiredMask |= mask; // update required mask + + //scorer, HitCollector, and scorers is delete in the SubScorer + scorers = _CLNEW SubScorer(scorer, required, prohibited, + bucketTable->newCollector(mask), scorers); + } + + void BooleanScorer::computeCoordFactors(){ + coordFactors = _CL_NEWARRAY(qreal,maxCoord); + for (int32_t i = 0; i < maxCoord; i++) + coordFactors[i] = getSimilarity()->coord(i, maxCoord-1); + } + + /*void BooleanScorer::score(HitCollector* results, const int32_t maxDoc) { + if (coordFactors == NULL) + computeCoordFactors(); + + while (currentDoc < maxDoc) { + currentDoc = (currentDoc+BucketTable_SIZE<maxDoc?currentDoc+BucketTable_SIZE:maxDoc); + for (SubScorer* t = scorers; t != NULL; t = t->next) + t->scorer->score((t->collector), currentDoc); + bucketTable->collectHits(results); + } + }*/ + + + + + BooleanScorer::SubScorer::SubScorer(Scorer* scr, const bool r, const bool p, HitCollector* c, SubScorer* nxt): + scorer(scr), + required(r), + prohibited(p), + collector(c), + next(nxt) + { + //Func - Constructor + //Pre - scr != NULL, + // c != NULL + // nxt may or may not be NULL + //Post - The instance has been created + + CND_PRECONDITION(scr != NULL,"scr is NULL"); + CND_PRECONDITION(c != NULL,"c is NULL"); + + done = !scorer->next(); + } + + BooleanScorer::SubScorer::~SubScorer(){ + //Func - Destructor + //Pre - true + //Post - The instance has been destroyed + + for (SubScorer * ptr = next; ptr; ){ + SubScorer* next = ptr->next; + ptr->next = NULL; + _CLDELETE(ptr); + ptr = next; + } + _CLDELETE(scorer); + _CLDELETE(collector); + } + + BooleanScorer::Bucket::Bucket(): + doc(-1), + score(0.0), + bits(0), + coord(0), + next(NULL) + { + } + BooleanScorer::Bucket::~Bucket(){ + } + + + + + BooleanScorer::BucketTable::BucketTable(BooleanScorer* scr): + scorer(scr), + first(NULL) + { + buckets = _CL_NEWARRAY(Bucket,BucketTable_SIZE); + } + BooleanScorer::BucketTable::~BucketTable(){ + clear(); + _CLDELETE_ARRAY(buckets); + } + + void BooleanScorer::BucketTable::clear(){ + //delete first; + first = NULL; + } + int32_t BooleanScorer::BucketTable::size() const { return BooleanScorer::BucketTable_SIZE; } + + HitCollector* BooleanScorer::BucketTable::newCollector(const int32_t mask) { + return _CLNEW Collector(mask, this); + } + + + + + + + + + + BooleanScorer::Collector::Collector(const int32_t msk, BucketTable* bucketTbl): + bucketTable(bucketTbl), + mask(msk) + { + } + + void BooleanScorer::Collector::collect(const int32_t doc, const qreal score){ + BucketTable* table = bucketTable; + int32_t i = doc & (BooleanScorer::BucketTable_SIZE-1); + Bucket* bucket = &table->buckets[i]; + + if (bucket->doc != doc) { // invalid bucket + bucket->doc = doc; // set doc + bucket->score = score; // initialize score + bucket->bits = mask; // initialize mask + bucket->coord = 1; // initialize coord + + bucket->next = table->first; // push onto valid list + table->first = bucket; + } else { // valid bucket + bucket->score += score; // increment score + bucket->bits |= mask; // add bits in mask + bucket->coord++; // increment coord + } + } + + + +CL_NS_END diff --git a/src/assistant/3rdparty/clucene/src/CLucene/search/BooleanScorer.h b/src/assistant/3rdparty/clucene/src/CLucene/search/BooleanScorer.h new file mode 100644 index 000000000..2147bc516 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/search/BooleanScorer.h @@ -0,0 +1,99 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +* +* Changes are Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies). +------------------------------------------------------------------------------*/ +#ifndef _lucene_search_BooleanScorer_ +#define _lucene_search_BooleanScorer_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include "Scorer.h" + +CL_NS_DEF(search) + +class BooleanScorer : public Scorer { +public: + class Bucket : LUCENE_BASE { + public: + int32_t doc; // tells if bucket is valid + qreal score; // incremental score + int32_t bits; // used for bool constraints + int32_t coord; // count of terms in score + Bucket* next; // next valid bucket + + Bucket(); + ~Bucket(); + }; + + class SubScorer: LUCENE_BASE { + public: + bool done; + Scorer* scorer; + bool required; + bool prohibited; + HitCollector* collector; + SubScorer* next; + SubScorer(Scorer* scr, const bool r, const bool p, HitCollector* c, SubScorer* nxt); + ~SubScorer(); + }; + + class BucketTable:LUCENE_BASE { + private: + BooleanScorer* scorer; + public: + Bucket* buckets; + Bucket* first; // head of valid list + + BucketTable(BooleanScorer* scr); + int32_t size() const; + HitCollector* newCollector(const int32_t mask); + void clear(); + ~BucketTable(); + + }; + + class Collector: public HitCollector { + private: + BucketTable* bucketTable; + int32_t mask; + public: + Collector(const int32_t mask, BucketTable* bucketTable); + + void collect(const int32_t doc, const qreal score); + }; + + SubScorer* scorers; + BucketTable* bucketTable; + + int32_t maxCoord; + int32_t nextMask; + + int32_t end; + Bucket* current; + +public: + LUCENE_STATIC_CONSTANT(int32_t,BucketTable_SIZE=1024); + int32_t requiredMask; + int32_t prohibitedMask; + qreal* coordFactors; + + BooleanScorer(Similarity* similarity); + ~BooleanScorer(); + void add(Scorer* scorer, const bool required, const bool prohibited); + int32_t doc() const { return current->doc; } + bool next(); + qreal score(); + bool skipTo(int32_t target); + void explain(int32_t doc, Explanation* ret); + TCHAR* toString(); + void computeCoordFactors(); +}; + +CL_NS_END +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/search/CachingWrapperFilter.cpp b/src/assistant/3rdparty/clucene/src/CLucene/search/CachingWrapperFilter.cpp new file mode 100644 index 000000000..694556ca7 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/search/CachingWrapperFilter.cpp @@ -0,0 +1,86 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#include "CLucene/StdHeader.h" +#include "CachingWrapperFilter.h" + +CL_NS_DEF(search) +CL_NS_USE(index) +CL_NS_USE(util) + +AbstractCachingFilter::AbstractCachingFilter(): + cache(false,true) +{ +} +AbstractCachingFilter::AbstractCachingFilter(const AbstractCachingFilter& copy): + cache(false,true) +{ +} +AbstractCachingFilter::~AbstractCachingFilter(){ +} +AbstractCachingFilter::BitSetHolder::BitSetHolder(CL_NS(util)::BitSet* bits, bool deleteBs){ + this->bits = bits; + this->deleteBs = deleteBs; +} +AbstractCachingFilter::BitSetHolder::~BitSetHolder(){ + if ( deleteBs ) + _CLDELETE(bits); +} + + +BitSet* AbstractCachingFilter::bits(IndexReader* reader){ + SCOPED_LOCK_MUTEX(cache.THIS_LOCK) + BitSetHolder* cached = cache.get(reader); + if ( cached != NULL ) + return cached->bits; + BitSet* bs = doBits(reader); + BitSetHolder* bsh = _CLNEW BitSetHolder(bs, doShouldDeleteBitSet(bs)); + cache.put(reader,bsh); + return bs; +} +void AbstractCachingFilter::closeCallback(CL_NS(index)::IndexReader* reader, void*){ + SCOPED_LOCK_MUTEX(cache.THIS_LOCK) + cache.remove(reader); +} + + + + +CachingWrapperFilter::CachingWrapperFilter(Filter* filter, bool deleteFilter){ + this->filter = filter; + this->deleteFilter = deleteFilter; +} +CachingWrapperFilter::CachingWrapperFilter(const CachingWrapperFilter& copy): + AbstractCachingFilter(copy) +{ + this->filter = copy.filter->clone(); + this->deleteFilter = true; +} +Filter* CachingWrapperFilter::clone() const{ + return _CLNEW CachingWrapperFilter(*this); +} +TCHAR* CachingWrapperFilter::toString(){ + TCHAR* fs = filter->toString(); + int len = _tcslen(fs)+23; + TCHAR* ret = _CL_NEWARRAY(TCHAR,len); + _sntprintf(ret,len,_T("CachingWrapperFilter(%s)"),fs); + _CLDELETE_CARRAY(fs); + return ret; +} +BitSet* CachingWrapperFilter::doBits(IndexReader* reader){ + return filter->bits(reader); +} +bool CachingWrapperFilter::doShouldDeleteBitSet( CL_NS(util)::BitSet* bits ){ + return filter->shouldDeleteBitSet(bits); +} +CachingWrapperFilter::~CachingWrapperFilter(){ + if ( deleteFilter ){ + _CLDELETE(filter); + }else + filter=NULL; +} + +CL_NS_END diff --git a/src/assistant/3rdparty/clucene/src/CLucene/search/CachingWrapperFilter.h b/src/assistant/3rdparty/clucene/src/CLucene/search/CachingWrapperFilter.h new file mode 100644 index 000000000..e48a18292 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/search/CachingWrapperFilter.h @@ -0,0 +1,80 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _lucene_search_CachingWrapperFilter_ +#define _lucene_search_CachingWrapperFilter_ + +#include "CLucene/util/BitSet.h" +#include "CLucene/index/IndexReader.h" +#include "Filter.h" + +CL_NS_DEF(search) +/** + * Wraps another filter's result and caches it. The purpose is to allow + * filters to implement this and allow itself to be cached. Alternatively, + * use the CachingWrapperFilter to cache the filter. + */ +class AbstractCachingFilter: public Filter +{ + class BitSetHolder: LUCENE_BASE{ + bool deleteBs; + public: + BitSetHolder(CL_NS(util)::BitSet* bits, bool deleteBs); + ~BitSetHolder(); + CL_NS(util)::BitSet* bits; + }; + void closeCallback(CL_NS(index)::IndexReader* reader, void* param); + typedef CL_NS(util)::CLHashMap<CL_NS(index)::IndexReader*, + BitSetHolder*, + CL_NS(util)::Compare::Void<CL_NS(index)::IndexReader>, + CL_NS(util)::Equals::Void<CL_NS(index)::IndexReader>, + CL_NS(util)::Deletor::Object<CL_NS(index)::IndexReader>, + CL_NS(util)::Deletor::Object<BitSetHolder> > CacheType; + + CacheType cache; + +protected: + AbstractCachingFilter( const AbstractCachingFilter& copy ); + virtual CL_NS(util)::BitSet* doBits( CL_NS(index)::IndexReader* reader ) = 0; + virtual bool doShouldDeleteBitSet( CL_NS(util)::BitSet* bits ){ return false; } + AbstractCachingFilter(); +public: + virtual ~AbstractCachingFilter(); + + /** Returns a BitSet with true for documents which should be permitted in + search results, and false for those that should not. */ + CL_NS(util)::BitSet* bits( CL_NS(index)::IndexReader* reader ); + + virtual Filter *clone() const = 0; + virtual TCHAR *toString() = 0; + + bool shouldDeleteBitSet( const CL_NS(util)::BitSet* bits ) const{ return false; } +}; + +/** + * Wraps another filter's result and caches it. The purpose is to allow + * filters to simply filter, and then wrap with this class to add + * caching, keeping the two concerns decoupled yet composable. + */ +class CachingWrapperFilter: public AbstractCachingFilter +{ +private: + Filter* filter; + bool deleteFilter; +protected: + CachingWrapperFilter( const CachingWrapperFilter& copy ); + CL_NS(util)::BitSet* doBits( CL_NS(index)::IndexReader* reader ); + bool doShouldDeleteBitSet( CL_NS(util)::BitSet* bits ); +public: + CachingWrapperFilter( Filter* filter, bool deleteFilter=true ); + ~CachingWrapperFilter(); + + Filter *clone() const; + TCHAR *toString(); +}; + +CL_NS_END +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/search/ChainedFilter.cpp b/src/assistant/3rdparty/clucene/src/CLucene/search/ChainedFilter.cpp new file mode 100644 index 000000000..4b6389c0f --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/search/ChainedFilter.cpp @@ -0,0 +1,213 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ + +#include <CLucene/StdHeader.h> +#include <CLucene/util/Misc.h> +#include "ChainedFilter.h" + +CL_NS_DEF(search) +CL_NS_USE(index) +CL_NS_USE(util) +CL_NS_USE(document) + + +ChainedFilter::ChainedFilter( Filter ** _filters, int _op ): + filters(_filters), + logicArray(NULL), + logic(_op) +{ +} +ChainedFilter::ChainedFilter( Filter** _filters, int* _array ): + filters(_filters), + logicArray(_array), + logic(-1) +{ +} +ChainedFilter::ChainedFilter( const ChainedFilter& copy ) : + logicArray( copy.logicArray ), + logic( copy.logic ) +{ + filters = copy.filters; +} +ChainedFilter::~ChainedFilter(void) +{ + +} + +Filter* ChainedFilter::clone() const { + return _CLNEW ChainedFilter(*this ); +} + +const TCHAR* ChainedFilter::getLogicString(int logic){ + if ( logic == ChainedFilter::OR ) + return _T("OR"); + else if ( logic == ChainedFilter::AND ) + return _T("AND"); + else if ( logic == ChainedFilter::ANDNOT ) + return _T("ANDNOT"); + else if ( logic == ChainedFilter::XOR ) + return _T("XOR"); + else if ( logic >= ChainedFilter::USER ){ + return _T("USER"); + } + return _T(""); +} + +TCHAR* ChainedFilter::toString() +{ + + Filter** filter = filters; + + StringBuffer buf(_T("ChainedFilter: [")); + int* la = logicArray; + while(*filter ) + { + if ( filter != filters ) + buf.appendChar(' '); + buf.append(getLogicString(logic==-1?*la:logic)); + buf.appendChar(' '); + + TCHAR* filterstr = (*filter)->toString(); + buf.append(filterstr); + _CLDELETE_ARRAY( filterstr ); + + filter++; + if ( logic == -1 ) + la++; + } + + buf.appendChar(']'); + + return buf.toString(); +} + + +/** Returns a BitSet with true for documents which should be permitted in +search results, and false for those that should not. */ +BitSet* ChainedFilter::bits( IndexReader* reader ) +{ + if( logic != -1 ) + return bits( reader, logic ); + else if( logicArray != NULL ) + return bits( reader, logicArray ); + else + return bits( reader, DEFAULT ); +} + + +BitSet* ChainedFilter::bits( IndexReader* reader, int logic ) +{ + BitSet* bts = NULL; + + Filter** filter = filters; + + // see discussion at top of file + if( *filter ) { + BitSet* tmp = (*filter)->bits( reader ); + if ( (*filter)->shouldDeleteBitSet(tmp) ) //if we are supposed to delete this BitSet, then + bts = tmp; //we can safely call it our own + else if ( tmp == NULL ){ + int32_t len = reader->maxDoc(); + bts = _CLNEW BitSet( len ); //bitset returned null, which means match _all_ + for (int32_t i=0;i<len;i++ ) + bts->set(i); + }else{ + bts = tmp->clone(); //else it is probably cached, so we need to copy it before using it. + } + filter++; + } + else + bts = _CLNEW BitSet( reader->maxDoc() ); + + while( *filter ) { + doChain( bts, reader, logic, *filter ); + filter++; + } + + return bts; +} + + +BitSet* ChainedFilter::bits( IndexReader* reader, int* _logicArray ) +{ + BitSet* bts = NULL; + + Filter** filter = filters; + int* logic = _logicArray; + + // see discussion at top of file + if( *filter ) { + BitSet* tmp = (*filter)->bits( reader ); + if ( (*filter)->shouldDeleteBitSet(tmp) ) //if we are supposed to delete this BitSet, then + bts = tmp; //we can safely call it our own + else if ( tmp == NULL ){ + int32_t len = reader->maxDoc(); + bts = _CLNEW BitSet( len ); //bitset returned null, which means match _all_ + for (int32_t i=0;i<len;i++ ) + bts->set(i); //todo: this could mean that we can skip certain types of filters + } + else + { + bts = tmp->clone(); //else it is probably cached, so we need to copy it before using it. + } + filter++; + logic++; + } + else + bts = _CLNEW BitSet( reader->maxDoc() ); + + while( *filter ) { + doChain( bts, reader, *logic, *filter ); + filter++; + logic++; + } + + return bts; +} + +void ChainedFilter::doUserChain( CL_NS(util)::BitSet* chain, CL_NS(util)::BitSet* filter, int logic ){ + _CLTHROWA(CL_ERR_Runtime,"User chain logic not implemented by superclass"); +} + +BitSet* ChainedFilter::doChain( BitSet* resultset, IndexReader* reader, int logic, Filter* filter ) +{ + BitSet* filterbits = filter->bits( reader ); + int32_t maxDoc = reader->maxDoc(); + int32_t i=0; + if ( logic >= ChainedFilter::USER ){ + doUserChain(resultset,filterbits,logic); + }else{ + switch( logic ) + { + case OR: + for( i=0; i < maxDoc; i++ ) + resultset->set( i, (resultset->get(i) || (filterbits==NULL || filterbits->get(i) ))?1:0 ); + break; + case AND: + for( i=0; i < maxDoc; i++ ) + resultset->set( i, (resultset->get(i) && (filterbits==NULL || filterbits->get(i) ))?1:0 ); + break; + case ANDNOT: + for( i=0; i < maxDoc; i++ ) + resultset->set( i, (resultset->get(i) && (filterbits==NULL || filterbits->get(i)))?0:1 ); + break; + case XOR: + for( i=0; i < maxDoc; i++ ) + resultset->set( i, resultset->get(i) ^ ((filterbits==NULL || filterbits->get(i) )?1:0) ); + break; + default: + doChain( resultset, reader, DEFAULT, filter ); + } + } + + if ( filter->shouldDeleteBitSet(filterbits) ) + _CLDELETE( filterbits ); + + return resultset; +} + +CL_NS_END diff --git a/src/assistant/3rdparty/clucene/src/CLucene/search/ChainedFilter.h b/src/assistant/3rdparty/clucene/src/CLucene/search/ChainedFilter.h new file mode 100644 index 000000000..f4d9d0049 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/search/ChainedFilter.h @@ -0,0 +1,86 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _lucene_search_ChainedFilter_ +#define _lucene_search_ChainedFilter_ + +#include "CLucene/index/IndexReader.h" +#include "CLucene/util/BitSet.h" +#include "CLucene/search/Filter.h" + +CL_NS_DEF(search) + +/* +Discussion - brian@unixpoet.com + +From ChainedFilter.java: + +... + +// First AND operation takes place against a completely false +// bitset and will always return zero results. Thanks to +// Daniel Armbrust for pointing this out and suggesting workaround. + +if (logic[0] == AND) +{ + result = (BitSet) chain[i].bits(reader).clone(); + ++i; +} + +... + +The observation is correct and it was buggy. The problem is that the same +issue remains for the ANDNOT logic op but with the inverse result: all bits +set to 1. The result of the other ops, i.e. OR, AND, XOR for the first filter +ends up just copying the bitset of the first filter (explicitly in the case of the AND). + +Why not do the same for the NAND? This will have the side effect of rendering the first op +in the logic array superflous - not a big problem. + +The only "problem" is that we will return different results then the Java +Lucene code - though I prefer CLucene to be a correct implementation and only maintain +API compat rather than full 100% compat with Lucene. +*/ +class ChainedFilter: public Filter +{ +public: + LUCENE_STATIC_CONSTANT(int, OR = 0); //set current bit if the chain is set OR if the filter bit is set + LUCENE_STATIC_CONSTANT(int, AND = 1); //set current bit if the chain is set AND the filter bit is set + LUCENE_STATIC_CONSTANT(int, ANDNOT = 2); //set current bit if the chain is not set AND the filter bit is not set + LUCENE_STATIC_CONSTANT(int, XOR = 3); //set current bit if the chain is set OR the filter bit is set BUT not both is set + + LUCENE_STATIC_CONSTANT(int, USER = 5); //add this value to user defined value, then override doUserChain + + LUCENE_STATIC_CONSTANT(int, DEFAULT = OR); + +protected: + Filter **filters; + int *logicArray; + int logic; + + ChainedFilter( const ChainedFilter& copy ); + CL_NS(util)::BitSet* bits( CL_NS(index)::IndexReader* reader, int logic ); + CL_NS(util)::BitSet* bits( CL_NS(index)::IndexReader* reader, int* logicArray ); + CL_NS(util)::BitSet* doChain( CL_NS(util)::BitSet* result, CL_NS(index)::IndexReader* reader, int logic, Filter* filter ); + + virtual void doUserChain( CL_NS(util)::BitSet* chain, CL_NS(util)::BitSet* filter, int logic ); + virtual const TCHAR* getLogicString(int logic); +public: + ChainedFilter( Filter** filters, int op = DEFAULT ); + ChainedFilter( Filter** filters, int* _array ); + virtual ~ChainedFilter(); + + /** Returns a BitSet with true for documents which should be permitted in + search results, and false for those that should not. */ + CL_NS(util)::BitSet* bits( CL_NS(index)::IndexReader* reader ); + + virtual Filter* clone() const; + + TCHAR* toString(); +}; + +CL_NS_END +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/search/Compare.h b/src/assistant/3rdparty/clucene/src/CLucene/search/Compare.h new file mode 100644 index 000000000..ab38b17f1 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/search/Compare.h @@ -0,0 +1,161 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _lucene_search_Compare_ +#define _lucene_search_Compare_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include "FieldSortedHitQueue.h" + +CL_NS_DEF(search) + + +class ScoreDocComparators:LUCENE_BASE { +protected: + ScoreDocComparators(){} +public: + ~ScoreDocComparators(){ + } + + class Relevance:public ScoreDocComparator { + public: + int32_t compare (struct ScoreDoc* i, struct ScoreDoc* j) { + if (i->score > j->score) return -1; + if (i->score < j->score) return 1; + return 0; + } + CL_NS(util)::Comparable* sortValue (struct ScoreDoc* i) { + return _CLNEW CL_NS(util)::Compare::Float (i->score); + } + int32_t sortType() { + return SortField::DOCSCORE; + } + }; + + class IndexOrder:public ScoreDocComparator{ + public: + IndexOrder(): + ScoreDocComparator() + { + + } + int32_t compare (struct ScoreDoc* i, struct ScoreDoc* j) { + if (i->doc < j->doc) return -1; + if (i->doc > j->doc) return 1; + return 0; + } + CL_NS(util)::Comparable* sortValue (struct ScoreDoc* i) { + return _CLNEW CL_NS(util)::Compare::Int32(i->doc); + } + int32_t sortType() { + return SortField::DOC; + } + }; + + + class String: public ScoreDocComparator { + FieldCache::StringIndex* index; +#ifdef _CL__CND_DEBUG + int32_t length; +#endif + public: + String(FieldCache::StringIndex* index, int32_t len) + { +#ifdef _CL__CND_DEBUG + this->length = len; +#endif + this->index = index; + } + + int32_t compare (struct ScoreDoc* i, struct ScoreDoc* j) { + CND_PRECONDITION(i->doc<length, "i->doc>=length") + CND_PRECONDITION(j->doc<length, "j->doc>=length") + if (index->order[i->doc] < index->order[j->doc]) return -1; + if (index->order[i->doc] > index->order[j->doc]) return 1; + return 0; + } + + CL_NS(util)::Comparable* sortValue (struct ScoreDoc* i) { + return _CLNEW CL_NS(util)::Compare::TChar(index->lookup[index->order[i->doc]]); + } + + int32_t sortType() { + return SortField::STRING; + } + }; + + class Int32:public ScoreDocComparator{ + int32_t* fieldOrder; +#ifdef _CL__CND_DEBUG + int32_t length; +#endif + public: + Int32(int32_t* fieldOrder, int32_t len) + { + this->fieldOrder = fieldOrder; +#ifdef _CL__CND_DEBUG + this->length = len; +#endif + } + + + int32_t compare (struct ScoreDoc* i, struct ScoreDoc* j) { + CND_PRECONDITION(i->doc<length, "i->doc>=length") + CND_PRECONDITION(j->doc<length, "j->doc>=length") + if (fieldOrder[i->doc] < fieldOrder[j->doc]) return -1; + if (fieldOrder[i->doc] > fieldOrder[j->doc]) return 1; + return 0; + } + + CL_NS(util)::Comparable* sortValue (struct ScoreDoc* i) { + CND_PRECONDITION(i->doc<length, "i->doc>=length") + return _CLNEW CL_NS(util)::Compare::Int32(fieldOrder[i->doc]); + } + + int32_t sortType() { + return SortField::INT; + } + }; + + class Float:public ScoreDocComparator { + qreal* fieldOrder; +#ifdef _CL__CND_DEBUG + int32_t length; +#endif + public: + Float(qreal* fieldOrder, int32_t len) + { + this->fieldOrder = fieldOrder; +#ifdef _CL__CND_DEBUG + this->length = len; +#endif + } + + int32_t compare (struct ScoreDoc* i, struct ScoreDoc* j) { + CND_PRECONDITION(i->doc<length, "i->doc>=length") + CND_PRECONDITION(j->doc<length, "j->doc>=length") + if (fieldOrder[i->doc] < fieldOrder[j->doc]) return -1; + if (fieldOrder[i->doc] > fieldOrder[j->doc]) return 1; + return 0; + } + + CL_NS(util)::Comparable* sortValue (struct ScoreDoc* i) { + CND_PRECONDITION(i->doc<length, "i->doc>=length") + return _CLNEW CL_NS(util)::Compare::Float(fieldOrder[i->doc]); + } + + int32_t sortType() { + return SortField::FLOAT; + } + }; +}; + + +CL_NS_END +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/search/ConjunctionScorer.cpp b/src/assistant/3rdparty/clucene/src/CLucene/search/ConjunctionScorer.cpp new file mode 100644 index 000000000..9b7846f8e --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/search/ConjunctionScorer.cpp @@ -0,0 +1,144 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#include "CLucene/StdHeader.h" +#include "ConjunctionScorer.h" +#include "CLucene/util/Arrays.h" + +CL_NS_USE(index) +CL_NS_USE(util) +CL_NS_DEF(search) + + Scorer* ConjunctionScorer::first() const{ + if ( scorers.end() == scorers.begin() ) + return NULL; + + return *scorers.begin(); + } //get First + Scorer* ConjunctionScorer::last() { + if ( scorers.end() == scorers.begin() ) + return NULL; + + CL_NS_STD(list)<Scorer*>::iterator i = scorers.end(); + --i; + return *i; + } //get Last + + class _ScorerSorter:public CL_NS(util)::Arrays::_Arrays<Scorer*>{ + public: + bool equals(Scorer* o1,Scorer* o2) const{ + return o1->doc() == o2->doc(); + } + int32_t compare(Scorer* o1,Scorer* o2) const{ + return o1->doc() - o2->doc(); + } + }; + _ScorerSorter __ScorerSorter; + + void ConjunctionScorer::sortScorers() { + // move scorers to an array + int32_t size = scorers.size(); + Scorer** array = _CL_NEWARRAY(Scorer*,size+1); + scorers.toArray(array); + scorers.clear(); // empty the list + + // note that this comparator is not consistent with equals! + __ScorerSorter.sort(array,size,0,size); + + for (int32_t i = 0; i<size; i++) { + scorers.push_back(array[i]); // re-build list, now sorted + } + _CLDELETE_ARRAY(array); + } + + bool ConjunctionScorer::doNext() { + while (more && first()->doc() < last()->doc()) { // find doc w/ all clauses + more = first()->skipTo(last()->doc()); // skip first upto last + Scorer* scorer = *scorers.begin(); + scorers.delete_front(); + scorers.push_back(scorer); // move first to last + } + return more; // found a doc with all clauses + } + + + void ConjunctionScorer::init() { + more = scorers.size() > 0; + + // compute coord factor + coord = getSimilarity()->coord(scorers.size(), scorers.size()); + + // move each scorer to its first entry + CL_NS_STD(list)<Scorer*>::iterator i = scorers.begin(); + while (more && i!=scorers.end()) { + more = ((Scorer*)*i)->next(); + ++i; + } + + if (more) + sortScorers(); // initial sort of list + + firstTime = false; + } + + ConjunctionScorer::ConjunctionScorer(Similarity* similarity): + Scorer(similarity), + scorers(false), + firstTime(true), + more(true), + coord(0.0) + { + } + ConjunctionScorer::~ConjunctionScorer(){ + scorers.setDoDelete(true); + } + + TCHAR *CL_NS(search)::Scorer::toString(void){ + return STRDUP_TtoT(_T("ConjunctionScorer")); + } + + + void ConjunctionScorer::add(Scorer* scorer){ + scorers.push_back(scorer); + } + + + int32_t ConjunctionScorer::doc() const{ return first()->doc(); } + + bool ConjunctionScorer::next() { + if (firstTime) { + init(); + } else if (more) { + more = last()->next(); // trigger further scanning + } + return doNext(); + } + + bool ConjunctionScorer::skipTo(int32_t target) { + CL_NS_STD(list)<Scorer*>::iterator i = scorers.begin(); + while (more && i!=scorers.end()) { + more = ((Scorer*)*i)->skipTo(target); + ++i; + } + if (more) + sortScorers(); // re-sort scorers + return doNext(); + } + + qreal ConjunctionScorer::score(){ + qreal score = 0.0f; // sum scores + CL_NS_STD(list)<Scorer*>::const_iterator i = scorers.begin(); + while (i!=scorers.end()){ + score += (*i)->score(); + ++i; + } + score *= coord; + return score; + } + + + +CL_NS_END diff --git a/src/assistant/3rdparty/clucene/src/CLucene/search/ConjunctionScorer.h b/src/assistant/3rdparty/clucene/src/CLucene/search/ConjunctionScorer.h new file mode 100644 index 000000000..4b6807209 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/search/ConjunctionScorer.h @@ -0,0 +1,50 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _lucene_search_ConjunctionScorer_ +#define _lucene_search_ConjunctionScorer_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif +#include "Scorer.h" +#include "Similarity.h" + +CL_NS_DEF(search) + +/** Scorer for conjunctions, sets of queries, all of which are required. */ +class ConjunctionScorer: public Scorer { +private: + CL_NS(util)::CLLinkedList<Scorer*,CL_NS(util)::Deletor::Object<Scorer> > scorers; + bool firstTime; + bool more; + qreal coord; + + Scorer* first() const; + Scorer* last(); + void sortScorers(); + bool doNext(); + void init(); +public: + ConjunctionScorer(Similarity* similarity); + virtual ~ConjunctionScorer(); + TCHAR* toString(void){ + return STRDUP_TtoT(_T("ConjunctionScorer")); + } + void add(Scorer* scorer); + int32_t doc() const; + bool next(); + bool skipTo(int32_t target); + qreal score(); + virtual void explain(int32_t doc, Explanation* ret) { + _CLTHROWA(CL_ERR_UnsupportedOperation,"UnsupportedOperationException: ConjunctionScorer::explain"); + } + + +}; + +CL_NS_END +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/search/DateFilter.cpp b/src/assistant/3rdparty/clucene/src/CLucene/search/DateFilter.cpp new file mode 100644 index 000000000..925858204 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/search/DateFilter.cpp @@ -0,0 +1,93 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#include "CLucene/StdHeader.h" +#include "DateFilter.h" + +CL_NS_USE(index) +CL_NS_USE(util) +CL_NS_USE(document) +CL_NS_DEF(search) + + DateFilter::~DateFilter(){ + _CLDECDELETE( start ); + _CLDECDELETE( end ); + } + + DateFilter::DateFilter(const DateFilter& copy): + start( _CL_POINTER(copy.start) ), + end ( _CL_POINTER(copy.end) ) + { + } + + /** Constructs a filter for field <code>f</code> matching times between + <code>from</code> and <code>to</code>. */ + DateFilter::DateFilter(const TCHAR* f, int64_t from, int64_t to) + { + TCHAR* tmp = DateField::timeToString(from); + start = _CLNEW Term(f, tmp); + _CLDELETE_CARRAY(tmp); + + tmp = DateField::timeToString(to); + end = _CLNEW Term(start, tmp); + _CLDELETE_CARRAY(tmp); + } + + /** Constructs a filter for field <code>f</code> matching times before + <code>time</code>. */ + DateFilter* DateFilter::Before(const TCHAR* field, int64_t time) { + return _CLNEW DateFilter(field, 0,time); + } + + /** Constructs a filter for field <code>f</code> matching times after + <code>time</code>. */ + DateFilter* DateFilter::After(const TCHAR* field, int64_t time) { + return _CLNEW DateFilter(field,time, DATEFIELD_DATE_MAX ); + } + + /** Returns a BitSet with true for documents which should be permitted in + search results, and false for those that should not. */ + BitSet* DateFilter::bits(IndexReader* reader) { + BitSet* bts = _CLNEW BitSet(reader->maxDoc()); + + TermEnum* enumerator = reader->terms(start); + if (enumerator->term(false) == NULL){ + _CLDELETE(enumerator); + return bts; + } + TermDocs* termDocs = reader->termDocs(); + + try { + while (enumerator->term(false)->compareTo(end) <= 0) { + termDocs->seek(enumerator->term(false)); + while (termDocs->next()) { + bts->set(termDocs->doc()); + } + if (!enumerator->next()) { + break; + } + } + } _CLFINALLY ( + termDocs->close(); + _CLDELETE(termDocs); + enumerator->close(); + _CLDELETE(enumerator); + ); + return bts; + } + + Filter* DateFilter::clone() const{ + return _CLNEW DateFilter(*this); + } + + TCHAR* DateFilter::toString(){ + size_t len = _tcslen(start->field()) + start->textLength() + end->textLength() + 8; + TCHAR* ret = _CL_NEWARRAY(TCHAR,len); + ret[0]=0; + _sntprintf(ret,len,_T("%s: [%s-%s]"), start->field(),start->text(),end->text()); + return ret; + } +CL_NS_END diff --git a/src/assistant/3rdparty/clucene/src/CLucene/search/DateFilter.h b/src/assistant/3rdparty/clucene/src/CLucene/search/DateFilter.h new file mode 100644 index 000000000..b37272b84 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/search/DateFilter.h @@ -0,0 +1,59 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _lucene_search_DateFilter_ +#define _lucene_search_DateFilter_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include "CLucene/document/DateField.h" +#include "CLucene/index/Term.h" +#include "CLucene/index/Terms.h" +#include "CLucene/index/IndexReader.h" +#include "CLucene/util/BitSet.h" +#include "Filter.h" + +CL_NS_DEF(search) + /** + * A Filter that restricts search results to a range of time. + * + * <p>For this to work, documents must have been indexed with a + * {@link DateField}. + */ + class DateFilter: public Filter { + private: + CL_NS(index)::Term* start; + CL_NS(index)::Term* end; + + protected: + DateFilter(const DateFilter& copy); + public: + ~DateFilter(); + + /** Constructs a filter for field <code>f</code> matching times between + <code>from</code> and <code>to</code>. */ + DateFilter(const TCHAR* f, int64_t from, int64_t to); + + /** Constructs a filter for field <code>f</code> matching times before + <code>time</code>. */ + static DateFilter* Before(const TCHAR* field, int64_t time) ; + + /** Constructs a filter for field <code>f</code> matching times after + <code>time</code>. */ + static DateFilter* After(const TCHAR* field, int64_t time) ; + + /** Returns a BitSet with true for documents which should be permitted in + search results, and false for those that should not. */ + CL_NS(util)::BitSet* bits(CL_NS(index)::IndexReader* reader) ; + + Filter* clone() const; + + TCHAR* toString(); + }; +CL_NS_END +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/search/ExactPhraseScorer.cpp b/src/assistant/3rdparty/clucene/src/CLucene/search/ExactPhraseScorer.cpp new file mode 100644 index 000000000..1fbf2e99d --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/search/ExactPhraseScorer.cpp @@ -0,0 +1,85 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#include "CLucene/StdHeader.h" +#include "ExactPhraseScorer.h" + +#include "PhraseScorer.h" +#include "CLucene/index/Terms.h" + +CL_NS_USE(index) +CL_NS_DEF(search) + + ExactPhraseScorer::ExactPhraseScorer(Weight* weight, TermPositions** tps, + int32_t* positions, Similarity* similarity, uint8_t* norms): + PhraseScorer(weight, tps, positions, similarity, norms){ + //Func - Constructor + //Pre - tps != NULL + // tpsLength >= 0 + // n != NULL + //Post - Instance has been created + + CND_PRECONDITION(tps != NULL,"tps is NULL"); + CND_PRECONDITION(tps[0] != NULL,"tps is NULL"); + //CND_PRECONDITION(n != NULL,"n is NULL") =this is checked already in PhraseScorer + + } + + qreal ExactPhraseScorer::phraseFreq(){ + //Func - Returns the freqency of the phrase + //Pre - first != NULL + // last != NULL + // pq != NULL + // size of the PhraseQueue pq is 0 + //Post - The frequency of the phrase has been returned + + CND_PRECONDITION(first != NULL,"first is NULL"); + CND_PRECONDITION(last != NULL,"last is NULL"); + CND_PRECONDITION(pq != NULL,"pq is NULL"); + CND_PRECONDITION(pq->size()==0,"pq is not empty"); + + //build pq from list + + //Add the nodes of the list of PhrasePositions and store them + //into the PhraseQueue pq so it can used to build + //a list of sorted nodes + for (PhrasePositions* pp = first; pp != NULL; pp = pp->_next) { + //Read the first TermPosition of the current PhrasePositions pp + pp->firstPosition(); + //Store the current PhrasePositions pp into the PhraseQueue pq + pq->put(pp); + } + //pqToList requires that first and last be NULL when it's called. + //This is done at the beginning of pqToList() + //In this case, the nodes of the linked list are referenced by pq (see + //above loop), so we can clear our pointers to the head and tail of the + //linked list without fear of leaking the nodes. + + //rebuild list from pq + pqToList(); + + //Initialize freq at 0 + int32_t freq = 0; + + //find position with all terms + do { + //scan forward in first + while (first->position < last->position){ + do{ + if (!first->nextPosition()){ + return (qreal)freq; + } + } while (first->position < last->position); + //Make the current first node the last node in the list + firstToLast(); + } + //all equal: a match has been found + freq++; + } while (last->nextPosition()); + + return (qreal)freq; + } +CL_NS_END diff --git a/src/assistant/3rdparty/clucene/src/CLucene/search/ExactPhraseScorer.h b/src/assistant/3rdparty/clucene/src/CLucene/search/ExactPhraseScorer.h new file mode 100644 index 000000000..d82aa9e9a --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/search/ExactPhraseScorer.h @@ -0,0 +1,31 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _lucene_search_ExactPhraseScorer_ +#define _lucene_search_ExactPhraseScorer_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include "PhraseScorer.h" +#include "CLucene/index/Terms.h" + +CL_NS_DEF(search) + + class ExactPhraseScorer: public PhraseScorer { + public: + ExactPhraseScorer(Weight* weight, CL_NS(index)::TermPositions** tps, int32_t* positions, + Similarity* similarity, uint8_t* norms ); + + ~ExactPhraseScorer(){ + } + protected: + //Returns the exact freqency of the phrase + qreal phraseFreq(); + }; +CL_NS_END +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/search/Explanation.cpp b/src/assistant/3rdparty/clucene/src/CLucene/search/Explanation.cpp new file mode 100644 index 000000000..87189b71b --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/search/Explanation.cpp @@ -0,0 +1,133 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#include "CLucene/StdHeader.h" +#include "Explanation.h" +#include "CLucene/util/StringBuffer.h" + +CL_NS_USE(util) +CL_NS_DEF(search) + + +Explanation::Explanation(qreal value, const TCHAR* description) { + this->value = value; + _tcsncpy(this->description,description,LUCENE_SEARCH_EXPLANATION_DESC_LEN); +} + +Explanation::Explanation() { + this->value = 0; + this->description[0]=0; +} + +Explanation::Explanation(const Explanation& copy){ + set(copy); +} +void Explanation::set(const Explanation& copy){ + this->value = copy.value; + STRCPY_TtoT(description,copy.description,LUCENE_SEARCH_EXPLANATION_DESC_LEN); + + details.clear(); + typedef CL_NS(util)::Deletor::Object<Explanation> Deletor; + CL_NS(util)::CLArrayList<Explanation*, Deletor>::const_iterator itr; + itr = copy.details.begin(); + while ( itr != copy.details.end() ){ + details.push_back( (*itr)->clone() ); + ++itr; + } +} + +Explanation::~Explanation(){ +} + +void Explanation::setDescription(const TCHAR* description) { + _tcsncpy(this->description,description,LUCENE_SEARCH_EXPLANATION_DESC_LEN); +} + + +Explanation* Explanation::clone() const{ + return _CLNEW Explanation(*this); +} + +qreal Explanation::getValue() const{ + return value; +} + +void Explanation::setValue(qreal value) { + this->value = value; +} + +const TCHAR* Explanation::getDescription() const { + return description; +} + +///todo: mem leaks +TCHAR* Explanation::toString(int32_t depth) { + StringBuffer buffer; + for (int32_t i = 0; i < depth; i++) { + buffer.append(_T(" ")); + } + buffer.appendFloat(getValue(),2); + buffer.append(_T(" = ")); + buffer.append(getDescription()); + buffer.append(_T("\n")); + + for ( uint32_t j=0;j<details.size();j++ ){ + TCHAR* tmp = details[j]->toString(depth+1); + buffer.append(tmp); + _CLDELETE_CARRAY(tmp); + } + return buffer.toString(); +} + +int Explanation::getDetailsLength(){ + return details.size(); +} +Explanation* Explanation::getDetail(int i){ + return details[i]; +} +/** The sub-nodes of this explanation node. */ +void Explanation::getDetails(Explanation** ret) { + uint32_t size = details.size(); + for ( uint32_t i=0;i<size;i++ ){ + ret[i] = details[i]->clone(); + } + ret[size] = NULL; +} + +/** Adds a sub-node to this explanation node. */ +void Explanation::addDetail(Explanation* detail) { + details.push_back(detail); +} + +/** Render an explanation as text. */ +TCHAR* Explanation::toString() { + return toString(0); +} + +/** Render an explanation as HTML. */ +///todo: mem leaks +TCHAR* Explanation::toHtml() { + StringBuffer buffer; + TCHAR* tmp; + buffer.append(_T("<ul>\n")); + + buffer.append(_T("<li>")); + buffer.appendFloat(getValue(),2); + buffer.append(_T(" = ")); + + buffer.append(getDescription()); + buffer.append(_T("</li>\n")); + + for ( uint32_t i=0;i<details.size();i++ ){ + tmp = details[i]->toHtml(); + buffer.append(tmp); + _CLDELETE_CARRAY(tmp); + } + buffer.append(_T("</ul>\n")); + + return buffer.toString(); +} +CL_NS_END diff --git a/src/assistant/3rdparty/clucene/src/CLucene/search/Explanation.h b/src/assistant/3rdparty/clucene/src/CLucene/search/Explanation.h new file mode 100644 index 000000000..7c95822b6 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/search/Explanation.h @@ -0,0 +1,66 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _lucene_search_Explanation +#define _lucene_search_Explanation + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +CL_NS_DEF(search) + + #define LUCENE_SEARCH_EXPLANATION_DESC_LEN 200 + class Explanation :LUCENE_BASE { + private: + qreal value; // the value of this node + TCHAR description[LUCENE_SEARCH_EXPLANATION_DESC_LEN]; // what it represents + CL_NS(util)::CLArrayList<Explanation*,CL_NS(util)::Deletor::Object<Explanation> > details; // sub-explanations + + TCHAR* toString(int32_t depth); + protected: + Explanation(const Explanation& copy); + public: + Explanation(); + ~Explanation(); + + Explanation(qreal value, const TCHAR* description); + void set(const Explanation& other); + + Explanation* clone() const; + + /** The value assigned to this explanation node. */ + qreal getValue() const; + + /** Sets the value assigned to this explanation node. */ + void setValue(qreal value); + + /** A description of this explanation node. */ + const TCHAR* getDescription() const; ///<returns reference + + /** Sets the description of this explanation node. */ + void setDescription(const TCHAR* description); + + /** The sub-nodes of this explanation node. + * @param ret this array of Explanations should be getDetailsLength()+1 in size. + The array will be null terminated. + */ + void getDetails(Explanation** ret); + int getDetailsLength(); + Explanation* getDetail(int i); + + /** Adds a sub-node to this explanation node. */ + void addDetail(Explanation* detail); + + /** Render an explanation as text. */ + TCHAR* toString(); + + /** Render an explanation as HTML. */ + TCHAR* toHtml(); + }; + +CL_NS_END +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/search/FieldCache.cpp b/src/assistant/3rdparty/clucene/src/CLucene/search/FieldCache.cpp new file mode 100644 index 000000000..fae672019 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/search/FieldCache.cpp @@ -0,0 +1,55 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#include "CLucene/StdHeader.h" +#include "FieldCache.h" +#include "FieldCacheImpl.h" + +CL_NS_DEF(search) + +FieldCache* FieldCache::DEFAULT = _CLNEW FieldCacheImpl(); +int32_t FieldCache::STRING_INDEX = -1; + +FieldCacheAuto::FieldCacheAuto(int32_t len, int32_t type){ + contentType = type; + contentLen = len; + ownContents = false; + + intArray=NULL; + floatArray=NULL; + stringIndex=NULL; + stringArray=NULL; + comparableArray=NULL; + sortComparator=NULL; + scoreDocComparator=NULL; +} +FieldCacheAuto::~FieldCacheAuto(){ + if ( contentType == FieldCacheAuto::INT_ARRAY ){ + _CLDELETE_ARRAY(intArray); + }else if ( contentType == FieldCacheAuto::FLOAT_ARRAY ){ + _CLDELETE_ARRAY(floatArray); + }else if ( contentType == FieldCacheAuto::STRING_INDEX ){ + _CLDELETE(stringIndex); + }else if ( contentType == FieldCacheAuto::STRING_ARRAY ){ + if ( ownContents ){ + for ( int32_t i=0;i<contentLen;i++ ) + _CLDELETE_CARRAY(stringArray[i]); + } + _CLDELETE_ARRAY(stringArray); + }else if ( contentType == FieldCacheAuto::COMPARABLE_ARRAY ){ + if ( ownContents ){ + for ( int32_t i=0;i<contentLen;i++ ) + _CLDELETE(comparableArray[i]); + } + _CLDELETE_ARRAY(comparableArray); + }else if ( contentType == FieldCacheAuto::SORT_COMPARATOR ){ + _CLDELETE(sortComparator); + }else if ( contentType == FieldCacheAuto::SCOREDOC_COMPARATOR ){ + _CLDELETE(scoreDocComparator); + } +} + +CL_NS_END diff --git a/src/assistant/3rdparty/clucene/src/CLucene/search/FieldCache.h b/src/assistant/3rdparty/clucene/src/CLucene/search/FieldCache.h new file mode 100644 index 000000000..eeec26f33 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/search/FieldCache.h @@ -0,0 +1,182 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _lucene_search_FieldCache_ +#define _lucene_search_FieldCache_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include "CLucene/index/IndexReader.h" +#include "Sort.h" + + +CL_NS_DEF(search) + +class FieldCacheAuto; //predefine + +/** + * Expert: Maintains caches of term values. + * + */ +class FieldCache :LUCENE_BASE { +public: + virtual ~FieldCache(){ + } + + /** Expert: Stores term text values and document ordering data. */ + class StringIndex:LUCENE_BASE { + public: + /** All the term values, in natural order. */ + TCHAR** lookup; + + /** For each document, an index into the lookup array. */ + int32_t* order; + + int count; + + /** Creates one of these objects + Consumes all memory given. + */ + StringIndex (int32_t* values, TCHAR** lookup, int count) { + this->count = count; + this->order = values; + this->lookup = lookup; + } + + ~StringIndex(){ + _CLDELETE_ARRAY(order); + + for ( int i=0;i<count;i++ ) + _CLDELETE_CARRAY(lookup[i]); + _CLDELETE_ARRAY(lookup); + } + }; + + + /** Indicator for FieldCache::StringIndex values in the cache. + NOTE: the value assigned to this constant must not be + the same as any of those in SortField!! + */ + static int32_t STRING_INDEX; + + /** Expert: The cache used internally by sorting and range query classes. */ + static FieldCache* DEFAULT; + + /** Checks the internal cache for an appropriate entry, and if none is + * found, reads the terms in <code>field</code> as integers and returns an array + * of size <code>reader.maxDoc()</code> of the value each document + * has in the given field. + * @param reader Used to get field values. + * @param field Which field contains the integers. + * @return The values in the given field for each document. + * @throws IOException If any error occurs. + */ + virtual FieldCacheAuto* getInts (CL_NS(index)::IndexReader* reader, const TCHAR* field) = 0; + + /** Checks the internal cache for an appropriate entry, and if + * none is found, reads the terms in <code>field</code> as floats and returns an array + * of size <code>reader.maxDoc()</code> of the value each document + * has in the given field. + * @param reader Used to get field values. + * @param field Which field contains the floats. + * @return The values in the given field for each document. + * @throws IOException If any error occurs. + */ + virtual FieldCacheAuto* getFloats (CL_NS(index)::IndexReader* reader, const TCHAR* field) = 0; + + /** Checks the internal cache for an appropriate entry, and if none + * is found, reads the term values in <code>field</code> and returns an array + * of size <code>reader.maxDoc()</code> containing the value each document + * has in the given field. + * @param reader Used to get field values. + * @param field Which field contains the strings. + * @return The values in the given field for each document. + * @throws IOException If any error occurs. + */ + virtual FieldCacheAuto* getStrings (CL_NS(index)::IndexReader* reader, const TCHAR* field) = 0; + + /** Checks the internal cache for an appropriate entry, and if none + * is found reads the term values in <code>field</code> and returns + * an array of them in natural order, along with an array telling + * which element in the term array each document uses. + * @param reader Used to get field values. + * @param field Which field contains the strings. + * @return Array of terms and index into the array for each document. + * @throws IOException If any error occurs. + */ + virtual FieldCacheAuto* getStringIndex (CL_NS(index)::IndexReader* reader, const TCHAR* field) = 0; + + /** Checks the internal cache for an appropriate entry, and if + * none is found reads <code>field</code> to see if it contains integers, floats + * or strings, and then calls one of the other methods in this class to get the + * values. For string values, a FieldCache::StringIndex is returned. After + * calling this method, there is an entry in the cache for both + * type <code>AUTO</code> and the actual found type. + * @param reader Used to get field values. + * @param field Which field contains the values. + * @return int32_t[], qreal[] or FieldCache::StringIndex. + * @throws IOException If any error occurs. + */ + virtual FieldCacheAuto* getAuto (CL_NS(index)::IndexReader* reader, const TCHAR* field) = 0; + + /** Checks the internal cache for an appropriate entry, and if none + * is found reads the terms out of <code>field</code> and calls the given SortComparator + * to get the sort values. A hit in the cache will happen if <code>reader</code>, + * <code>field</code>, and <code>comparator</code> are the same (using <code>equals()</code>) + * as a previous call to this method. + * @param reader Used to get field values. + * @param field Which field contains the values. + * @param comparator Used to convert terms into something to sort by. + * @return Array of sort objects, one for each document. + * @throws IOException If any error occurs. + */ + virtual FieldCacheAuto* getCustom (CL_NS(index)::IndexReader* reader, const TCHAR* field, SortComparator* comparator) = 0; +}; + +/** A class holding an AUTO field. In java lucene an Object + is used, but we use this. + contentType: + 1 - integer array + 2 - float array + 3 - FieldCache::StringIndex object + This class is also used when returning getInt, getFloat, etc + because we have no way of returning the size of the array and this + class can be used to determine the array size +*/ +class FieldCacheAuto:LUCENE_BASE{ +public: + enum{ + INT_ARRAY=1, + FLOAT_ARRAY=2, + STRING_INDEX=3, + STRING_ARRAY=4, + COMPARABLE_ARRAY=5, + SORT_COMPARATOR=6, + SCOREDOC_COMPARATOR=7 + }; + + FieldCacheAuto(int32_t len, int32_t type); + ~FieldCacheAuto(); + ///if contents should be deleted too, depending on type + bool ownContents; + int32_t contentLen; //number of items in the list + uint8_t contentType; + int32_t* intArray; //item 1 + qreal* floatArray; //item 2 + FieldCache::StringIndex* stringIndex; //item 3 + TCHAR** stringArray; //item 4 + CL_NS(util)::Comparable** comparableArray; //item 5 + SortComparator* sortComparator; //item 6 + ScoreDocComparator* scoreDocComparator; //item 7 + +}; + + +CL_NS_END + +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/search/FieldCacheImpl.cpp b/src/assistant/3rdparty/clucene/src/CLucene/search/FieldCacheImpl.cpp new file mode 100644 index 000000000..62052097e --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/search/FieldCacheImpl.cpp @@ -0,0 +1,529 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#include "CLucene/StdHeader.h" +#include "FieldCacheImpl.h" + +CL_NS_USE(util) +CL_NS_USE(index) +CL_NS_DEF(search) + +FieldCacheImpl::FieldCacheImpl(): + cache(false,true){ +} +FieldCacheImpl::~FieldCacheImpl(){ + cache.clear(); +} + +FieldCacheImpl::FileEntry::FileEntry (const TCHAR* field, int32_t type) { + this->field = CLStringIntern::intern(field CL_FILELINE); + this->type = type; + this->custom = NULL; + this->_hashCode = 0; + } + + /** Creates one of these objects for a custom comparator. */ + FieldCacheImpl::FileEntry::FileEntry (const TCHAR* field, SortComparatorSource* custom) { + this->field = CLStringIntern::intern(field CL_FILELINE); + this->type = SortField::CUSTOM; + this->custom = custom; + this->_hashCode = 0; + } + FieldCacheImpl::FileEntry::~FileEntry(){ + CLStringIntern::unintern(field); + } + + size_t FieldCacheImpl::FileEntry::hashCode(){ + if ( _hashCode == 0 ){ + //todo: cache hashcode? + size_t ret = Misc::thashCode(field); + if ( custom != NULL ) + ret = ret ^ custom->hashCode(); + ret = ret ^ (type*7); //type with a seed + _hashCode = ret; + } + return _hashCode; + } + int32_t FieldCacheImpl::FileEntry::compareTo(const FieldCacheImpl::FileEntry* other) const{ + if ( other->field == this->field ){ + if ( other->type == this->type ){ + if ( other->custom == NULL ){ + if ( this->custom == NULL ) + return 0; //both null + else + return 1; + }else if ( this->custom == NULL ) + return -1; + else if ( other->custom < this->custom ) + return -1; + else if ( other->custom > this->custom ) + return 1; + else + return 0; + }else if ( other->type > this->type ) + return 1; + else + return -1; + + }else + return _tcscmp(other->field,this->field); + } + + /** Two of these are equal iff they reference the same field and type. */ + /*bool FieldCacheImpl::FileEntry::equals (FileEntry* other) { + if (other->field == field && other->type == type) { + if (other->custom == NULL) { + if (custom == NULL) + return true; + } else if (other->custom->equals (custom)) { + return true; + } + } + }*/ + + /** Composes a hashcode based on the field and type. */ + /*size_t FieldCacheImpl::FileEntry::hashCode() { + return field->hashCode() ^ type ^ (custom==NULL ? 0 : custom->hashCode()); + }*/ + + + + + + /** See if an object is in the cache. */ + FieldCacheAuto* FieldCacheImpl::lookup (IndexReader* reader, const TCHAR* field, int32_t type) { + FieldCacheAuto* ret = NULL; + FileEntry* entry = _CLNEW FileEntry (field, type); + { + SCOPED_LOCK_MUTEX(THIS_LOCK) + fieldcacheCacheReaderType* readerCache = cache.get(reader); + if (readerCache != NULL) + ret = readerCache->get (entry); + _CLDELETE(entry); + } + return ret; + } + + + /** See if a custom object is in the cache. */ + FieldCacheAuto* FieldCacheImpl::lookup (IndexReader* reader, const TCHAR* field, SortComparatorSource* comparer) { + FieldCacheAuto* ret = NULL; + FileEntry* entry = _CLNEW FileEntry (field, comparer); + { + SCOPED_LOCK_MUTEX(THIS_LOCK) + fieldcacheCacheReaderType* readerCache = cache.get(reader); + if (readerCache != NULL) + ret = readerCache->get (entry); + _CLDELETE(entry); +} + return ret; + } + + void FieldCacheImpl::closeCallback(CL_NS(index)::IndexReader* reader, void* fieldCacheImpl){ + FieldCacheImpl* fci = (FieldCacheImpl*)fieldCacheImpl; + SCOPED_LOCK_MUTEX(fci->THIS_LOCK) + fci->cache.remove(reader); + } + + /** Put an object into the cache. */ + void FieldCacheImpl::store (IndexReader* reader, const TCHAR* field, int32_t type, FieldCacheAuto* value) { + FileEntry* entry = _CLNEW FileEntry (field, type); + { + SCOPED_LOCK_MUTEX(THIS_LOCK) + fieldcacheCacheReaderType* readerCache = cache.get(reader); + if (readerCache == NULL) { + readerCache = _CLNEW fieldcacheCacheReaderType; + cache.put(reader,readerCache); + reader->addCloseCallback(closeCallback, this); + } + readerCache->put (entry, value); + //this is supposed to return the previous value, but it needs to be deleted!!! + } + } + + /** Put a custom object into the cache. */ + void FieldCacheImpl::store (IndexReader* reader, const TCHAR* field, SortComparatorSource* comparer, FieldCacheAuto* value) { + FileEntry* entry = _CLNEW FileEntry (field, comparer); + { + SCOPED_LOCK_MUTEX(THIS_LOCK) + fieldcacheCacheReaderType* readerCache = cache.get(reader); + if (readerCache == NULL) { + readerCache = _CLNEW fieldcacheCacheReaderType; + cache.put(reader, readerCache); + reader->addCloseCallback(FieldCacheImpl::closeCallback, this); + } + readerCache->put(entry, value); + //this is supposed to return the previous value, but it needs to be deleted!!! + } + } + + + + + + // inherit javadocs + FieldCacheAuto* FieldCacheImpl::getInts (IndexReader* reader, const TCHAR* field) { + field = CLStringIntern::intern(field CL_FILELINE); + FieldCacheAuto* ret = lookup (reader, field, SortField::INT); + if (ret == NULL) { + int32_t retLen = reader->maxDoc(); + int32_t* retArray = _CL_NEWARRAY(int32_t,retLen); + memset(retArray,0,sizeof(int32_t)*retLen); + if (retLen > 0) { + TermDocs* termDocs = reader->termDocs(); + + Term* term = _CLNEW Term (field, LUCENE_BLANK_STRING, false); + TermEnum* termEnum = reader->terms (term); + _CLDECDELETE(term); + try { + if (termEnum->term(false) == NULL) { + _CLTHROWA(CL_ERR_Runtime,"no terms in field"); //todo: add detailed error: + field); + } + do { + Term* term = termEnum->term(false); + if (term->field() != field) + break; + + TCHAR* end; + int32_t termval = (int32_t)_tcstoi64(term->text(), &end, 10); + termDocs->seek (termEnum); + while (termDocs->next()) { + retArray[termDocs->doc()] = termval; + } + } while (termEnum->next()); + } _CLFINALLY( + termDocs->close(); + _CLDELETE(termDocs); + termEnum->close(); + _CLDELETE(termEnum); + ) + } + + FieldCacheAuto* fa = _CLNEW FieldCacheAuto(retLen,FieldCacheAuto::INT_ARRAY); + fa->intArray = retArray; + + store (reader, field, SortField::INT, fa); + CLStringIntern::unintern(field); + return fa; + } + CLStringIntern::unintern(field); + return ret; + } + + // inherit javadocs + FieldCacheAuto* FieldCacheImpl::getFloats (IndexReader* reader, const TCHAR* field){ + field = CLStringIntern::intern(field CL_FILELINE); + FieldCacheAuto* ret = lookup (reader, field, SortField::FLOAT); + if (ret == NULL) { + int32_t retLen = reader->maxDoc(); + qreal* retArray = _CL_NEWARRAY(qreal,retLen); + memset(retArray,0,sizeof(qreal)*retLen); + if (retLen > 0) { + TermDocs* termDocs = reader->termDocs(); + + Term* term = _CLNEW Term (field, LUCENE_BLANK_STRING, false); + TermEnum* termEnum = reader->terms (term); + _CLDECDELETE(term); + + try { + if (termEnum->term(false) == NULL) { + _CLTHROWA(CL_ERR_Runtime,"no terms in field "); //todo: make richer error + field); + } + do { + Term* term = termEnum->term(false); + if (term->field() != field) + break; + + TCHAR* tmp; + qreal termval = _tcstod(term->text(),&tmp); + termDocs->seek (termEnum); + while (termDocs->next()) { + retArray[termDocs->doc()] = termval; + } + } while (termEnum->next()); + } _CLFINALLY( + termDocs->close(); + _CLDELETE(termDocs); + termEnum->close(); + _CLDELETE(termEnum); + ) + } + + FieldCacheAuto* fa = _CLNEW FieldCacheAuto(retLen,FieldCacheAuto::FLOAT_ARRAY); + fa->floatArray = retArray; + + store (reader, field, SortField::FLOAT, fa); + CLStringIntern::unintern(field); + return fa; + } + CLStringIntern::unintern(field); + return ret; + } + + + // inherit javadocs + FieldCacheAuto* FieldCacheImpl::getStrings (IndexReader* reader, const TCHAR* field){ + //todo: this is not really used, i think? + field = CLStringIntern::intern(field CL_FILELINE); + FieldCacheAuto* ret = lookup (reader, field, SortField::STRING); + if (ret == NULL) { + int32_t retLen = reader->maxDoc(); + TCHAR** retArray = _CL_NEWARRAY(TCHAR*,retLen+1); + memset(retArray,0,sizeof(TCHAR*)*(retLen+1)); + if (retLen > 0) { + TermDocs* termDocs = reader->termDocs(); + + Term* term = _CLNEW Term (field, LUCENE_BLANK_STRING, false); + TermEnum* termEnum = reader->terms (term); + _CLDECDELETE(term); + + try { + if (termEnum->term(false) == NULL) { + _CLTHROWA(CL_ERR_Runtime,"no terms in field "); //todo: extend to + field); + } + do { + Term* term = termEnum->term(false); + if (term->field() != field) + break; + const TCHAR* termval = term->text(); + termDocs->seek (termEnum); + while (termDocs->next()) { + retArray[termDocs->doc()] = STRDUP_TtoT(termval); //todo: any better way of doing this??? + } + } while (termEnum->next()); + } _CLFINALLY( + retArray[retLen]=NULL; + termDocs->close(); + _CLDELETE(termDocs); + termEnum->close(); + _CLDELETE(termEnum); + ) + } + + + FieldCacheAuto* fa = _CLNEW FieldCacheAuto(retLen,FieldCacheAuto::STRING_ARRAY); + fa->stringArray = retArray; + fa->ownContents=true; + store (reader, field, SortField::STRING, fa); + CLStringIntern::unintern(field); + return fa; + } + CLStringIntern::unintern(field); + return ret; + } + + // inherit javadocs + FieldCacheAuto* FieldCacheImpl::getStringIndex (IndexReader* reader, const TCHAR* field){ + field = CLStringIntern::intern(field CL_FILELINE); + FieldCacheAuto* ret = lookup (reader, field, STRING_INDEX); + int32_t t = 0; // current term number + if (ret == NULL) { + int32_t retLen = reader->maxDoc(); + int32_t* retArray = _CL_NEWARRAY(int32_t,retLen); + memset(retArray,0,sizeof(int32_t)*retLen); + + TCHAR** mterms = _CL_NEWARRAY(TCHAR*,retLen+2); + mterms[0]=NULL; + if ( retLen > 0 ) { + TermDocs* termDocs = reader->termDocs(); + + Term* term = _CLNEW Term (field, LUCENE_BLANK_STRING, false); + TermEnum* termEnum = reader->terms (term); + _CLDECDELETE(term); + + + CND_PRECONDITION(t+1 <= retLen, "t out of bounds"); + + // an entry for documents that have no terms in this field + // should a document with no terms be at top or bottom? + // this puts them at the top - if it is changed, FieldDocSortedHitQueue + // needs to change as well. + mterms[t++] = NULL; + + try { + if (termEnum->term(false) == NULL) { + _CLTHROWA(CL_ERR_Runtime,"no terms in field"); //todo: make rich message " + field); + } + do { + Term* term = termEnum->term(false); + if (term->field() != field) + break; + + // store term text + // we expect that there is at most one term per document + if (t >= retLen+1) + _CLTHROWA(CL_ERR_Runtime,"there are more terms than documents in field"); //todo: rich error \"" + field + "\""); + mterms[t] = STRDUP_TtoT(term->text()); + + termDocs->seek (termEnum); + while (termDocs->next()) { + retArray[termDocs->doc()] = t; + } + + t++; + } while (termEnum->next()); + CND_PRECONDITION(t<retLen+2,"t out of bounds"); + mterms[t] = NULL; + } _CLFINALLY( + termDocs->close(); + _CLDELETE(termDocs); + termEnum->close(); + _CLDELETE(termEnum); + ); + + if (t == 0) { + // if there are no terms, make the term array + // have a single NULL entry + _CLDELETE_ARRAY(mterms); + mterms = _CL_NEWARRAY(TCHAR*,1); //todo: delete old mterms? + mterms[0]=NULL; + } else if (t < retLen) { //todo: check, was mterms.length + // if there are less terms than documents, + // trim off the dead array space + //const TCHAR** terms = _CL_NEWARRAY(TCHAR,t); + //System.arraycopy (mterms, 0, terms, 0, t); + //mterms = terms; + + //we simply shorten the length of the array... + + } + } + FieldCache::StringIndex* value = _CLNEW FieldCache::StringIndex (retArray, mterms,t); + + FieldCacheAuto* fa = _CLNEW FieldCacheAuto(retLen,FieldCacheAuto::STRING_INDEX); + fa->stringIndex = value; + fa->ownContents=true; + store (reader, field, STRING_INDEX, fa); + CLStringIntern::unintern(field); + return fa; + } + CLStringIntern::unintern(field); + return ret; + } + + // inherit javadocs + FieldCacheAuto* FieldCacheImpl::getAuto (IndexReader* reader, const TCHAR* field) { + field = CLStringIntern::intern(field CL_FILELINE); + FieldCacheAuto* ret = lookup (reader, field, SortField::AUTO); + if (ret == NULL) { + Term* term = _CLNEW Term (field, LUCENE_BLANK_STRING, false); + TermEnum* enumerator = reader->terms (term); + _CLDECDELETE(term); + + try { + Term* term = enumerator->term(false); + if (term == NULL) { + _CLTHROWA(CL_ERR_Runtime,"no terms in field - cannot determine sort type"); //todo: make rich error: " + field + " + } + if (term->field() == field) { + const TCHAR* termtext = term->text(); + size_t termTextLen = term->textLength(); + + bool isint=true; + for ( size_t i=0;i<termTextLen;i++ ){ + if ( _tcschr(_T("0123456789 +-"),termtext[i]) == NULL ){ + isint = false; + break; + } + } + if ( isint ) + ret = getInts (reader, field); + else{ + bool isfloat=true; + + int32_t searchLen = termTextLen; + if ( termtext[termTextLen-1] == 'f' ) + searchLen--; + for ( int32_t i=0;i<searchLen;i++ ){ + if ( _tcschr(_T("0123456789 Ee.+-"),termtext[i]) == NULL ){ + isfloat = false; + break; + } + } + if ( isfloat ) + ret = getFloats (reader, field); + else{ + ret = getStringIndex (reader, field); + } + } + + if (ret != NULL) { + store (reader, field, SortField::AUTO, ret); + } + } else { + _CLTHROWA (CL_ERR_Runtime,"field does not appear to be indexed"); //todo: make rich error: \"" + field + "\" + } + } _CLFINALLY( enumerator->close(); _CLDELETE(enumerator) ); + + } + CLStringIntern::unintern(field); + return ret; + } + + + // inherit javadocs + FieldCacheAuto* FieldCacheImpl::getCustom (IndexReader* reader, const TCHAR* field, SortComparator* comparator){ + field = CLStringIntern::intern(field CL_FILELINE); + + FieldCacheAuto* ret = lookup (reader, field, comparator); + if (ret == NULL) { + int32_t retLen = reader->maxDoc(); + Comparable** retArray = _CL_NEWARRAY(Comparable*,retLen); + memset(retArray,0,sizeof(Comparable*)*retLen); + if (retLen > 0) { + TermDocs* termDocs = reader->termDocs(); + TermEnum* termEnum = reader->terms (); + + try { + if (termEnum->term(false) == NULL) { + _CLTHROWA(CL_ERR_Runtime,"no terms in field "); //todo: make rich error + field); + } + do { + Term* term = termEnum->term(false); + if (term->field() != field) + break; + Comparable* termval = comparator->getComparable (term->text()); + termDocs->seek (termEnum); + while (termDocs->next()) { + retArray[termDocs->doc()] = termval; + } + } while (termEnum->next()); + } _CLFINALLY ( + termDocs->close(); + _CLDELETE(termDocs); + termEnum->close(); + _CLDELETE(termEnum); + ); + } + + FieldCacheAuto* fa = _CLNEW FieldCacheAuto(retLen,FieldCacheAuto::COMPARABLE_ARRAY); + fa->comparableArray = retArray; + fa->ownContents=true; + store (reader, field, SortField::CUSTOM, fa); + CLStringIntern::unintern(field); + return fa; + } + CLStringIntern::unintern(field); + return ret; + } + + + FieldCacheImpl::fieldcacheCacheReaderType::fieldcacheCacheReaderType(){ + setDeleteKey(false); + setDeleteValue(false); + } + FieldCacheImpl::fieldcacheCacheReaderType::~fieldcacheCacheReaderType(){ + iterator itr = begin(); + while ( itr != end() ){ + FileEntry* f = itr->first; + if ( f->getType() != SortField::AUTO ) + _CLDELETE( itr->second ); + _CLDELETE( f ); + ++itr; + } + clear(); + } +CL_NS_END diff --git a/src/assistant/3rdparty/clucene/src/CLucene/search/FieldCacheImpl.h b/src/assistant/3rdparty/clucene/src/CLucene/search/FieldCacheImpl.h new file mode 100644 index 000000000..ac3c4cabc --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/search/FieldCacheImpl.h @@ -0,0 +1,144 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _lucene_search_FieldCacheImpl_ +#define _lucene_search_FieldCacheImpl_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include "CLucene/index/IndexReader.h" +#include "FieldCache.h" +#include "Sort.h" + + +CL_NS_DEF(search) + + +/** + * Expert: The default cache implementation, storing all values in memory. + * + */ +class FieldCacheImpl: public FieldCache { +public: + DEFINE_MUTEX(THIS_LOCK) + + /** Expert: Every key in the internal cache is of this type. */ + class FileEntry:LUCENE_BASE { + const TCHAR* field; // which Field + int32_t type; // which SortField type + SortComparatorSource* custom; // which custom comparator + size_t _hashCode; + public: + /** Creates one of these objects. */ + FileEntry (const TCHAR* field, int32_t type); + + /** Creates one of these objects for a custom comparator. */ + FileEntry (const TCHAR* field, SortComparatorSource* custom); + ~FileEntry(); + + int32_t getType() const{ return type; } + + /** Two of these are equal iff they reference the same field and type. */ + bool equals (FileEntry* other) const; + + /** Composes a hashcode based on the field and type. */ + size_t hashCode(); + + int32_t compareTo(const FileEntry* other) const; + + class Compare:LUCENE_BASE, public CL_NS(util)::Compare::_base //<Term*> + { + public: + bool operator()( FileEntry* f1, FileEntry* f2 ) const{ + return ( f1->compareTo(f2) < 0 ); + } + size_t operator()( FileEntry* t ) const{ + return t->hashCode(); + } + }; + class Equals:LUCENE_BASE, public CL_NS(util)::Compare::_base //<Term*> + { + public: + bool operator()( FileEntry* f1, FileEntry* f2 ) const{ + return ( f1->compareTo(f2) == 0 ); + } + }; + }; + + FieldCacheImpl(); + ~FieldCacheImpl(); +private: + + ///the type that is stored in the field cache. can't use a typedef because + ///the decorated name would become too long + class fieldcacheCacheReaderType: public CL_NS(util)::CLHashMap<FileEntry*, + FieldCacheAuto*, + FileEntry::Compare, + FileEntry::Equals, + CL_NS(util)::Deletor::Object<FileEntry>, + CL_NS(util)::Deletor::Object<FieldCacheAuto> >{ + public: + fieldcacheCacheReaderType(); + ~fieldcacheCacheReaderType(); + }; + + //note: typename gets too long if using cacheReaderType as a typename + typedef CL_NS(util)::CLHashMap<CL_NS(index)::IndexReader*, + fieldcacheCacheReaderType*, + CL_NS(util)::Compare::Void<CL_NS(index)::IndexReader>, + CL_NS(util)::Equals::Void<CL_NS(index)::IndexReader>, + CL_NS(util)::Deletor::Object<CL_NS(index)::IndexReader>, + CL_NS(util)::Deletor::Object<fieldcacheCacheReaderType> > fieldcacheCacheType; + + /** The internal cache. Maps FileEntry to array of interpreted term values. **/ + //todo: make indexreader remove itself from here when the reader is shut + fieldcacheCacheType cache; + + /** See if an object is in the cache. */ + FieldCacheAuto* lookup (CL_NS(index)::IndexReader* reader, const TCHAR* field, int32_t type) ; + + /** See if a custom object is in the cache. */ + FieldCacheAuto* lookup (CL_NS(index)::IndexReader* reader, const TCHAR* field, SortComparatorSource* comparer); + + /** Put an object into the cache. */ + void store (CL_NS(index)::IndexReader* reader, const TCHAR* field, int32_t type, FieldCacheAuto* value); + + /** Put a custom object into the cache. */ + void store (CL_NS(index)::IndexReader* reader, const TCHAR* field, SortComparatorSource* comparer, FieldCacheAuto* value); + +public: + + // inherit javadocs + FieldCacheAuto* getInts (CL_NS(index)::IndexReader* reader, const TCHAR* field); + + // inherit javadocs + FieldCacheAuto* getFloats (CL_NS(index)::IndexReader* reader, const TCHAR* field); + + // inherit javadocs + FieldCacheAuto* getStrings (CL_NS(index)::IndexReader* reader, const TCHAR* field); + + // inherit javadocs + FieldCacheAuto* getStringIndex (CL_NS(index)::IndexReader* reader, const TCHAR* field); + + // inherit javadocs + FieldCacheAuto* getAuto (CL_NS(index)::IndexReader* reader, const TCHAR* field); + + // inherit javadocs + FieldCacheAuto* getCustom (CL_NS(index)::IndexReader* reader, const TCHAR* field, SortComparator* comparator); + + + /** + * Callback for when IndexReader closes. This causes + * any cache to be removed for the specified reader. + */ + static void closeCallback(CL_NS(index)::IndexReader* reader, void* fieldCacheImpl); +}; + + +CL_NS_END +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/search/FieldDoc.h b/src/assistant/3rdparty/clucene/src/CLucene/search/FieldDoc.h new file mode 100644 index 000000000..6ce915acf --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/search/FieldDoc.h @@ -0,0 +1,70 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _lucene_search_FieldDoc_ +#define _lucene_search_FieldDoc_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include "ScoreDoc.h" + +CL_NS_DEF(search) + +/** + * Expert: A ScoreDoc which also contains information about + * how to sort the referenced document. In addition to the + * document number and score, this object contains an array + * of values for the document from the field(s) used to sort. + * For example, if the sort criteria was to sort by fields + * "a", "b" then "c", the <code>fields</code> object array + * will have three elements, corresponding respectively to + * the term values for the document in fields "a", "b" and "c". + * The class of each element in the array will be either + * Integer, Float or String depending on the type of values + * in the terms of each field. + * + * @see ScoreDoc + * @see TopFieldDocs + */ +class FieldDoc: public ScoreDoc { +public: + + /** Expert: The values which are used to sort the referenced document. + * The order of these will match the original sort criteria given by a + * Sort object. Each Object will be either an Integer, Float or String, + * depending on the type of values in the terms of the original field. + * @see Sort + * @see Searchable#search(Query,Filter,int32_t,Sort) + */ + CL_NS(util)::Comparable** fields; + + /** Expert: Creates one of these objects with empty sort information. */ + FieldDoc (int32_t doc, qreal score): + ScoreDoc(doc,score) { + fields=NULL; + } + + /** Expert: Creates one of these objects with the given sort information. */ + FieldDoc (int32_t doc, qreal score, CL_NS(util)::Comparable** fields): + ScoreDoc(doc,score) + { + this->fields = fields; + } + + ~FieldDoc(){ + if ( fields != NULL ){ + for ( int i=0;fields[i]!=NULL;i++ ) + _CLDELETE(fields[i]); + _CLDELETE_ARRAY(fields); + } + } +}; + +CL_NS_END +#endif + diff --git a/src/assistant/3rdparty/clucene/src/CLucene/search/FieldDocSortedHitQueue.cpp b/src/assistant/3rdparty/clucene/src/CLucene/search/FieldDocSortedHitQueue.cpp new file mode 100644 index 000000000..0a5210903 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/search/FieldDocSortedHitQueue.cpp @@ -0,0 +1,171 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#include "CLucene/StdHeader.h" +#include "FieldDocSortedHitQueue.h" + + +CL_NS_USE(util) +CL_NS_DEF(search) + + +FieldDoc::FieldDoc (int32_t doc, qreal score) +{ + this->scoreDoc.doc = doc; + this->scoreDoc.score = score; + fields=NULL; +} + +FieldDoc::FieldDoc (int32_t doc, qreal score, CL_NS(util)::Comparable** fields) +{ + this->scoreDoc.doc = doc; + this->scoreDoc.score = score; + this->fields = fields; +} + +FieldDoc::~FieldDoc(){ + if ( fields != NULL ){ + for ( int i=0;fields[i]!=NULL;i++ ) + _CLDELETE(fields[i]); + _CLDELETE_ARRAY(fields); + } +} + + + +FieldDocSortedHitQueue::FieldDocSortedHitQueue (SortField** fields, int32_t size) { + this->fields = fields; + _countsize(); + //this->collators = hasCollators (fields); + initialize (size,true); +} + +bool FieldDocSortedHitQueue::lessThan (FieldDoc* docA, FieldDoc* docB) { + int32_t n = fieldsLen; + int32_t c = 0; + qreal f1,f2,r1,r2; + int32_t i1,i2; + const TCHAR *s1, *s2; + + for (int32_t i=0; i<n && c==0; ++i) { + int32_t type = fields[i]->getType(); + if (fields[i]->getReverse()) { + switch (type) { + case SortField::DOCSCORE: + r1 = __REINTERPRET_CAST(Compare::Float*, docA->fields[i])->getValue(); + r2 = __REINTERPRET_CAST(Compare::Float*, docB->fields[i])->getValue(); + if (r1 < r2) c = -1; + if (r1 > r2) c = 1; + break; + case SortField::DOC: + case SortField::INT: + i1 = __REINTERPRET_CAST(Compare::Int32*, docA->fields[i])->getValue(); + i2 = __REINTERPRET_CAST(Compare::Int32*, docB->fields[i])->getValue(); + if (i1 > i2) c = -1; + if (i1 < i2) c = 1; + break; + case SortField::STRING: + s1 = __REINTERPRET_CAST(Compare::TChar*, docA->fields[i])->getValue(); + s2 = __REINTERPRET_CAST(Compare::TChar*, docB->fields[i])->getValue(); + if (s2 == NULL) c = -1; // could be NULL if there are + else if (s1 == NULL) c = 1; // no terms in the given field + else c = _tcscmp(s2,s1); //else if (fields[i].getLocale() == NULL) { + + /*todo: collators not impl + } else { + c = collators[i].compare (s2, s1); + }*/ + break; + case SortField::FLOAT: + f1 = __REINTERPRET_CAST(Compare::Float*, docA->fields[i])->getValue(); + f2 = __REINTERPRET_CAST(Compare::Float*, docB->fields[i])->getValue(); + if (f1 > f2) c = -1; + if (f1 < f2) c = 1; + break; + case SortField::CUSTOM: + c = docB->fields[i]->compareTo (docA->fields[i]); + break; + case SortField::AUTO: + // we cannot handle this - even if we determine the type of object (qreal or + // Integer), we don't necessarily know how to compare them (both SCORE and + // qreal both contain floats, but are sorted opposite of each other). Before + // we get here, each AUTO should have been replaced with its actual value. + _CLTHROWA (CL_ERR_Runtime,"FieldDocSortedHitQueue cannot use an AUTO SortField"); + default: + _CLTHROWA (CL_ERR_Runtime, "invalid SortField type"); //todo: rich error... : "+type); + } + } else { + switch (type) { + case SortField::DOCSCORE: + r1 = __REINTERPRET_CAST(Compare::Float*, docA->fields[i])->getValue(); + r2 = __REINTERPRET_CAST(Compare::Float*, docB->fields[i])->getValue(); + if (r1 > r2) c = -1; + if (r1 < r2) c = 1; + break; + case SortField::DOC: + case SortField::INT: + i1 = __REINTERPRET_CAST(Compare::Int32*, docA->fields[i])->getValue(); + i2 = __REINTERPRET_CAST(Compare::Int32*, docB->fields[i])->getValue(); + if (i1 < i2) c = -1; + if (i1 > i2) c = 1; + break; + case SortField::STRING: + s1 = __REINTERPRET_CAST(Compare::TChar*, docA->fields[i])->getValue(); + s2 = __REINTERPRET_CAST(Compare::TChar*, docB->fields[i])->getValue(); + // NULL values need to be sorted first, because of how FieldCache.getStringIndex() + // works - in that routine, any documents without a value in the given field are + // put first. + if (s1 == NULL) c = -1; // could be NULL if there are + else if (s2 == NULL) c = 1; // no terms in the given field + else c = _tcscmp(s1,s2); //else if (fields[i].getLocale() == NULL) { + + /* todo: collators not implemented } else { + c = collators[i].compare (s1, s2); + }*/ + break; + case SortField::FLOAT: + f1 = __REINTERPRET_CAST(Compare::Float*, docA->fields[i])->getValue(); + f2 = __REINTERPRET_CAST(Compare::Float*, docB->fields[i])->getValue(); + if (f1 < f2) c = -1; + if (f1 > f2) c = 1; + break; + case SortField::CUSTOM: + c = docA->fields[i]->compareTo (docB->fields[i]); + break; + case SortField::AUTO: + // we cannot handle this - even if we determine the type of object (qreal or + // Integer), we don't necessarily know how to compare them (both SCORE and + // qreal both contain floats, but are sorted opposite of each other). Before + // we get here, each AUTO should have been replaced with its actual value. + _CLTHROWA (CL_ERR_Runtime,"FieldDocSortedHitQueue cannot use an AUTO SortField"); + default: + _CLTHROWA (CL_ERR_Runtime,"invalid SortField type"); //todo: rich error... : "+type); + } + } + } + return c > 0; +} + +void FieldDocSortedHitQueue::setFields (SortField** fields) { + SCOPED_LOCK_MUTEX(THIS_LOCK) + if (this->fields == NULL) { + this->fields = fields; + _countsize(); + //this->collators = hasCollators (fields); + }else if ( fields == NULL ) + this->fields = NULL; +} + +FieldDocSortedHitQueue::~FieldDocSortedHitQueue(){ + if ( fields != NULL ){ + for ( int i=0;fields[i]!=NULL;i++ ) + _CLDELETE(fields[i]); + _CLDELETE_ARRAY(fields); + } +} + +CL_NS_END + diff --git a/src/assistant/3rdparty/clucene/src/CLucene/search/FieldDocSortedHitQueue.h b/src/assistant/3rdparty/clucene/src/CLucene/search/FieldDocSortedHitQueue.h new file mode 100644 index 000000000..5a46b3b65 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/search/FieldDocSortedHitQueue.h @@ -0,0 +1,159 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _lucene_search_FieldDocSortedHitQueue_ +#define _lucene_search_FieldDocSortedHitQueue_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include "Sort.h" +#include "CLucene/util/PriorityQueue.h" + +CL_NS_DEF(search) + +/** + * Expert: A ScoreDoc which also contains information about + * how to sort the referenced document. In addition to the + * document number and score, this object contains an array + * of values for the document from the field(s) used to sort. + * For example, if the sort criteria was to sort by fields + * "a", "b" then "c", the <code>fields</code> object array + * will have three elements, corresponding respectively to + * the term values for the document in fields "a", "b" and "c". + * The class of each element in the array will be either + * Integer, Float or String depending on the type of values + * in the terms of each field. + * + * @see ScoreDoc + * @see TopFieldDocs + */ +class FieldDoc: LUCENE_BASE { +public: + //FieldDoc did inherit from ScoreDoc, but now we make the scoredoc a member + struct ScoreDoc scoreDoc; + + /** Expert: The values which are used to sort the referenced document. + * The order of these will match the original sort criteria given by a + * Sort object. Each Object will be either an Integer, Float or String, + * depending on the type of values in the terms of the original field. + * @see Sort + * @see Searchable#search(Query,Filter,int32_t,Sort) + */ + CL_NS(util)::Comparable** fields; + + /** Expert: Creates one of these objects with empty sort information. */ + FieldDoc (int32_t doc, qreal score); + /** Expert: Creates one of these objects with the given sort information. */ + FieldDoc (int32_t doc, qreal score, CL_NS(util)::Comparable** fields); + ~FieldDoc(); +}; + +/** + * Expert: Collects sorted results from Searchable's and collates them. + * The elements put into this queue must be of type FieldDoc. + */ +class FieldDocSortedHitQueue: + public CL_NS(util)::PriorityQueue<FieldDoc*,CL_NS(util)::Deletor::Object<FieldDoc> > +{ +private: + DEFINE_MUTEX(THIS_LOCK) + + // this cannot contain AUTO fields - any AUTO fields should + // have been resolved by the time this class is used. + SortField** fields; + int32_t fieldsLen; + + void _countsize(){ + fieldsLen=0; + while(fields[fieldsLen]!=NULL) + fieldsLen++; + } + + // used in the case where the fields are sorted by locale + // based strings + //todo: not implemented in clucene because locales has not been implemented + //Collator[] collators; //volatile + +public: + /** + * Creates a hit queue sorted by the given list of fields. + * @param fields Field names, in priority order (highest priority first). + * @param size The number of hits to retain. Must be greater than zero. + */ + FieldDocSortedHitQueue (SortField** fields, int32_t size); + ~FieldDocSortedHitQueue(); + + + /** + * Allows redefinition of sort fields if they are <code>NULL</code>. + * This is to handle the case using ParallelMultiSearcher where the + * original list contains AUTO and we don't know the actual sort + * type until the values come back. The fields can only be set once. + * This method is thread safe. + * @param fields + */ + void setFields (SortField** fields); + + /** Returns the fields being used to sort. */ + SortField** getFields() { + return fields; + } + + /** Returns an array of collators, possibly <code>NULL</code>. The collators + * correspond to any SortFields which were given a specific locale. + * @param fields Array of sort fields. + * @return Array, possibly <code>NULL</code>. + + private Collator[] hasCollators (SortField[] fields) { + if (fields == NULL) return NULL; + Collator[] ret = new Collator[fields.length]; + for (int32_t i=0; i<fields.length; ++i) { + Locale locale = fields[i].getLocale(); + if (locale != NULL) + ret[i] = Collator.getInstance (locale); + } + return ret; + }*/ + +protected: + /** + * Returns whether <code>a</code> is less relevant than <code>b</code>. + * @param a FieldDoc + * @param b FieldDoc + * @return <code>true</code> if document <code>a</code> should be sorted after document <code>b</code>. + */ + bool lessThan (FieldDoc* docA, FieldDoc* docB); +}; + + +/** +* Expert: Returned by low-level sorted search implementations. +* +* @see Searchable#search(Query,Filter,int32_t,Sort) +*/ +class TopFieldDocs: public TopDocs { +public: + /// The fields which were used to sort results by. + SortField** fields; + + FieldDoc** fieldDocs; + + /** Creates one of these objects. + * @param totalHits Total number of hits for the query. + * @param fieldDocs The top hits for the query. + * @param scoreDocs The top hits for the query. + * @param scoreDocsLen Length of fieldDocs and scoreDocs + * @param fields The sort criteria used to find the top hits. + */ + TopFieldDocs (int32_t totalHits, FieldDoc** fieldDocs, int32_t scoreDocsLen, SortField** fields); + ~TopFieldDocs(); +}; + +CL_NS_END +#endif + diff --git a/src/assistant/3rdparty/clucene/src/CLucene/search/FieldSortedHitQueue.cpp b/src/assistant/3rdparty/clucene/src/CLucene/search/FieldSortedHitQueue.cpp new file mode 100644 index 000000000..04f45e931 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/search/FieldSortedHitQueue.cpp @@ -0,0 +1,212 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#include "CLucene/StdHeader.h" +#include "FieldSortedHitQueue.h" +#include "FieldDocSortedHitQueue.h" +#include "Compare.h" + +CL_NS_USE(util) +CL_NS_USE(index) +CL_NS_DEF(search) + +FieldSortedHitQueue::hitqueueCacheType FieldSortedHitQueue::Comparators(false,true); + +FieldSortedHitQueue::FieldSortedHitQueue (IndexReader* reader, SortField** _fields, int32_t size): + fieldsLen(0), + maxscore(1.0f) +{ + while ( _fields[fieldsLen] != 0 ) + fieldsLen++; + + comparators = _CL_NEWARRAY(ScoreDocComparator*,fieldsLen+1); + SortField** tmp = _CL_NEWARRAY(SortField*,fieldsLen+1); + for (int32_t i=0; i<fieldsLen; ++i) { + const TCHAR* fieldname = _fields[i]->getField(); + //todo: fields[i].getLocale(), not implemented + comparators[i] = getCachedComparator (reader, fieldname, _fields[i]->getType(), _fields[i]->getFactory()); + tmp[i] = _CLNEW SortField (fieldname, comparators[i]->sortType(), _fields[i]->getReverse()); + } + comparatorsLen = fieldsLen; + comparators[fieldsLen]=NULL; + tmp[fieldsLen] = NULL; + this->fields = tmp; + + initialize(size,true); +} + + +bool FieldSortedHitQueue::lessThan (FieldDoc* docA, FieldDoc* docB) { + // keep track of maximum score + if (docA->scoreDoc.score > maxscore) maxscore = docA->scoreDoc.score; + if (docB->scoreDoc.score > maxscore) maxscore = docB->scoreDoc.score; + + // run comparators + int32_t c = 0; + for ( int32_t i=0; c==0 && i<comparatorsLen; ++i ) { + c = (fields[i]->getReverse()) ? comparators[i]->compare (&docB->scoreDoc, &docA->scoreDoc) : + comparators[i]->compare (&docA->scoreDoc, &docB->scoreDoc); + } + // avoid random sort order that could lead to duplicates (bug #31241): + if (c == 0) + return docA->scoreDoc.doc > docB->scoreDoc.doc; + return c > 0; +} + + +//static +ScoreDocComparator* FieldSortedHitQueue::comparatorString (IndexReader* reader, const TCHAR* field) { + //const TCHAR* field = CLStringIntern::intern(fieldname CL_FILELINE); + FieldCacheAuto* fa = FieldCache::DEFAULT->getStringIndex (reader, field); + //CLStringIntern::unintern(field); + + CND_PRECONDITION(fa->contentType==FieldCacheAuto::STRING_INDEX,"Content type is incorrect"); + fa->ownContents = false; + return _CLNEW ScoreDocComparators::String(fa->stringIndex, fa->contentLen); +} + +//static +ScoreDocComparator* FieldSortedHitQueue::comparatorInt (IndexReader* reader, const TCHAR* field){ + //const TCHAR* field = CLStringIntern::intern(fieldname CL_FILELINE); + FieldCacheAuto* fa = FieldCache::DEFAULT->getInts (reader, field); + //CLStringIntern::unintern(field); + + CND_PRECONDITION(fa->contentType==FieldCacheAuto::INT_ARRAY,"Content type is incorrect"); + return _CLNEW ScoreDocComparators::Int32(fa->intArray, fa->contentLen); + } + +//static + ScoreDocComparator* FieldSortedHitQueue::comparatorFloat (IndexReader* reader, const TCHAR* field) { + //const TCHAR* field = CLStringIntern::intern(fieldname CL_FILELINE); + FieldCacheAuto* fa = FieldCache::DEFAULT->getFloats (reader, field); + //CLStringIntern::unintern(field); + + CND_PRECONDITION(fa->contentType==FieldCacheAuto::FLOAT_ARRAY,"Content type is incorrect"); + return _CLNEW ScoreDocComparators::Float (fa->floatArray, fa->contentLen); + } +//static + ScoreDocComparator* FieldSortedHitQueue::comparatorAuto (IndexReader* reader, const TCHAR* field){ + //const TCHAR* field = CLStringIntern::intern(fieldname CL_FILELINE); + FieldCacheAuto* fa = FieldCache::DEFAULT->getAuto (reader, field); + //CLStringIntern::unintern(field); + + if (fa->contentType == FieldCacheAuto::STRING_INDEX ) { + return comparatorString (reader, field); + } else if (fa->contentType == FieldCacheAuto::INT_ARRAY) { + return comparatorInt (reader, field); + } else if (fa->contentType == FieldCacheAuto::FLOAT_ARRAY) { + return comparatorFloat (reader, field); + } else if (fa->contentType == FieldCacheAuto::STRING_ARRAY) { + return comparatorString (reader, field); + } else { + _CLTHROWA(CL_ERR_Runtime, "unknown data type in field"); //todo: rich error information: '"+field+"'"); + } + } + + + //todo: Locale locale, not implemented yet + ScoreDocComparator* FieldSortedHitQueue::getCachedComparator (IndexReader* reader, const TCHAR* fieldname, int32_t type, SortComparatorSource* factory){ + if (type == SortField::DOC) + return ScoreDocComparator::INDEXORDER; + if (type == SortField::DOCSCORE) + return ScoreDocComparator::RELEVANCE; + ScoreDocComparator* comparator = lookup (reader, fieldname, type, factory); + if (comparator == NULL) { + switch (type) { + case SortField::AUTO: + comparator = comparatorAuto (reader, fieldname); + break; + case SortField::INT: + comparator = comparatorInt (reader, fieldname); + break; + case SortField::FLOAT: + comparator = comparatorFloat (reader, fieldname); + break; + case SortField::STRING: + //if (locale != NULL) + // comparator = comparatorStringLocale (reader, fieldname, locale); + //else + comparator = comparatorString (reader, fieldname); + break; + case SortField::CUSTOM: + comparator = factory->newComparator (reader, fieldname); + break; + default: + _CLTHROWA(CL_ERR_Runtime,"unknown field type"); + //todo: extend error + //throw _CLNEW RuntimeException ("unknown field type: "+type); + } + store (reader, fieldname, type, factory, comparator); + } + return comparator; + } + + + FieldDoc* FieldSortedHitQueue::fillFields (FieldDoc* doc) const{ + int32_t n = comparatorsLen; + Comparable** fields = _CL_NEWARRAY(Comparable*,n+1); + for (int32_t i=0; i<n; ++i) + fields[i] = comparators[i]->sortValue(&doc->scoreDoc); + fields[n]=NULL; + doc->fields = fields; + if (maxscore > 1.0f) + doc->scoreDoc.score /= maxscore; // normalize scores + return doc; + } + + ScoreDocComparator* FieldSortedHitQueue::lookup (IndexReader* reader, const TCHAR* field, int32_t type, SortComparatorSource* factory) { + ScoreDocComparator* sdc = NULL; + FieldCacheImpl::FileEntry* entry = (factory != NULL) + ? _CLNEW FieldCacheImpl::FileEntry (field, factory) + : _CLNEW FieldCacheImpl::FileEntry (field, type); + + { + SCOPED_LOCK_MUTEX(Comparators.THIS_LOCK) + hitqueueCacheReaderType* readerCache = Comparators.get(reader); + if (readerCache == NULL){ + _CLDELETE(entry); + return NULL; + } + + sdc = readerCache->get (entry); + _CLDELETE(entry); + } + return sdc; + } + + void FieldSortedHitQueue::closeCallback(CL_NS(index)::IndexReader* reader, void*){ + SCOPED_LOCK_MUTEX(Comparators.THIS_LOCK) + Comparators.remove(reader); + } + + //static + void FieldSortedHitQueue::store (IndexReader* reader, const TCHAR* field, int32_t type, SortComparatorSource* factory, ScoreDocComparator* value) { + FieldCacheImpl::FileEntry* entry = (factory != NULL) + ? _CLNEW FieldCacheImpl::FileEntry (field, factory) + : _CLNEW FieldCacheImpl::FileEntry (field, type); + + { + SCOPED_LOCK_MUTEX(Comparators.THIS_LOCK) + hitqueueCacheReaderType* readerCache = Comparators.get(reader); + if (readerCache == NULL) { + readerCache = _CLNEW hitqueueCacheReaderType(true); + Comparators.put(reader,readerCache); + reader->addCloseCallback(FieldSortedHitQueue::closeCallback,NULL); + } + readerCache->put (entry, value); + //return NULL; //supposed to return previous value... + } + } + +FieldSortedHitQueue::~FieldSortedHitQueue(){ + _CLDELETE_ARRAY(comparators); + if ( fields != NULL ){ + for ( int i=0;fields[i]!=NULL;i++ ) + _CLDELETE(fields[i]); + _CLDELETE_ARRAY(fields); + } +} +CL_NS_END diff --git a/src/assistant/3rdparty/clucene/src/CLucene/search/FieldSortedHitQueue.h b/src/assistant/3rdparty/clucene/src/CLucene/search/FieldSortedHitQueue.h new file mode 100644 index 000000000..d7b16ce9e --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/search/FieldSortedHitQueue.h @@ -0,0 +1,216 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _lucene_search_FieldSortedHitQueue_ +#define _lucene_search_FieldSortedHitQueue_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include "FieldCache.h" +#include "Sort.h" +#include "FieldDocSortedHitQueue.h" +#include "SearchHeader.h" +#include "FieldCacheImpl.h" +#include "CLucene/util/PriorityQueue.h" + +CL_NS_DEF(search) + + +/** + * Expert: A hit queue for sorting by hits by terms in more than one field. + * Uses <code>FieldCache.DEFAULT</code> for maintaining internal term lookup tables. + * + * @see Searchable#search(Query,Filter,int32_t,Sort) + * @see FieldCache + */ +class FieldSortedHitQueue: public CL_NS(util)::PriorityQueue<FieldDoc*, + CL_NS(util)::Deletor::Object<FieldDoc> > { + + ///the type that is stored in the field cache. can't use a typedef because + ///the decorated name would become too long + class hitqueueCacheReaderType: public CL_NS(util)::CLHashMap<FieldCacheImpl::FileEntry*, + ScoreDocComparator*, + FieldCacheImpl::FileEntry::Compare, + FieldCacheImpl::FileEntry::Equals, + CL_NS(util)::Deletor::Object<FieldCacheImpl::FileEntry>, + CL_NS(util)::Deletor::Object<ScoreDocComparator> >{ + + public: + hitqueueCacheReaderType(bool deleteValue){ + setDeleteKey(true); + setDeleteValue(deleteValue); + } + ~hitqueueCacheReaderType(){ + clear(); + } + + }; + +public: //todo: remove this and below after close callback is implemented + //note: typename gets too long if using cacheReaderType as a typename + typedef CL_NS(util)::CLHashMap<CL_NS(index)::IndexReader*, + hitqueueCacheReaderType*, + CL_NS(util)::Compare::Void<CL_NS(index)::IndexReader>, + CL_NS(util)::Equals::Void<CL_NS(index)::IndexReader>, + CL_NS(util)::Deletor::Object<CL_NS(index)::IndexReader>, + CL_NS(util)::Deletor::Object<hitqueueCacheReaderType> > hitqueueCacheType; + + /** Internal cache of comparators. Similar to FieldCache, only + * caches comparators instead of term values. + */ + static hitqueueCacheType Comparators; +private: + + /** Returns a comparator if it is in the cache.*/ + static ScoreDocComparator* lookup (CL_NS(index)::IndexReader* reader, const TCHAR* field, int32_t type, SortComparatorSource* factory); + + /** Stores a comparator into the cache. + returns the valid ScoreDocComparator. + */ + static void store (CL_NS(index)::IndexReader* reader, const TCHAR* field, int32_t type, SortComparatorSource* factory, ScoreDocComparator* value); + + + //todo: Locale locale, not implemented yet + static ScoreDocComparator* getCachedComparator (CL_NS(index)::IndexReader* reader, + const TCHAR* fieldname, int32_t type, SortComparatorSource* factory); + + + /** + * Returns a comparator for sorting hits according to a field containing integers. + * @param reader Index to use. + * @param fieldname Field containg integer values. + * @return Comparator for sorting hits. + * @throws IOException If an error occurs reading the index. + */ + static ScoreDocComparator* comparatorInt (CL_NS(index)::IndexReader* reader, const TCHAR* fieldname); + + /** + * Returns a comparator for sorting hits according to a field containing floats. + * @param reader Index to use. + * @param fieldname Field containg float values. + * @return Comparator for sorting hits. + * @throws IOException If an error occurs reading the index. + */ + static ScoreDocComparator* comparatorFloat (CL_NS(index)::IndexReader* reader, const TCHAR* fieldname); + + /** + * Returns a comparator for sorting hits according to a field containing strings. + * @param reader Index to use. + * @param fieldname Field containg string values. + * @return Comparator for sorting hits. + * @throws IOException If an error occurs reading the index. + */ + static ScoreDocComparator* comparatorString (CL_NS(index)::IndexReader* reader, const TCHAR* fieldname); + + + //todo: + /** + * Returns a comparator for sorting hits according to a field containing strings. + * @param reader Index to use. + * @param fieldname Field containg string values. + * @return Comparator for sorting hits. + * @throws IOException If an error occurs reading the index. + + static ScoreDocComparator* comparatorStringLocale (IndexReader* reader, TCHAR* fieldname, Locale locale){ + Collator collator = Collator.getInstance (locale); + TCHAR* field = fieldname.intern(); + TCHAR** index = FieldCache.DEFAULT.getStrings (reader, field); + return _CLNEW ScoreDocComparator() { + + public int32_t compare (ScoreDoc i, ScoreDoc j) { + return collator.compare (index[i.doc], index[j.doc]); + } + + public Comparable sortValue (ScoreDoc i) { + return index[i.doc]; + } + + public int32_t sortType() { + return SortField.STRING; + } + }; + }*/ + + /** + * Returns a comparator for sorting hits according to values in the given field. + * The terms in the field are looked at to determine whether they contain integers, + * floats or strings. Once the type is determined, one of the other static methods + * in this class is called to get the comparator. + * @param reader Index to use. + * @param fieldname Field containg values. + * @return Comparator for sorting hits. + * @throws IOException If an error occurs reading the index. + */ + static ScoreDocComparator* comparatorAuto (CL_NS(index)::IndexReader* reader, const TCHAR* fieldname); + + +protected: + /** Stores a comparator corresponding to each field being sorted by */ + ScoreDocComparator** comparators; + int32_t comparatorsLen; + + /** Stores the sort criteria being used. */ + SortField** fields; + int32_t fieldsLen; + + /** Stores the maximum score value encountered, for normalizing. + * we only care about scores greater than 1.0 - if all the scores + * are less than 1.0, we don't have to normalize. */ + qreal maxscore; + + /** + * Returns whether <code>a</code> is less relevant than <code>b</code>. + * @param a ScoreDoc + * @param b ScoreDoc + * @return <code>true</code> if document <code>a</code> should be sorted after document <code>b</code>. + */ + bool lessThan (FieldDoc* docA, FieldDoc* docB); +public: + + /** + * Creates a hit queue sorted by the given list of fields. + * @param reader Index to use. + * @param fields Field names, in priority order (highest priority first). Cannot be <code>null</code> or empty. + * @param size The number of hits to retain. Must be greater than zero. + * @throws IOException + */ + FieldSortedHitQueue (CL_NS(index)::IndexReader* reader, SortField** fields, int32_t size); + + ~FieldSortedHitQueue(); + + /** + * Callback for when IndexReader closes. This causes + * any Comparators to be removed for the specified reader. + */ + static void closeCallback(CL_NS(index)::IndexReader* reader, void* param); + + /** + * Given a FieldDoc object, stores the values used + * to sort the given document. These values are not the raw + * values out of the index, but the internal representation + * of them. This is so the given search hit can be collated + * by a MultiSearcher with other search hits. + * @param doc The FieldDoc to store sort values into. + * @return The same FieldDoc passed in. + * @see Searchable#search(Query,Filter,int32_t,Sort) + */ + FieldDoc* fillFields (FieldDoc* doc) const; + + void setFields (SortField** fields){ + this->fields = fields; + } + + /** Returns the SortFields being used by this hit queue. */ + SortField** getFields() { + return fields; + } +}; + + +CL_NS_END +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/search/Filter.h b/src/assistant/3rdparty/clucene/src/CLucene/search/Filter.h new file mode 100644 index 000000000..309c5a9d6 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/search/Filter.h @@ -0,0 +1,46 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _lucene_search_Filter_ +#define _lucene_search_Filter_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include "CLucene/index/IndexReader.h" +#include "CLucene/util/BitSet.h" + +CL_NS_DEF(search) + // Abstract base class providing a mechanism to restrict searches to a subset + // of an index. + class Filter: LUCENE_BASE { + public: + virtual ~Filter(){ + } + + virtual Filter* clone() const = 0; + + /** + * Returns a BitSet with true for documents which should be permitted in + * search results, and false for those that should not. + * MEMORY: read shouldDeleteBitSet + */ + virtual CL_NS(util)::BitSet* bits(CL_NS(index)::IndexReader* reader)=0; + + /** + * Because of the problem of cached bitsets with the CachingWrapperFilter, + * CLucene has no way of knowing whether to delete the bitset returned from bits(). + * To properly clean memory from bits(), pass the bitset to this function. The + * Filter should be deleted if this function returns true. + */ + virtual bool shouldDeleteBitSet(const CL_NS(util)::BitSet* bs) const{ return true; } + + //Creates a user-readable version of this query and returns it as as string + virtual TCHAR* toString()=0; + }; +CL_NS_END +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/search/FilteredTermEnum.cpp b/src/assistant/3rdparty/clucene/src/CLucene/search/FilteredTermEnum.cpp new file mode 100644 index 000000000..f90ceeaaf --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/search/FilteredTermEnum.cpp @@ -0,0 +1,136 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#include "CLucene/StdHeader.h" + +#include "FilteredTermEnum.h" + +CL_NS_USE(index) +CL_NS_DEF(search) + + + FilteredTermEnum::FilteredTermEnum(){ + //Func - Constructor + //Pre - true + //Post - Instance has been created + + currentTerm = NULL; + actualEnum = NULL; + } + + FilteredTermEnum::~FilteredTermEnum() { + //Func - Destructor + //Pre - true + //Post - The instance has been destroyed + + close(); + } + + int32_t FilteredTermEnum::docFreq() const { + //Func - Returns the docFreq of the current Term in the enumeration. + //Pre - next() must have been called at least once + //Post - if actualEnum is NULL result is -1 otherwise the frequencey is returned + + if (actualEnum == NULL){ + return -1; + } + return actualEnum->docFreq(); + } + + bool FilteredTermEnum::next() { + //Func - Increments the enumeration to the next element. + //Pre - true + //Post - Returns True if the enumeration has been moved to the next element otherwise false + + //The actual enumerator is not initialized! + if (actualEnum == NULL){ + return false; + } + + //Finalize the currentTerm and reset it to NULL + _CLDECDELETE( currentTerm ); + + //Iterate through the enumeration + while (currentTerm == NULL) { + if (endEnum()) + return false; + if (actualEnum->next()) { + //Order term not to return reference ownership here. */ + Term* term = actualEnum->term(false); + //Compare the retrieved term + if (termCompare(term)){ + //Matched so finalize the current + _CLDECDELETE(currentTerm); + //Get a reference to the matched term + currentTerm = _CL_POINTER(term); + return true; + } + }else + return false; + } + _CLDECDELETE(currentTerm); + currentTerm = NULL; + + return false; + } + + Term* FilteredTermEnum::term() { + //Func - Returns the current Term in the enumeration. + //Pre - next() must have been called at least once + // pointer is true or false + //Post - if pre(pointer) is true the reference counter of currentTerm is increased + // and current Term is returned otherwise currentTerm is only returned + + return _CL_POINTER(currentTerm); + } + Term* FilteredTermEnum::term(bool pointer) { + if ( pointer ) + return _CL_POINTER(currentTerm); + else + return currentTerm; + } + + void FilteredTermEnum::close(){ + //Func - Closes the enumeration to further activity, freeing resources. + //Pre - true + //Post - The Enumeration has been closed + + //Check if actualEnum is valid + if (actualEnum){ + //Close the enumeration + actualEnum->close(); + } + + //Destroy the enumeration + _CLDELETE(actualEnum); + + //Destroy currentTerm + _CLDECDELETE(currentTerm); + } + + void FilteredTermEnum::setEnum(TermEnum* actualEnum) { + //Func - Sets the actual Enumeration + //Pre - actualEnum != NULL + //Post - The instance has been created + + CND_PRECONDITION(actualEnum != NULL,"actualEnum is NULL"); + + _CLDELETE(this->actualEnum); + + this->actualEnum = actualEnum; + + // Find the first term that matches + //Ordered term not to return reference ownership here. + Term* term = actualEnum->term(false); + if (term != NULL && termCompare(term)){ + _CLDECDELETE(currentTerm); + currentTerm = _CL_POINTER(term); + }else{ + next(); + } + } + +CL_NS_END diff --git a/src/assistant/3rdparty/clucene/src/CLucene/search/FilteredTermEnum.h b/src/assistant/3rdparty/clucene/src/CLucene/search/FilteredTermEnum.h new file mode 100644 index 000000000..035ae384e --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/search/FilteredTermEnum.h @@ -0,0 +1,61 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _lucene_search_FilteredTermEnum_ +#define _lucene_search_FilteredTermEnum_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include "CLucene/index/Term.h" +#include "CLucene/index/Terms.h" + +CL_NS_DEF(search) + //FilteredTermEnum is an abstract class for enumerating a subset of all terms. + // + //Term enumerations are always ordered by term->compareTo(). Each term in + //the enumeration is greater than all that precede it. + + class FilteredTermEnum: public CL_NS(index)::TermEnum { + public: + //Constructor + FilteredTermEnum(); + //Destructor + virtual ~FilteredTermEnum(); + + //Equality measure on the term + virtual qreal difference() = 0; + + //Returns the docFreq of the current Term in the enumeration. + int32_t docFreq() const ; + + //Increments the enumeration to the next element + bool next() ; + + //Returns a pointer to the current Term in the enumeration. + CL_NS(index)::Term* term(); + CL_NS(index)::Term* term(bool pointer); + + //Closes the enumeration to further activity, freeing resources. + void close(); + + protected: + //Equality compare on the term */ + virtual bool termCompare(CL_NS(index)::Term* term) = 0; + + //Indiciates the end of the enumeration has been reached + virtual bool endEnum() = 0; + + void setEnum(CL_NS(index)::TermEnum* actualEnum) ; + + private: + CL_NS(index)::Term* currentTerm; + CL_NS(index)::TermEnum* actualEnum; + + }; +CL_NS_END +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/search/FuzzyQuery.cpp b/src/assistant/3rdparty/clucene/src/CLucene/search/FuzzyQuery.cpp new file mode 100644 index 000000000..e95d48da3 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/search/FuzzyQuery.cpp @@ -0,0 +1,357 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#include "CLucene/StdHeader.h" +#include "FuzzyQuery.h" + +CL_NS_USE(index) +CL_NS_USE(util) +CL_NS_DEF(search) + + /** + * Constructor for enumeration of all terms from specified <code>reader</code> which share a prefix of + * length <code>prefixLength</code> with <code>term</code> and which have a fuzzy similarity > + * <code>minSimilarity</code>. + * + * @param reader Delivers terms. + * @param term Pattern term. + * @param minSimilarity Minimum required similarity for terms from the reader. Default value is 0.5f. + * @param prefixLength Length of required common prefix. Default value is 0. + * @throws IOException + */ + FuzzyTermEnum::FuzzyTermEnum(const IndexReader* reader, Term* term, qreal minSimilarity, size_t prefixLength): + distance(0), + _endEnum(false), + prefix(LUCENE_BLANK_STRING), + prefixLength(0), + minimumSimilarity(minSimilarity) + { + //Func - Constructor + //Pre - reader contains a valid reference to an IndexReader + // term != NULL + //Post - The instance has been created + + CND_PRECONDITION(term != NULL,"term is NULL"); + + scale_factor = 1.0f / (1.0f - minimumSimilarity); + searchTerm = _CL_POINTER(term); + + text = STRDUP_TtoT(term->text()); + textLen = term->textLength(); + + + //Initialize e to NULL + e = NULL; + eWidth = 0; + eHeight = 0; + + if(prefixLength > 0 && prefixLength < textLen){ + this->prefixLength = prefixLength; + + prefix = _CL_NEWARRAY(TCHAR,prefixLength+1); + _tcsncpy(prefix,text,prefixLength); + prefix[prefixLength]='\0'; + + textLen = prefixLength; + text[textLen]='\0'; + } + + + //Set the enumeration + Term* trm = _CLNEW Term(term, prefix); + setEnum(reader->terms(trm)); + _CLDECDELETE(trm); + } + + FuzzyTermEnum::~FuzzyTermEnum(){ + //Func - Destructor + //Pre - true + //Post - FuzzyTermEnum has been destroyed + + //Close the enumeration + close(); + } + + bool FuzzyTermEnum::endEnum() { + //Func - Returns the fact if the current term in the enumeration has reached the end + //Pre - true + //Post - The boolean value of endEnum has been returned + + return _endEnum; + } + + void FuzzyTermEnum::close(){ + //Func - Close the enumeration + //Pre - true + //Post - The enumeration has been closed + + FilteredTermEnum::close(); + + //Finalize the searchTerm + _CLDECDELETE(searchTerm); + //Destroy e + _CLDELETE_ARRAY(e); + + _CLDELETE_CARRAY(text); + + if ( prefix != LUCENE_BLANK_STRING ) + _CLDELETE_CARRAY(prefix); + } + + bool FuzzyTermEnum::termCompare(Term* term) { + //Func - Compares term with the searchTerm using the Levenshtein distance. + //Pre - term is NULL or term points to a Term + //Post - if pre(term) is NULL then false is returned otherwise + // if the distance of the current term in the enumeration is bigger than the FUZZY_THRESHOLD + // then true is returned + + if (term == NULL){ + return false; //Note that endEnum is not set to true! + } + + const TCHAR* termText = term->text(); + size_t termTextLen = term->textLength(); + + //Check if the field name of searchTerm of term match + //(we can use == because fields are interned) + if ( searchTerm->field() == term->field() && + (prefixLength==0 || _tcsncmp(termText,prefix,prefixLength)==0 )) { + + const TCHAR* target = termText+prefixLength; + size_t targetLen = termTextLen-prefixLength; + + //Calculate the Levenshtein distance + int32_t dist = editDistance(text, target, textLen, targetLen); + distance = 1 - ((qreal)dist / (qreal)min(textLen, targetLen)); + return (distance > minimumSimilarity); + } + _endEnum = true; + return false; + } + + qreal FuzzyTermEnum::difference() { + //Func - Returns the difference between the distance and the fuzzy threshold + // multiplied by the scale factor + //Pre - true + //Post - The difference is returned + + return (qreal)((distance - minimumSimilarity) * scale_factor ); + } + + + /** Finds and returns the smallest of three integers + precondition: Must define int32_t __t for temporary storage and result + */ + #define min3(a, b, c) __t = (a < b) ? a : b; __t = (__t < c) ? __t : c; + + int32_t FuzzyTermEnum::editDistance(const TCHAR* s, const TCHAR* t, const int32_t n, const int32_t m) { + //Func - Calculates the Levenshtein distance also known as edit distance is a measure of similiarity + // between two strings where the distance is measured as the number of character + // deletions, insertions or substitutions required to transform one string to + // the other string. + //Pre - s != NULL and contains the source string + // t != NULL and contains the target string + // n >= 0 and contains the length of the source string + // m >= 0 and containts the length of th target string + //Post - The distance has been returned + + CND_PRECONDITION(s != NULL, "s is NULL"); + CND_PRECONDITION(t != NULL, "t is NULL"); + CND_PRECONDITION(n >= 0," n is a negative number"); + CND_PRECONDITION(n >= 0," n is a negative number"); + + int32_t i; // iterates through s + int32_t j; // iterates through t + TCHAR s_i; // ith character of s + + if (n == 0) + return m; + if (m == 0) + return n; + + //Check if the array must be reallocated because it is too small or does not exist + if (e == NULL || eWidth <= n || eHeight <= m) { + //Delete e if possible + _CLDELETE_ARRAY(e); + //resize e + eWidth = max(eWidth, n+1); + eHeight = max(eHeight, m+1); + e = _CL_NEWARRAY(int32_t,eWidth*eHeight); + } + + CND_CONDITION(e != NULL,"e is NULL"); + + // init matrix e + for (i = 0; i <= n; i++){ + e[i + (0*eWidth)] = i; + } + for (j = 0; j <= m; j++){ + e[0 + (j*eWidth)] = j; + } + + int32_t __t; //temporary variable for min3 + + // start computing edit distance + for (i = 1; i <= n; i++) { + s_i = s[i - 1]; + for (j = 1; j <= m; j++) { + if (s_i != t[j-1]){ + min3(e[i + (j*eWidth) - 1], e[i + ((j-1)*eWidth)], e[i + ((j-1)*eWidth)-1]); + e[i + (j*eWidth)] = __t+1; + }else{ + min3(e[i + (j*eWidth) -1]+1, e[i + ((j-1)*eWidth)]+1, e[i + ((j-1)*eWidth)-1]); + e[i + (j*eWidth)] = __t; + } + } + } + + // we got the result! + return e[n + ((m)*eWidth)]; + } + + + /** + * Create a new FuzzyQuery that will match terms with a similarity + * of at least <code>minimumSimilarity</code> to <code>term</code>. + * If a <code>prefixLength</code> > 0 is specified, a common prefix + * of that length is also required. + * + * @param term the term to search for + * @param minimumSimilarity a value between 0 and 1 to set the required similarity + * between the query term and the matching terms. For example, for a + * <code>minimumSimilarity</code> of <code>0.5</code> a term of the same length + * as the query term is considered similar to the query term if the edit distance + * between both terms is less than <code>length(term)*0.5</code> + * @param prefixLength length of common (non-fuzzy) prefix + * @throws IllegalArgumentException if minimumSimilarity is > 1 or < 0 + * or if prefixLength < 0 or > <code>term.text().length()</code>. + */ + FuzzyQuery::FuzzyQuery(Term* term, qreal minimumSimilarity, size_t prefixLength): + MultiTermQuery(term) + { + //Func - Constructor + //Pre - term != NULL + //Post - The instance has been created + + CND_PRECONDITION(term != NULL,"term is NULL"); + + if (minimumSimilarity > 1.0f) + _CLTHROWA(CL_ERR_IllegalArgument,"minimumSimilarity > 1"); + else if (minimumSimilarity < 0.0f) + _CLTHROWA(CL_ERR_IllegalArgument,"minimumSimilarity < 0"); + + this->minimumSimilarity = minimumSimilarity; + + if(prefixLength >= term->textLength()) + _CLTHROWA(CL_ERR_IllegalArgument,"prefixLength >= term.textLength()"); + this->prefixLength = prefixLength; + + } + + + qreal FuzzyQuery::defaultMinSimilarity = 0.5f; + + FuzzyQuery::~FuzzyQuery(){ + //Func - Destructor + //Pre - true + //Post - Instance has been destroyed + } + + TCHAR* FuzzyQuery::toString(const TCHAR* field) const{ + //Func - Returns the query string + //Pre - field != NULL + //Post - The query string has been returned + + CND_PRECONDITION(field != NULL,"field is NULL"); + + StringBuffer buffer; + const TCHAR* b = MultiTermQuery::toString(field); + + buffer.append ( b ); + _CLDELETE_CARRAY(b); + buffer.append( _T("~") ); + + buffer.appendFloat(minimumSimilarity,1); + + return buffer.toString(); + } + + const TCHAR* FuzzyQuery::getQueryName() const{ + //Func - Returns the name of the query + //Pre - true + //post - The string FuzzyQuery has been returned + + return getClassName(); + } + const TCHAR* FuzzyQuery::getClassName(){ + //Func - Returns the name of the query + //Pre - true + //post - The string FuzzyQuery has been returned + + return _T("FuzzyQuery"); + } + + + /** + * Returns the minimum similarity that is required for this query to match. + * @return float value between 0.0 and 1.0 + */ + qreal FuzzyQuery::getMinSimilarity() const { + return minimumSimilarity; + } + + FuzzyQuery::FuzzyQuery(const FuzzyQuery& clone): + MultiTermQuery(clone) + { + this->minimumSimilarity = clone.getMinSimilarity(); + this->prefixLength = clone.getPrefixLength(); + + //if(prefixLength < 0) + // _CLTHROWA(CL_ERR_IllegalArgument,"prefixLength < 0"); + //else + if(prefixLength >= clone.getTerm()->textLength()) + _CLTHROWA(CL_ERR_IllegalArgument,"prefixLength >= term.textLength()"); + + } + + Query* FuzzyQuery::clone() const{ + return _CLNEW FuzzyQuery(*this); + } + size_t FuzzyQuery::hashCode() const{ + //todo: we should give the query a seeding value... but + //need to do it for all hascode functions + size_t val = Similarity::floatToByte(getBoost()) ^ getTerm()->hashCode(); + val ^= Similarity::floatToByte(this->getMinSimilarity()); + val ^= this->getPrefixLength(); + return val; + } + bool FuzzyQuery::equals(Query* other) const{ + if (!(other->instanceOf(FuzzyQuery::getClassName()))) + return false; + + FuzzyQuery* fq = (FuzzyQuery*)other; + return (this->getBoost() == fq->getBoost()) + && this->getMinSimilarity() == fq->getMinSimilarity() + && this->getPrefixLength() == fq->getPrefixLength() + && getTerm()->equals(fq->getTerm()); + } + + /** + * Returns the prefix length, i.e. the number of characters at the start + * of a term that must be identical (not fuzzy) to the query term if the query + * is to match that term. + */ + size_t FuzzyQuery::getPrefixLength() const { + return prefixLength; + } + + FilteredTermEnum* FuzzyQuery::getEnum(IndexReader* reader){ + Term* term = getTerm(false); + FuzzyTermEnum* ret = _CLNEW FuzzyTermEnum(reader, term, minimumSimilarity, prefixLength); + return ret; + } + +CL_NS_END diff --git a/src/assistant/3rdparty/clucene/src/CLucene/search/FuzzyQuery.h b/src/assistant/3rdparty/clucene/src/CLucene/search/FuzzyQuery.h new file mode 100644 index 000000000..e58637bb9 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/search/FuzzyQuery.h @@ -0,0 +1,156 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _lucene_search_FuzzyQuery_ +#define _lucene_search_FuzzyQuery_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include "CLucene/index/IndexReader.h" +#include "CLucene/index/Term.h" +#include "MultiTermQuery.h" + + +CL_NS_DEF(search) + + // class FuzzyQuery implements the fuzzy search query + class FuzzyQuery: public MultiTermQuery { + private: + qreal minimumSimilarity; + size_t prefixLength; + protected: + FuzzyQuery(const FuzzyQuery& clone); + public: + static qreal defaultMinSimilarity; + + /** + * Create a new FuzzyQuery that will match terms with a similarity + * of at least <code>minimumSimilarity</code> to <code>term</code>. + * If a <code>prefixLength</code> > 0 is specified, a common prefix + * of that length is also required. + * + * @param term the term to search for + * @param minimumSimilarity a value between 0 and 1 to set the required similarity + * between the query term and the matching terms. For example, for a + * <code>minimumSimilarity</code> of <code>0.5</code> a term of the same length + * as the query term is considered similar to the query term if the edit distance + * between both terms is less than <code>length(term)*0.5</code> + * @param prefixLength length of common (non-fuzzy) prefix + * @throws IllegalArgumentException if minimumSimilarity is > 1 or < 0 + * or if prefixLength < 0 or > <code>term.text().length()</code>. + */ + FuzzyQuery(CL_NS(index)::Term* term, qreal minimumSimilarity=defaultMinSimilarity, size_t prefixLength=0); + //Destructor + ~FuzzyQuery(); + + TCHAR* toString(const TCHAR* field) const; + + //Returns the name "FuzzyQuery" + static const TCHAR* getClassName(); + const TCHAR* getQueryName() const; + + Query* clone() const; + bool equals(Query * other) const; + size_t hashCode() const; + + /** + * Returns the minimum similarity that is required for this query to match. + * @return float value between 0.0 and 1.0 + */ + qreal getMinSimilarity() const; + + /** + * Returns the prefix length, i.e. the number of characters at the start + * of a term that must be identical (not fuzzy) to the query term if the query + * is to match that term. + */ + size_t getPrefixLength() const; + + protected: + FilteredTermEnum* getEnum(CL_NS(index)::IndexReader* reader); + }; + + /** FuzzyTermEnum is a subclass of FilteredTermEnum for enumerating all + * terms that are similiar to the specified filter term. + * + * Term enumerations are always ordered by Term.compareTo(). Each term in + * the enumeration is greater than all that precede it. + */ + class FuzzyTermEnum: public FilteredTermEnum { + private: + qreal distance; + bool _endEnum; + + CL_NS(index)::Term* searchTerm; + TCHAR* text; + size_t textLen; + TCHAR* prefix; + size_t prefixLength; + qreal minimumSimilarity; + double scale_factor; + + + /** + * This static array saves us from the time required to create a new array + * everytime editDistance is called. + */ + int32_t* e; + int32_t eWidth; + int32_t eHeight; + + /****************************** + * Compute Levenshtein distance + ******************************/ + + /** + Levenshtein distance also known as edit distance is a measure of similiarity + between two strings where the distance is measured as the number of character + deletions, insertions or substitutions required to transform one string to + the other string. + <p>This method takes in four parameters; two strings and their respective + lengths to compute the Levenshtein distance between the two strings. + The result is returned as an integer. + */ + int32_t editDistance(const TCHAR* s, const TCHAR* t, const int32_t n, const int32_t m) ; + + protected: + /** + The termCompare method in FuzzyTermEnum uses Levenshtein distance to + calculate the distance between the given term and the comparing term. + */ + bool termCompare(CL_NS(index)::Term* term) ; + + ///Returns the fact if the current term in the enumeration has reached the end + bool endEnum(); + public: + + /** + * Empty prefix and minSimilarity of 0.5f are used. + * + * @param reader + * @param term + * @throws IOException + * @see #FuzzyTermEnum(IndexReader, Term, qreal, int32_t) + */ + FuzzyTermEnum(const CL_NS(index)::IndexReader* reader, CL_NS(index)::Term* term, qreal minSimilarity=FuzzyQuery::defaultMinSimilarity, size_t prefixLength=0); + /** Destructor */ + ~FuzzyTermEnum(); + /** Close the enumeration */ + void close(); + + /** Returns the difference between the distance and the fuzzy threshold + * multiplied by the scale factor + */ + qreal difference(); + + + const char* getObjectName(){ return FuzzyTermEnum::getClassName(); } + static const char* getClassName(){ return "FuzzyTermEnum"; } + }; +CL_NS_END +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/search/HitQueue.cpp b/src/assistant/3rdparty/clucene/src/CLucene/search/HitQueue.cpp new file mode 100644 index 000000000..c9aecc6b4 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/search/HitQueue.cpp @@ -0,0 +1,107 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#include "CLucene/StdHeader.h" +#include "HitQueue.h" + +CL_NS_DEF(search) + +void HitQueue::upHeap(){ + size_t i = _size; + ScoreDoc node = heap[i]; // save bottom node (WAS object) + int32_t j = ((uint32_t)i) >> 1; + while (j > 0 && lessThan(node,heap[j])) { + heap[i] = heap[j]; // shift parents down + i = j; + j = ((uint32_t)j) >> 1; + } + heap[i] = node; // install saved node +} +void HitQueue::downHeap(){ + size_t i = 1; + ScoreDoc node = heap[i]; // save top node + size_t j = i << 1; // find smaller child + size_t k = j + 1; + if (k <= _size && lessThan(heap[k], heap[j])) { + j = k; + } + while (j <= _size && lessThan(heap[j],node)) { + heap[i] = heap[j]; // shift up child + i = j; + j = i << 1; + k = j + 1; + if (k <= _size && lessThan(heap[k], heap[j])) { + j = k; + } + } + heap[i] = node; // install saved node +} + +void HitQueue::adjustTop(){ + downHeap(); +} +size_t HitQueue::size(){ + return _size; +} + +struct ScoreDoc& HitQueue::top(){ + if ( _size == 0 ) + _CLTHROWA(CL_ERR_IndexOutOfBounds, "Attempted to access empty hitqueue::top"); + return heap[1]; +} + +void HitQueue::put(struct ScoreDoc& element){ + if ( _size>=maxSize ) + _CLTHROWA(CL_ERR_IndexOutOfBounds,"add is out of bounds"); + + _size++; + heap[_size] = element; + upHeap(); +} + +ScoreDoc HitQueue::pop(){ + if (_size > 0) { + ScoreDoc result = heap[1]; // save first value + heap[1] = heap[_size]; // move last to first + + _size--; + downHeap(); // adjust heap + return result; + } else + _CLTHROWA(CL_ERR_IndexOutOfBounds, "Attempted to access empty hitqueue::top"); +} + +bool HitQueue::insert(struct ScoreDoc& element){ + if(_size < maxSize){ + put(element); + return true; + }else if(_size > 0 && !lessThan(element, heap[1])){ + heap[1] = element; + adjustTop(); + return true; + }else + return false; +} + +HitQueue::HitQueue(const int32_t maxSize){ + _size = 0; + this->maxSize = maxSize; + int32_t heapSize = maxSize + 1; + heap = _CL_NEWARRAY(ScoreDoc,heapSize); +} +HitQueue::~HitQueue(){ + _CLDELETE_ARRAY(heap); +} + +bool HitQueue::lessThan(struct ScoreDoc& hitA, struct ScoreDoc& hitB){ + if (hitA.score == hitB.score) + return hitA.doc > hitB.doc; + else + return hitA.score < hitB.score; +} + + +CL_NS_END diff --git a/src/assistant/3rdparty/clucene/src/CLucene/search/HitQueue.h b/src/assistant/3rdparty/clucene/src/CLucene/search/HitQueue.h new file mode 100644 index 000000000..0bd196a7f --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/search/HitQueue.h @@ -0,0 +1,55 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _lucene_search_HitQueue_ +#define _lucene_search_HitQueue_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include "SearchHeader.h" + +CL_NS_DEF(search) + +/** +* An optimised PriorityQueue which takes ScoreDoc structs. Some by-ref passing +* and memory related optimisations have been done. +*/ +class HitQueue: LUCENE_BASE { +private: + ScoreDoc* heap; + size_t _size; + size_t maxSize; + + void upHeap(); + void downHeap(); + +protected: + bool lessThan(struct ScoreDoc& hitA, struct ScoreDoc& hitB); + +public: + void adjustTop(); + struct ScoreDoc& top(); + void put(struct ScoreDoc& element); + ScoreDoc pop(); + /** + * Adds element to the PriorityQueue in log(size) time if either + * the PriorityQueue is not full, or not lessThan(element, top()). + * @param element + * @return true if element is added, false otherwise. + */ + bool insert(struct ScoreDoc& element); + /** + * Returns the number of elements currently stored in the PriorityQueue. + */ + size_t size(); + HitQueue(const int32_t maxSize); + ~HitQueue(); + +}; +CL_NS_END +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/search/Hits.cpp b/src/assistant/3rdparty/clucene/src/CLucene/search/Hits.cpp new file mode 100644 index 000000000..38c489f44 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/search/Hits.cpp @@ -0,0 +1,174 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#include "CLucene/StdHeader.h" + +#include "SearchHeader.h" +#include "CLucene/document/Document.h" +#include "CLucene/index/IndexReader.h" +#include "Filter.h" +#include "CLucene/search/SearchHeader.h" + +CL_NS_USE(document) +CL_NS_USE(util) +CL_NS_USE(index) + +CL_NS_DEF(search) + + HitDoc::HitDoc(const qreal s, const int32_t i) + { + //Func - Constructor + //Pre - true + //Post - The instance has been created + + next = NULL; + prev = NULL; + doc = NULL; + score = s; + id = i; + } + + HitDoc::~HitDoc(){ + //Func - Destructor + //Pre - true + //Post - The instance has been destroyed + + _CLDELETE(doc); + } + + + Hits::Hits(Searcher* s, Query* q, Filter* f, const Sort* _sort): + query(q), searcher(s), filter(f), sort(_sort) + { + //Func - Constructor + //Pre - s contains a valid reference to a searcher s + // q contains a valid reference to a Query + // f is NULL or contains a pointer to a filter + //Post - The instance has been created + + _length = 0; + first = NULL; + last = NULL; + numDocs = 0; + maxDocs = 200; + + //retrieve 100 initially + getMoreDocs(50); + } + + Hits::~Hits(){ + + } + int32_t Hits::length() const { + return _length; + } + + Document& Hits::doc(const int32_t n){ + HitDoc* hitDoc = getHitDoc(n); + + // Update LRU cache of documents + remove(hitDoc); // remove from list, if there + addToFront(hitDoc); // add to front of list + if (numDocs > maxDocs) { // if cache is full + HitDoc* oldLast = last; + remove(last); // flush last + + _CLDELETE( oldLast->doc ); + oldLast->doc = NULL; + } + + if (hitDoc->doc == NULL){ + hitDoc->doc = _CLNEW Document; + searcher->doc(hitDoc->id, hitDoc->doc); // cache miss: read document + } + + return *hitDoc->doc; + } + + int32_t Hits::id (const int32_t n){ + return getHitDoc(n)->id; + } + + qreal Hits::score(const int32_t n){ + return getHitDoc(n)->score; + } + + void Hits::getMoreDocs(const size_t m){ + size_t _min = m; + { + size_t nHits = hitDocs.size(); + if ( nHits > _min) + _min = nHits; + } + + size_t n = _min * 2; // double # retrieved + TopDocs* topDocs = NULL; + if ( sort==NULL ) + topDocs = (TopDocs*)((Searchable*)searcher)->_search(query, filter, n); + else + topDocs = (TopDocs*)((Searchable*)searcher)->_search(query, filter, n, sort); + _length = topDocs->totalHits; + ScoreDoc* scoreDocs = topDocs->scoreDocs; + int32_t scoreDocsLength = topDocs->scoreDocsLength; + + qreal scoreNorm = 1.0f; + //Check that scoreDocs is a valid pointer before using it + if (scoreDocs != NULL){ + if (_length > 0 && scoreDocs[0].score > 1.0f){ + scoreNorm = 1.0f / scoreDocs[0].score; + } + + int32_t end = scoreDocsLength < _length ? scoreDocsLength : _length; + for (int32_t i = hitDocs.size(); i < end; i++) { + hitDocs.push_back(_CLNEW HitDoc(scoreDocs[i].score*scoreNorm, scoreDocs[i].doc)); + } + } + + _CLDELETE(topDocs); + } + + HitDoc* Hits::getHitDoc(const size_t n){ + if (n >= _length){ + TCHAR buf[100]; + _sntprintf(buf, 100,_T("Not a valid hit number: %d"),n); + _CLTHROWT(CL_ERR_IndexOutOfBounds, buf ); + } + if (n >= hitDocs.size()) + getMoreDocs(n); + + return hitDocs[n]; + } + + void Hits::addToFront(HitDoc* hitDoc) { // insert at front of cache + if (first == NULL) + last = hitDoc; + else + first->prev = hitDoc; + + hitDoc->next = first; + first = hitDoc; + hitDoc->prev = NULL; + + numDocs++; + } + + void Hits::remove(const HitDoc* hitDoc) { // remove from cache + if (hitDoc->doc == NULL) // it's not in the list + return; // abort + + if (hitDoc->next == NULL) + last = hitDoc->prev; + else + hitDoc->next->prev = hitDoc->prev; + + if (hitDoc->prev == NULL) + first = hitDoc->next; + else + hitDoc->prev->next = hitDoc->next; + + numDocs--; + } +CL_NS_END diff --git a/src/assistant/3rdparty/clucene/src/CLucene/search/IndexSearcher.cpp b/src/assistant/3rdparty/clucene/src/CLucene/search/IndexSearcher.cpp new file mode 100644 index 000000000..c948cfa4b --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/search/IndexSearcher.cpp @@ -0,0 +1,362 @@ +/* + * Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team + * + * Distributable under the terms of either the Apache License (Version 2.0) or + * the GNU Lesser General Public License, as specified in the COPYING file. + * + * Changes are Copyright(C) 2007, 2008 by Nokia Corporation and/or its subsidiary(-ies), all rights reserved. +*/ +#include "CLucene/StdHeader.h" +#include "IndexSearcher.h" + +#include "SearchHeader.h" +#include "Scorer.h" +#include "FieldDocSortedHitQueue.h" +#include "CLucene/store/Directory.h" +#include "CLucene/document/Document.h" +#include "CLucene/index/IndexReader.h" +#include "CLucene/index/Term.h" +#include "CLucene/util/BitSet.h" +#include "FieldSortedHitQueue.h" + +CL_NS_USE(index) +CL_NS_USE(util) +CL_NS_USE(document) + +CL_NS_DEF(search) + +class SimpleTopDocsCollector : public HitCollector +{ +private: + qreal minScore; + const CL_NS(util)::BitSet* bits; + HitQueue* hq; + size_t nDocs; + int32_t* totalHits; + +public: + SimpleTopDocsCollector(const CL_NS(util)::BitSet* bs, HitQueue* hitQueue, + int32_t* totalhits, size_t ndocs, const qreal ms=-1.0f) + : minScore(ms), + bits(bs), + hq(hitQueue), + nDocs(ndocs), + totalHits(totalhits) {} + ~SimpleTopDocsCollector() {} + + void collect(const int32_t doc, const qreal score) + { + if (score > 0.0f // ignore zeroed buckets + && (bits == NULL || bits->get(doc))) { // skip docs not in bits + ++totalHits[0]; + if (hq->size() < nDocs || (minScore==-1.0f || score >= minScore)) { + ScoreDoc sd = {doc, score}; + hq->insert(sd); // update hit queue + if ( minScore != -1.0f ) + minScore = hq->top().score; // maintain minScore + } + } + } +}; + +class SortedTopDocsCollector : public HitCollector +{ +private: + const CL_NS(util)::BitSet* bits; + FieldSortedHitQueue* hq; + size_t nDocs; + int32_t* totalHits; +public: + SortedTopDocsCollector(const CL_NS(util)::BitSet* bs, + FieldSortedHitQueue* hitQueue, int32_t* totalhits, size_t _nDocs) + : bits(bs), + hq(hitQueue), + nDocs(_nDocs), + totalHits(totalhits) + { + } + ~SortedTopDocsCollector() {} + + void collect(const int32_t doc, const qreal score) + { + if (score > 0.0f && // ignore zeroed buckets + (bits==NULL || bits->get(doc))) { // skip docs not in bits + ++totalHits[0]; + // TODO: see jlucene way... with fields def??? + FieldDoc* fd = _CLNEW FieldDoc(doc, score); + if ( !hq->insert(fd) ) // update hit queue + _CLDELETE(fd); + } + } +}; + +class SimpleFilteredCollector : public HitCollector +{ +private: + CL_NS(util)::BitSet* bits; + HitCollector* results; +public: + SimpleFilteredCollector(CL_NS(util)::BitSet* bs, HitCollector* collector) + : bits(bs), + results(collector) {} + ~SimpleFilteredCollector() {} + +protected: + void collect(const int32_t doc, const qreal score) + { + // skip docs not in bits + if (bits->get(doc)) + results->collect(doc, score); + } +}; + + +IndexSearcher::IndexSearcher(const QString& path) +{ + //Func - Constructor + // Creates a searcher searching the index in the named directory. + //Pre - path != NULL + //Post - The instance has been created + + CND_PRECONDITION(!path.isEmpty(), "path is NULL"); + + reader = IndexReader::open(path); + readerOwner = true; +} + +IndexSearcher::IndexSearcher(CL_NS(store)::Directory* directory) +{ + //Func - Constructor + // Creates a searcher searching the index in the specified directory. + //Pre - path != NULL + //Post - The instance has been created + + CND_PRECONDITION(directory != NULL, "directory is NULL"); + + reader = IndexReader::open(directory); + readerOwner = true; +} + +IndexSearcher::IndexSearcher(IndexReader* r) +{ + //Func - Constructor + // Creates a searcher searching the index with the provide IndexReader + //Pre - path != NULL + //Post - The instance has been created + + reader = r; + readerOwner = false; +} + +IndexSearcher::~IndexSearcher() +{ + //Func - Destructor + //Pre - true + //Post - The instance has been destroyed + + close(); +} + +void IndexSearcher::close() +{ + //Func - Frees resources associated with this Searcher. + //Pre - true + //Post - The resources associated have been freed + if (readerOwner && reader){ + reader->close(); + _CLDELETE(reader); + } +} + +// inherit javadoc +int32_t IndexSearcher::docFreq(const Term* term) const +{ + //Func - + //Pre - reader != NULL + //Post - + + CND_PRECONDITION(reader != NULL, "reader is NULL"); + return reader->docFreq(term); +} + +// inherit javadoc +bool IndexSearcher::doc(int32_t i, CL_NS(document)::Document* d) +{ + //Func - Retrieves i-th document found + // For use by HitCollector implementations. + //Pre - reader != NULL + //Post - The i-th document has been returned + + CND_PRECONDITION(reader != NULL, "reader is NULL"); + return reader->document(i,d); +} + +// inherit javadoc +int32_t IndexSearcher::maxDoc() const +{ + //Func - Return total number of documents including the ones marked deleted + //Pre - reader != NULL + //Post - The total number of documents including the ones marked deleted + // has been returned + + CND_PRECONDITION(reader != NULL, "reader is NULL"); + return reader->maxDoc(); +} + +TopDocs* IndexSearcher::_search(Query* query, Filter* filter, const int32_t nDocs) +{ + //Func - + //Pre - reader != NULL + //Post - + + CND_PRECONDITION(reader != NULL, "reader is NULL"); + CND_PRECONDITION(query != NULL, "query is NULL"); + + Weight* weight = query->weight(this); + Scorer* scorer = weight->scorer(reader); + if (scorer == NULL){ + return _CLNEW TopDocs(0, NULL, 0); + } + + BitSet* bits = filter != NULL ? filter->bits(reader) : NULL; + HitQueue* hq = _CLNEW HitQueue(nDocs); + + //Check hq has been allocated properly + CND_CONDITION(hq != NULL, "Could not allocate memory for HitQueue hq"); + + int32_t* totalHits = _CL_NEWARRAY(int32_t,1); + totalHits[0] = 0; + + SimpleTopDocsCollector hitCol(bits,hq,totalHits,nDocs,0.0f); + scorer->score( &hitCol ); + _CLDELETE(scorer); + + int32_t scoreDocsLength = hq->size(); + + ScoreDoc* scoreDocs = _CL_NEWARRAY(ScoreDoc,scoreDocsLength); + + for (int32_t i = scoreDocsLength-1; i >= 0; --i) // put docs in array + scoreDocs[i] = hq->pop(); + + int32_t totalHitsInt = totalHits[0]; + + _CLDELETE(hq); + if ( bits != NULL && filter->shouldDeleteBitSet(bits) ) + _CLDELETE(bits); + _CLDELETE_ARRAY(totalHits); + Query* wq = weight->getQuery(); + if ( query != wq ) //query was re-written + _CLLDELETE(wq); + _CLDELETE(weight); + + return _CLNEW TopDocs(totalHitsInt, scoreDocs, scoreDocsLength); +} + +// inherit javadoc +TopFieldDocs* IndexSearcher::_search(Query* query, Filter* filter, + const int32_t nDocs, const Sort* sort) +{ + CND_PRECONDITION(reader != NULL, "reader is NULL"); + CND_PRECONDITION(query != NULL, "query is NULL"); + + Weight* weight = query->weight(this); + Scorer* scorer = weight->scorer(reader); + if (scorer == NULL) { + return _CLNEW TopFieldDocs(0, NULL, 0, NULL ); + } + + BitSet* bits = filter != NULL ? filter->bits(reader) : NULL; + FieldSortedHitQueue hq(reader, sort->getSort(), nDocs); + int32_t* totalHits = _CL_NEWARRAY(int32_t,1); + totalHits[0]=0; + + SortedTopDocsCollector hitCol(bits,&hq,totalHits,nDocs); + scorer->score(&hitCol); + _CLDELETE(scorer); + + int32_t hqLen = hq.size(); + FieldDoc** fieldDocs = _CL_NEWARRAY(FieldDoc*,hqLen); + for (int32_t i = hqLen-1; i >= 0; --i){ // put docs in array + fieldDocs[i] = hq.fillFields (hq.pop()); + } + + Query* wq = weight->getQuery(); + if ( query != wq ) //query was re-written + _CLLDELETE(wq); + _CLDELETE(weight); + + SortField** hqFields = hq.getFields(); + hq.setFields(NULL); //move ownership of memory over to TopFieldDocs + int32_t totalHits0 = totalHits[0]; + if ( bits != NULL && filter->shouldDeleteBitSet(bits) ) + _CLDELETE(bits); + _CLDELETE_ARRAY(totalHits); + return _CLNEW TopFieldDocs(totalHits0, fieldDocs, hqLen, hqFields ); +} + +void IndexSearcher::_search(Query* query, Filter* filter, HitCollector* results) +{ + //Func - _search an index and fetch the results + // Applications should only use this if they need all of the + // matching documents. The high-level search API (search(Query)) + // is usually more efficient, as it skips non-high-scoring hits. + //Pre - query is a valid reference to a query filter may or may not be NULL + // results is a valid reference to a HitCollector and used to store the results + //Post - filter if non-NULL, a bitset used to eliminate some documents + + CND_PRECONDITION(reader != NULL, "reader is NULL"); + CND_PRECONDITION(query != NULL, "query is NULL"); + + BitSet* bits = NULL; + SimpleFilteredCollector* fc = NULL; + + if (filter != NULL){ + bits = filter->bits(reader); + fc = _CLNEW SimpleFilteredCollector(bits, results); + } + + Weight* weight = query->weight(this); + Scorer* scorer = weight->scorer(reader); + if (scorer != NULL) { + if (fc == NULL){ + scorer->score(results); + }else{ + scorer->score((HitCollector*)fc); + } + _CLDELETE(scorer); + } + + _CLDELETE(fc); + _CLDELETE(weight); + if ( bits != NULL && filter->shouldDeleteBitSet(bits) ) + _CLDELETE(bits); +} + +Query* IndexSearcher::rewrite(Query* original) +{ + Query* query = original; + Query* last = original; + for (Query* rewrittenQuery = query->rewrite(reader); + rewrittenQuery != query; + rewrittenQuery = query->rewrite(reader)) { + query = rewrittenQuery; + if ( query != last && last != original) { + _CLDELETE(last); + } + last = query; + } + return query; +} + +void IndexSearcher::explain(Query* query, int32_t doc, Explanation* ret) +{ + Weight* weight = query->weight(this); + weight->explain(reader, doc, ret); + + Query* wq = weight->getQuery(); + if ( query != wq ) //query was re-written + _CLLDELETE(wq); + _CLDELETE(weight); +} + +CL_NS_END diff --git a/src/assistant/3rdparty/clucene/src/CLucene/search/IndexSearcher.h b/src/assistant/3rdparty/clucene/src/CLucene/search/IndexSearcher.h new file mode 100644 index 000000000..307e0266d --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/search/IndexSearcher.h @@ -0,0 +1,73 @@ +/* + * Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team + * + * Distributable under the terms of either the Apache License (Version 2.0) or + * the GNU Lesser General Public License, as specified in the COPYING file. + * + * Changes are Copyright(C) 2007, 2008 by Nokia Corporation and/or its subsidiary(-ies), all rights reserved. +*/ +#ifndef _lucene_search_IndexSearcher_ +#define _lucene_search_IndexSearcher_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include <QtCore/QString> + +#include "SearchHeader.h" +#include "CLucene/store/Directory.h" +#include "CLucene/document/Document.h" +#include "CLucene/index/IndexReader.h" +#include "CLucene/index/Term.h" +#include "CLucene/util/BitSet.h" +#include "HitQueue.h" +#include "FieldSortedHitQueue.h" + +CL_NS_DEF(search) +/** Implements search over a single IndexReader. +* +* <p>Applications usually need only call the inherited {@link search(Query*)} +* or {@link search(Query*,Filter*)} methods. +*/ +class IndexSearcher:public Searcher{ + CL_NS(index)::IndexReader* reader; + bool readerOwner; + +public: + /// Creates a searcher searching the index in the named directory. + IndexSearcher(const QString& path); + + /// Creates a searcher searching the index in the specified directory. + IndexSearcher(CL_NS(store)::Directory* directory); + + /// Creates a searcher searching the provided index. + IndexSearcher(CL_NS(index)::IndexReader* r); + + ~IndexSearcher(); + + /// Frees resources associated with this Searcher. + void close(); + + int32_t docFreq(const CL_NS(index)::Term* term) const; + + bool doc(int32_t i, CL_NS(document)::Document* document); + + int32_t maxDoc() const; + + TopDocs* _search(Query* query, Filter* filter, const int32_t nDocs); + TopFieldDocs* _search(Query* query, Filter* filter, const int32_t nDocs, + const Sort* sort); + + void _search(Query* query, Filter* filter, HitCollector* results); + + CL_NS(index)::IndexReader* getReader() { + return reader; } + + Query* rewrite(Query* original); + void explain(Query* query, int32_t doc, Explanation* ret); +}; + +CL_NS_END + +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/search/MultiSearcher.cpp b/src/assistant/3rdparty/clucene/src/CLucene/search/MultiSearcher.cpp new file mode 100644 index 000000000..bed7f0d61 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/search/MultiSearcher.cpp @@ -0,0 +1,227 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#include "CLucene/StdHeader.h" +#include "MultiSearcher.h" + +#include "SearchHeader.h" +#include "HitQueue.h" +#include "CLucene/document/Document.h" +#include "CLucene/index/Term.h" +#include "FieldDocSortedHitQueue.h" + +CL_NS_USE(index) +CL_NS_USE(util) +CL_NS_USE(document) + +CL_NS_DEF(search) + + /** Creates a searcher which searches <i>searchers</i>. */ + MultiSearcher::MultiSearcher(Searchable** _searchables): + _maxDoc(0) { + searchablesLen = 0; + while ( _searchables[searchablesLen] != NULL ) + ++searchablesLen; + + searchables=_CL_NEWARRAY(Searchable*,searchablesLen+1); + starts = _CL_NEWARRAY(int32_t,searchablesLen + 1); // build starts array + for (int32_t i = 0; i < searchablesLen; ++i) { + searchables[i]=_searchables[i]; + starts[i] = _maxDoc; + _maxDoc += searchables[i]->maxDoc(); // compute maxDocs + } + starts[searchablesLen] = _maxDoc; + } + + MultiSearcher::~MultiSearcher() { + _CLDELETE_ARRAY(searchables); + _CLDELETE_ARRAY(starts); + } + + + // inherit javadoc + void MultiSearcher::close() { + for (int32_t i = 0; i < searchablesLen; ++i){ + searchables[i]->close(); + searchables[i]=NULL; + } + } + + int32_t MultiSearcher::docFreq(const Term* term) const { + int32_t docFreq = 0; + for (int32_t i = 0; i < searchablesLen; ++i) + docFreq += searchables[i]->docFreq(term); + return docFreq; + } + + /** For use by {@link HitCollector} implementations. */ + bool MultiSearcher::doc(int32_t n, Document* d) { + int32_t i = subSearcher(n); // find searcher index + return searchables[i]->doc(n - starts[i], d); // dispatch to searcher + } + + int32_t MultiSearcher::searcherIndex(int32_t n) const{ + return subSearcher(n); + } + + /** Returns index of the searcher for document <code>n</code> in the array + * used to construct this searcher. */ + int32_t MultiSearcher::subSearcher(int32_t n) const{ + // replace w/ call to Arrays.binarySearch in Java 1.2 + int32_t lo = 0; // search starts array + int32_t hi = searchablesLen - 1; // for first element less + // than n, return its index + int32_t mid,midValue; + while (hi >= lo) { + mid = (lo + hi) >> 1; + midValue = starts[mid]; + if (n < midValue) + hi = mid - 1; + else if (n > midValue) + lo = mid + 1; + else{ // found a match + while (mid+1 < searchablesLen && starts[mid+1] == midValue) { + ++mid; // scan to last match + } + return mid; + } + } + return hi; + } + + /** Returns the document number of document <code>n</code> within its + * sub-index. */ + int32_t MultiSearcher::subDoc(int32_t n) const{ + return n - starts[subSearcher(n)]; + } + + int32_t MultiSearcher::maxDoc() const{ + return _maxDoc; + } + + TopDocs* MultiSearcher::_search(Query* query, Filter* filter, const int32_t nDocs) { + HitQueue* hq = _CLNEW HitQueue(nDocs); + int32_t totalHits = 0; + TopDocs* docs; + int32_t j; + ScoreDoc* scoreDocs; + for (int32_t i = 0; i < searchablesLen; i++) { // search each searcher + docs = searchables[i]->_search(query, filter, nDocs); + totalHits += docs->totalHits; // update totalHits + scoreDocs = docs->scoreDocs; + for ( j = 0; j <docs->scoreDocsLength; ++j) { // merge scoreDocs int_to hq + scoreDocs[j].doc += starts[i]; // convert doc + if ( !hq->insert(scoreDocs[j])) + break; // no more scores > minScore + } + + _CLDELETE(docs); + } + + int32_t scoreDocsLen = hq->size(); + scoreDocs = _CL_NEWARRAY(ScoreDoc, scoreDocsLen); + {//MSVC 6 scope fix + for (int32_t i = scoreDocsLen-1; i >= 0; --i) // put docs in array + scoreDocs[i] = hq->pop(); + } + + //cleanup + _CLDELETE(hq); + + return _CLNEW TopDocs(totalHits, scoreDocs, scoreDocsLen); + } + + /** Lower-level search API. + * + * <p>{@link HitCollector#collect(int32_t,qreal)} is called for every non-zero + * scoring document. + * + * <p>Applications should only use this if they need <i>all</i> of the + * matching documents. The high-level search API ({@link + * Searcher#search(Query)}) is usually more efficient, as it skips + * non-high-scoring hits. + * + * @param query to match documents + * @param filter if non-null, a bitset used to eliminate some documents + * @param results to receive hits + */ + void MultiSearcher::_search(Query* query, Filter* filter, HitCollector* results){ + for (int32_t i = 0; i < searchablesLen; ++i) { + /* DSR:CL_BUG: Old implementation leaked and was misconceived. We need + ** to have the original HitCollector ($results) collect *all* hits; + ** the MultiHitCollector instantiated below serves only to adjust + ** (forward by starts[i]) the docNo passed to $results. + ** Old implementation instead created a sort of linked list of + ** MultiHitCollectors that applied the adjustments in $starts + ** cumulatively (and was never deleted). */ + HitCollector *docNoAdjuster = _CLNEW MultiHitCollector(results, starts[i]); + searchables[i]->_search(query, filter, docNoAdjuster); + _CLDELETE(docNoAdjuster); + } + } + + TopFieldDocs* MultiSearcher::_search (Query* query, Filter* filter, const int32_t n, const Sort* sort){ + FieldDocSortedHitQueue* hq = NULL; + int32_t totalHits = 0; + TopFieldDocs* docs; + int32_t j; + FieldDoc** fieldDocs; + + for (int32_t i = 0; i < searchablesLen; ++i) { // search each searcher + docs = searchables[i]->_search (query, filter, n, sort); + if (hq == NULL){ + hq = _CLNEW FieldDocSortedHitQueue (docs->fields, n); + docs->fields = NULL; //hit queue takes fields memory + } + + totalHits += docs->totalHits; // update totalHits + fieldDocs = docs->fieldDocs; + for(j = 0;j<docs->scoreDocsLength;++j){ // merge scoreDocs into hq + fieldDocs[j]->scoreDoc.doc += starts[i]; // convert doc + if (!hq->insert (fieldDocs[j]) ) + break; // no more scores > minScore + } + for ( int32_t x=0;x<j;++x ) + fieldDocs[x]=NULL; //move ownership of FieldDoc to the hitqueue + + _CLDELETE(docs); + } + + int32_t hqlen = hq->size(); + fieldDocs = _CL_NEWARRAY(FieldDoc*,hqlen); + for (j = hqlen - 1; j >= 0; j--) // put docs in array + fieldDocs[j] = hq->pop(); + + SortField** hqFields = hq->getFields(); + hq->setFields(NULL); //move ownership of memory over to TopFieldDocs + _CLDELETE(hq); + + return _CLNEW TopFieldDocs (totalHits, fieldDocs, hqlen, hqFields); + } + + Query* MultiSearcher::rewrite(Query* original) { + Query** queries = _CL_NEWARRAY(Query*,searchablesLen+1); + for (int32_t i = 0; i < searchablesLen; ++i) + queries[i] = searchables[i]->rewrite(original); + queries[searchablesLen]=NULL; + return original->combine(queries); + } + + void MultiSearcher::explain(Query* query, int32_t doc, Explanation* ret) { + int32_t i = subSearcher(doc); // find searcher index + searchables[i]->explain(query,doc-starts[i], ret); // dispatch to searcher + } + + MultiHitCollector::MultiHitCollector(HitCollector* _results, int32_t _start): + results(_results), + start(_start) { + } + + void MultiHitCollector::collect(const int32_t doc, const qreal score) { + results->collect(doc + start, score); + } + +CL_NS_END diff --git a/src/assistant/3rdparty/clucene/src/CLucene/search/MultiSearcher.h b/src/assistant/3rdparty/clucene/src/CLucene/search/MultiSearcher.h new file mode 100644 index 000000000..1021fbbec --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/search/MultiSearcher.h @@ -0,0 +1,95 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _lucene_search_multisearcher +#define _lucene_search_multisearcher + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include "SearchHeader.h" +#include "CLucene/document/Document.h" +#include "CLucene/index/Term.h" + +CL_NS_DEF(search) + + class MultiHitCollector: public HitCollector{ + private: + HitCollector* results; + int32_t start; + public: + MultiHitCollector(HitCollector* _results, int32_t _start); + void collect(const int32_t doc, const qreal score) ; + }; + + + /** Implements search over a set of <code>Searchables</code>. + * + * <p>Applications usually need only call the inherited {@link #search(Query)} + * or {@link #search(Query,Filter)} methods. + */ + class MultiSearcher: public Searcher { + private: + Searchable** searchables; + int32_t searchablesLen; + int32_t* starts; + int32_t _maxDoc; + protected: + int32_t* getStarts() { + return starts; + } + + public: + /** Creates a searcher which searches <i>Searchables</i>. */ + MultiSearcher(Searchable** searchables); + + ~MultiSearcher(); + + /** Frees resources associated with this <code>Searcher</code>. */ + void close() ; + + int32_t docFreq(const CL_NS(index)::Term* term) const ; + + /** For use by {@link HitCollector} implementations. */ + bool doc(int32_t n, CL_NS(document)::Document* document); + + /** For use by {@link HitCollector} implementations to identify the + * index of the sub-searcher that a particular hit came from. */ + int32_t searcherIndex(int32_t n) const; + + int32_t subSearcher(int32_t n) const; + + int32_t subDoc(int32_t n) const; + + int32_t maxDoc() const; + + TopDocs* _search(Query* query, Filter* filter, const int32_t nDocs) ; + + TopFieldDocs* _search (Query* query, Filter* filter, const int32_t n, const Sort* sort); + + /** Lower-level search API. + * + * <p>{@link HitCollector#collect(int32_t,qreal)} is called for every non-zero + * scoring document. + * + * <p>Applications should only use this if they need <i>all</i> of the + * matching documents. The high-level search API ({@link + * Searcher#search(Query)}) is usually more efficient, as it skips + * non-high-scoring hits. + * + * @param query to match documents + * @param filter if non-null, a bitset used to eliminate some documents + * @param results to receive hits + */ + void _search(Query* query, Filter* filter, HitCollector* results); + + Query* rewrite(Query* original); + void explain(Query* query, int32_t doc, Explanation* ret); + }; + +CL_NS_END +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/search/MultiTermQuery.cpp b/src/assistant/3rdparty/clucene/src/CLucene/search/MultiTermQuery.cpp new file mode 100644 index 000000000..3bf8d7a26 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/search/MultiTermQuery.cpp @@ -0,0 +1,98 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#include "CLucene/StdHeader.h" +#include "MultiTermQuery.h" + +CL_NS_USE(index) +CL_NS_USE(util) +CL_NS_DEF(search) + +/** Constructs a query for terms matching <code>term</code>. */ + + MultiTermQuery::MultiTermQuery(Term* t){ + //Func - Constructor + //Pre - t != NULL + //Post - The instance has been created + + CND_PRECONDITION(t != NULL, "t is NULL"); + + term = _CL_POINTER(t); + + } + MultiTermQuery::MultiTermQuery(const MultiTermQuery& clone): + Query(clone) + { + term = _CLNEW Term(clone.getTerm(false),clone.getTerm(false)->text()); + } + + MultiTermQuery::~MultiTermQuery(){ + //Func - Destructor + //Pre - true + //Post - The instance has been destroyed + + _CLDECDELETE(term); + } + + Term* MultiTermQuery::getTerm(bool pointer) const{ + if ( pointer ) + return _CL_POINTER(term); + else + return term; + } + + Query* MultiTermQuery::rewrite(IndexReader* reader) { + FilteredTermEnum* enumerator = getEnum(reader); + BooleanQuery* query = _CLNEW BooleanQuery(); + try { + do { + Term* t = enumerator->term(false); + if (t != NULL) { + TermQuery* tq = _CLNEW TermQuery(t); // found a match + tq->setBoost(getBoost() * enumerator->difference()); // set the boost + query->add(tq,true, false, false); // add to q + } + } while (enumerator->next()); + } _CLFINALLY ( enumerator->close(); _CLDELETE(enumerator) ); + + //if we only added one clause and the clause is not prohibited then + //we can just return the query + if (query->getClauseCount() == 1) { // optimize 1-clause queries + BooleanClause* c=0; + query->getClauses(&c); + + if (!c->prohibited) { // just return clause + c->deleteQuery=false; + Query* ret = c->query; + + _CLDELETE(query); + return ret; + } + } + return query; + } + + Query* MultiTermQuery::combine(Query** queries) { + return Query::mergeBooleanQueries(queries); + } + + /** Prints a user-readable version of this query. */ + TCHAR* MultiTermQuery::toString(const TCHAR* field) const{ + StringBuffer buffer; + + if ( field==NULL || _tcscmp(term->field(),field)!=0 ) { + buffer.append(term->field()); + buffer.append( _T(":")); + } + buffer.append(term->text()); + if (getBoost() != 1.0f) { + buffer.appendChar ( '^' ); + buffer.appendFloat( getBoost(),1); + } + return buffer.toString(); + } + +CL_NS_END diff --git a/src/assistant/3rdparty/clucene/src/CLucene/search/MultiTermQuery.h b/src/assistant/3rdparty/clucene/src/CLucene/search/MultiTermQuery.h new file mode 100644 index 000000000..d37645359 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/search/MultiTermQuery.h @@ -0,0 +1,62 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _lucene_search_MultiTermQuery_ +#define _lucene_search_MultiTermQuery_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include "CLucene/util/StringBuffer.h" +#include "CLucene/index/IndexReader.h" +#include "CLucene/index/Term.h" +#include "CLucene/index/Terms.h" +#include "FilteredTermEnum.h" +#include "SearchHeader.h" +#include "BooleanQuery.h" +#include "TermQuery.h" + +CL_NS_DEF(search) + /** + * A {@link Query} that matches documents containing a subset of terms provided + * by a {@link FilteredTermEnum} enumeration. + * <P> + * <code>MultiTermQuery</code> is not designed to be used by itself. + * <BR> + * The reason being that it is not intialized with a {@link FilteredTermEnum} + * enumeration. A {@link FilteredTermEnum} enumeration needs to be provided. + * <P> + * For example, {@link WildcardQuery} and {@link FuzzyQuery} extend + * <code>MultiTermQuery</code> to provide {@link WildcardTermEnum} and + * {@link FuzzyTermEnum}, respectively. + */ + class MultiTermQuery: public Query { + private: + CL_NS(index)::Term* term; + protected: + MultiTermQuery(const MultiTermQuery& clone); + + /** Construct the enumeration to be used, expanding the pattern term. */ + virtual FilteredTermEnum* getEnum(CL_NS(index)::IndexReader* reader) = 0; + public: + /** Constructs a query for terms matching <code>term</code>. */ + MultiTermQuery(CL_NS(index)::Term* t); + + virtual ~MultiTermQuery(); + + /** Returns the pattern term. */ + CL_NS(index)::Term* getTerm(bool pointer=true) const; + + Query* combine(Query** queries); + + /** Prints a user-readable version of this query. */ + TCHAR* toString(const TCHAR* field) const; + + Query* rewrite(CL_NS(index)::IndexReader* reader); + }; +CL_NS_END +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/search/PhrasePositions.cpp b/src/assistant/3rdparty/clucene/src/CLucene/search/PhrasePositions.cpp new file mode 100644 index 000000000..7611056e7 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/search/PhrasePositions.cpp @@ -0,0 +1,116 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#include "CLucene/StdHeader.h" +#include "PhrasePositions.h" + +#include "CLucene/index/Terms.h" + +CL_NS_USE(index) +CL_NS_DEF(search) + + PhrasePositions::PhrasePositions(TermPositions* Tp, const int32_t OffSet){ + //Func - Constructor + //Pre - t != NULL + // OffSet != NULL + //Post - The instance has been created + + CND_PRECONDITION(Tp != NULL,"Tp is NULL"); + CND_PRECONDITION(OffSet >= 0 ,"OffSet is a negative number"); + + tp = Tp; + offset = OffSet; + position = 0; + count = 0; + doc = 0; + + _next = NULL; + } + + PhrasePositions::~PhrasePositions(){ + //Func - Destructor + //Pre - true + //Post - The instance has been deleted + + //delete next Phrase position and by doing that + //all PhrasePositions in the list + _CLDELETE(_next); + + //Check if tp is valid + if ( tp != NULL ){ + //Close TermPositions tp + tp->close(); + _CLDELETE(tp); + } + } + + bool PhrasePositions::next(){ + //Func - Increments to next doc + //Pre - tp != NULL + //Post - if there was no next then doc = INT_MAX otherwise + // doc contains the current document number + + CND_PRECONDITION(tp != NULL,"tp is NULL"); + + //Move to the next in TermPositions tp + if (!tp->next()) { + //There is no next so close the stream + tp->close(); + //delete tp and reset tp to NULL + _CLVDELETE(tp); //todo: not a clucene object... should be + //Assign Doc sentinel value + doc = INT_MAX; + return false; + }else{ + doc = tp->doc(); + position = 0; + return true; + } + } + bool PhrasePositions::skipTo(int32_t target){ + if (!tp->skipTo(target)) { + tp->close(); // close stream + doc = LUCENE_INT32_MAX_SHOULDBE; // sentinel value + return false; + } + doc = tp->doc(); + position = 0; + return true; + } + void PhrasePositions::firstPosition(){ + //Func - Read the first TermPosition + //Pre - tp != NULL + //Post - + + CND_PRECONDITION(tp != NULL,"tp is NULL"); + + //read first pos + count = tp->freq(); + //Move to the next TermPosition + nextPosition(); + } + + bool PhrasePositions::nextPosition(){ + //Func - Move to the next position + //Pre - tp != NULL + //Post - + + CND_PRECONDITION(tp != NULL,"tp is NULL"); + + if (count-- > 0) { + //read subsequent pos's + position = tp->nextPosition() - offset; + + //Check position always bigger than or equal to 0 + //bvk: todo, bug??? position < 0 occurs, cant figure out why, + //old version does it too and will fail the "SearchTest" test + //CND_CONDITION(position >= 0, "position has become a negative number"); + return true; + }else{ + return false; + } + } +CL_NS_END diff --git a/src/assistant/3rdparty/clucene/src/CLucene/search/PhrasePositions.h b/src/assistant/3rdparty/clucene/src/CLucene/search/PhrasePositions.h new file mode 100644 index 000000000..b6c8437b9 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/search/PhrasePositions.h @@ -0,0 +1,41 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _lucene_search_PhrasePositions_ +#define _lucene_search_PhrasePositions_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include "CLucene/index/Terms.h" + +CL_NS_DEF(search) + + class PhrasePositions:LUCENE_BASE { + public: + int32_t doc; // current doc + int32_t position; // position in doc + int32_t count; // remaining pos in this doc + int32_t offset; // position in phrase + CL_NS(index)::TermPositions* tp; // stream of positions + PhrasePositions* _next; // used to make lists + + + //Constructor + PhrasePositions(CL_NS(index)::TermPositions* Tp, const int32_t o); + //Destructor + ~PhrasePositions(); + + bool next(); + bool skipTo(int32_t target); + + void firstPosition(); + + bool nextPosition(); + }; +CL_NS_END +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/search/PhraseQuery.cpp b/src/assistant/3rdparty/clucene/src/CLucene/search/PhraseQuery.cpp new file mode 100644 index 000000000..899cb3cfe --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/search/PhraseQuery.cpp @@ -0,0 +1,463 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#include "CLucene/StdHeader.h" +#include "PhraseQuery.h" + +#include "SearchHeader.h" +#include "Scorer.h" +#include "BooleanQuery.h" +#include "TermQuery.h" + +#include "CLucene/index/Term.h" +#include "CLucene/index/Terms.h" +#include "CLucene/index/IndexReader.h" + +#include "CLucene/util/StringBuffer.h" +#include "CLucene/util/VoidList.h" +#include "CLucene/util/Arrays.h" + +#include "ExactPhraseScorer.h" +#include "SloppyPhraseScorer.h" + +CL_NS_USE(index) +CL_NS_USE(util) +CL_NS_DEF(search) + + PhraseQuery::PhraseQuery(): + terms(false) + { + //Func - Constructor + //Pre - true + //Post - An empty PhraseQuery has been created + + slop = 0; + + field = NULL; + } + PhraseQuery::PhraseQuery(const PhraseQuery& clone): + Query(clone), terms(false) + { + slop = clone.slop; + field = clone.field; + int32_t size=clone.positions.size(); + { //msvc6 scope fix + for ( int32_t i=0;i<size;i++ ){ + int32_t n = clone.positions[i]; + this->positions.push_back( n ); + } + } + size=clone.terms.size(); + { //msvc6 scope fix + for ( int32_t i=0;i<size;i++ ){ + this->terms.push_back( _CL_POINTER(clone.terms[i])); + } + } + } + Query* PhraseQuery::clone() const{ + return _CLNEW PhraseQuery(*this); + } + bool PhraseQuery::equals(CL_NS(search)::Query *other) const{ + if (!(other->instanceOf(PhraseQuery::getClassName()))) + return false; + + PhraseQuery* pq = (PhraseQuery*)other; + bool ret = (this->getBoost() == pq->getBoost()) + && (this->slop == pq->slop); + + if ( ret ){ + CLListEquals<CL_NS(index)::Term,Term::Equals, + const CL_NS(util)::CLVector<CL_NS(index)::Term*>, + const CL_NS(util)::CLVector<CL_NS(index)::Term*> > comp; + ret = comp.equals(&this->terms,&pq->terms); + } + + if ( ret ){ + CLListEquals<int32_t,Equals::Int32, + const CL_NS(util)::CLVector<int32_t,CL_NS(util)::Deletor::DummyInt32>, + const CL_NS(util)::CLVector<int32_t,CL_NS(util)::Deletor::DummyInt32> > comp; + ret = comp.equals(&this->positions,&pq->positions); + } + return ret; + } + + + PhraseQuery::~PhraseQuery(){ + //Func - Destructor + //Pre - true + //Post 0 The instance has been destroyed + + //Iterate through all the terms + for (uint32_t i = 0; i < terms.size(); i++){ + _CLLDECDELETE(terms[i]); + } + positions.clear(); + } + + size_t PhraseQuery::hashCode() const { + //todo: do cachedHashCode, and invalidate on add/remove clause + size_t ret = Similarity::floatToByte(getBoost()) ^ Similarity::floatToByte(slop); + + { //msvc6 scope fix + for ( int32_t i=0;terms.size();i++ ) + ret = 31 * ret + terms[i]->hashCode(); + } + { //msvc6 scope fix + for ( int32_t i=0;positions.size();i++ ) + ret = 31 * ret + positions[i]; + } + return ret; + } + + const TCHAR* PhraseQuery::getClassName(){ + return _T("PhraseQuery"); + } + const TCHAR* PhraseQuery::getQueryName() const{ + //Func - Returns the string "PhraseQuery" + //Pre - true + //Post - The string "PhraseQuery" has been returned + return getClassName(); + } + + + /** + * Adds a term to the end of the query phrase. + * The relative position of the term is the one immediately after the last term added. + */ + void PhraseQuery::add(Term* term) { + CND_PRECONDITION(term != NULL,"term is NULL"); + + int32_t position = 0; + + if(positions.size() > 0) + position = (positions[positions.size()-1]) + 1; + + add(term, position); + } + + void PhraseQuery::add(Term* term, int32_t position) { + //Func - Adds a term to the end of the query phrase. + //Pre - term != NULL + //Post - The term has been added if its field matches the field of the PhraseQuery + // and true is returned otherwise false is returned + CND_PRECONDITION(term != NULL,"term is NULL"); + + if (terms.size() == 0) + field = term->field(); + else{ + //Check if the field of the _CLNEW term matches the field of the PhraseQuery + //can use != because fields are interned + if ( term->field() != field){ + //return false; + TCHAR buf[200]; + _sntprintf(buf,200,_T("All phrase terms must be in the same field: %s"),term->field()); + _CLTHROWT(CL_ERR_IllegalArgument,buf); + } + } + //Store the _CLNEW term + terms.push_back(_CL_POINTER(term)); + + positions.push_back(position); + } + + void PhraseQuery::getPositions(Array<int32_t>& result) const{ + result.length = positions.size(); + result.values = _CL_NEWARRAY(int32_t,result.length); + for(int32_t i = 0; i < result.length; i++){ + result.values[i] = positions[i]; + } + } + int32_t* PhraseQuery::getPositions() const{ + CND_WARNING(false,"getPositions() is deprecated") + + Array<int32_t> arr; + getPositions(arr); + return arr.values; + } + + Weight* PhraseQuery::_createWeight(Searcher* searcher) { + if (terms.size() == 1) { // optimize one-term case + Term* term = terms[0]; + Query* termQuery = _CLNEW TermQuery(term); + termQuery->setBoost(getBoost()); + Weight* ret = termQuery->_createWeight(searcher); + _CLDELETE(termQuery); + return ret; + } + return _CLNEW PhraseWeight(searcher,this); + } + + + Term** PhraseQuery::getTerms() const{ + //Func - added by search highlighter + //Pre - + //Post - + + //Let size contain the number of terms + int32_t size = terms.size(); + Term** ret = _CL_NEWARRAY(Term*,size+1); + + CND_CONDITION(ret != NULL,"Could not allocated memory for ret"); + + //Iterate through terms and copy each pointer to ret + for ( int32_t i=0;i<size;i++ ){ + ret[i] = terms[i]; + } + ret[size] = NULL; + return ret; + } + + TCHAR* PhraseQuery::toString(const TCHAR* f) const{ + //Func - Prints a user-readable version of this query. + //Pre - f != NULL + //Post - The query string has been returned + + if ( terms.size()== 0 ) + return NULL; + + StringBuffer buffer; + if ( f==NULL || _tcscmp(field,f)!=0) { + buffer.append(field); + buffer.append( _T(":")); + } + + buffer.append( _T("\"") ); + + Term *T = NULL; + + //iterate through all terms + for (uint32_t i = 0; i < terms.size(); i++) { + //Get the i-th term + T = terms[i]; + + //Ensure T is a valid Term + CND_CONDITION(T !=NULL,"T is NULL"); + + buffer.append( T->text() ); + //Check if i is at the end of terms + if (i != terms.size()-1){ + buffer.append(_T(" ")); + } + } + + buffer.append( _T("\"") ); + + if (slop != 0) { + buffer.append(_T("~")); + buffer.appendFloat(slop,0); + } + + //Check if there is an other boost factor than 1.0 + if (getBoost() != 1.0f) { + buffer.append(_T("^")); + buffer.appendFloat( getBoost(),1 ); + } + + //return the query string + return buffer.toString(); + } + + + + + + + + + PhraseQuery::PhraseWeight::PhraseWeight(Searcher* searcher, PhraseQuery* _this) { + this->_this=_this; + this->value = 0; + this->idf = 0; + this->queryNorm = 0; + this->queryWeight = 0; + this->searcher = searcher; + } + + TCHAR* PhraseQuery::PhraseWeight::toString() { + return STRDUP_TtoT(_T("weight(PhraseQuery)")); + } + PhraseQuery::PhraseWeight::~PhraseWeight(){ + } + + + Query* PhraseQuery::PhraseWeight::getQuery() { return _this; } + qreal PhraseQuery::PhraseWeight::getValue() { return value; } + + qreal PhraseQuery::PhraseWeight::sumOfSquaredWeights(){ + idf = _this->getSimilarity(searcher)->idf(&_this->terms, searcher); + queryWeight = idf * _this->getBoost(); // compute query weight + return queryWeight * queryWeight; // square it + } + + void PhraseQuery::PhraseWeight::normalize(qreal queryNorm) { + this->queryNorm = queryNorm; + queryWeight *= queryNorm; // normalize query weight + value = queryWeight * idf; // idf for document + } + + Scorer* PhraseQuery::PhraseWeight::scorer(IndexReader* reader) { + //Func - + //Pre - + //Post - + + //Get the length of terms + int32_t tpsLength = _this->terms.size(); + + //optimize zero-term case + if (tpsLength == 0) + return NULL; + + TermPositions** tps = _CL_NEWARRAY(TermPositions*,tpsLength+1); + + //Check if tps has been allocated properly + CND_CONDITION(tps != NULL,"Could not allocate memory for tps"); + + TermPositions* p = NULL; + + //Iterate through all terms + int32_t size = _this->terms.size(); + for (int32_t i = 0; i < size; i++) { + //Get the termPostitions for the i-th term + p = reader->termPositions(_this->terms[i]); + + //Check if p is valid + if (p == NULL) { + //Delete previous retrieved termPositions + while (--i >= 0){ + _CLVDELETE(tps[i]); //todo: not a clucene object... should be + } + _CLDELETE_ARRAY(tps); + return NULL; + } + + //Store p at i in tps + tps[i] = p; + } + tps[tpsLength] = NULL; + + Scorer* ret = NULL; + + Array<int32_t> positions; + _this->getPositions(positions); + int32_t slop = _this->getSlop(); + if ( slop != 0) + // optimize exact case + //todo: need to pass these: this, tps, + ret = _CLNEW SloppyPhraseScorer(this,tps,positions.values, + _this->getSimilarity(searcher), + slop, reader->norms(_this->field)); + else + ret = _CLNEW ExactPhraseScorer(this, tps, positions.values, + _this->getSimilarity(searcher), + reader->norms(_this->field)); + positions.deleteArray(); + + CND_CONDITION(ret != NULL,"Could not allocate memory for ret"); + + //tps can be deleted safely. SloppyPhraseScorer or ExactPhraseScorer will take care + //of its values + + _CLDELETE_ARRAY(tps); + return ret; + } + + void PhraseQuery::PhraseWeight::explain(IndexReader* reader, int32_t doc, Explanation* result){ + TCHAR descbuf[LUCENE_SEARCH_EXPLANATION_DESC_LEN+1]; + TCHAR* tmp; + + tmp = getQuery()->toString(); + _sntprintf(descbuf,LUCENE_SEARCH_EXPLANATION_DESC_LEN,_T("weight(%s in %d), product of:"), + tmp,doc); + _CLDELETE_CARRAY(tmp); + result->setDescription(descbuf); + + StringBuffer docFreqs; + StringBuffer query; + query.appendChar('\"'); + for (uint32_t i = 0; i < _this->terms.size(); i++) { + if (i != 0) { + docFreqs.appendChar(' '); + query.appendChar(' '); + } + + Term* term = _this->terms[i]; + + docFreqs.append(term->text()); + docFreqs.appendChar('='); + docFreqs.appendInt(searcher->docFreq(term)); + + query.append(term->text()); + } + query.appendChar('\"'); + + _sntprintf(descbuf,LUCENE_SEARCH_EXPLANATION_DESC_LEN, + _T("idf(%s: %s)"),_this->field,docFreqs.getBuffer()); + Explanation* idfExpl = _CLNEW Explanation(idf, descbuf); + + // explain query weight + Explanation* queryExpl = _CLNEW Explanation; + tmp = getQuery()->toString(); + _sntprintf(descbuf,LUCENE_SEARCH_EXPLANATION_DESC_LEN, + _T("queryWeight(%s), product of:"),tmp); + _CLDELETE_CARRAY(tmp); + queryExpl->setDescription(descbuf); + + Explanation* boostExpl = _CLNEW Explanation(_this->getBoost(), _T("boost")); + if (_this->getBoost() != 1.0f) + queryExpl->addDetail(boostExpl); + queryExpl->addDetail(idfExpl); + + Explanation* queryNormExpl = _CLNEW Explanation(queryNorm,_T("queryNorm")); + queryExpl->addDetail(queryNormExpl); + + queryExpl->setValue(boostExpl->getValue() * + idfExpl->getValue() * + queryNormExpl->getValue()); + + result->addDetail(queryExpl); + + // explain field weight + Explanation* fieldExpl = _CLNEW Explanation; + _sntprintf(descbuf,LUCENE_SEARCH_EXPLANATION_DESC_LEN, + _T("fieldWeight(%s:%s in %d), product of:"), + _this->field,query.getBuffer(),doc); + fieldExpl->setDescription(descbuf); + + + Explanation* tfExpl = _CLNEW Explanation; + scorer(reader)->explain(doc, tfExpl); + fieldExpl->addDetail(tfExpl); + fieldExpl->addDetail(idfExpl); + + Explanation* fieldNormExpl = _CLNEW Explanation(); + uint8_t* fieldNorms = reader->norms(_this->field); + qreal fieldNorm = + fieldNorms!=NULL ? Similarity::decodeNorm(fieldNorms[doc]) : 0.0f; + fieldNormExpl->setValue(fieldNorm); + + + _sntprintf(descbuf,LUCENE_SEARCH_EXPLANATION_DESC_LEN, + _T("fieldNorm(field=%s, doc=%d)"),_this->field,doc); + fieldNormExpl->setDescription(descbuf); + fieldExpl->addDetail(fieldNormExpl); + + fieldExpl->setValue(tfExpl->getValue() * + idfExpl->getValue() * + fieldNormExpl->getValue()); + + result->addDetail(fieldExpl); + + // combine them + result->setValue(queryExpl->getValue() * fieldExpl->getValue()); + + if (queryExpl->getValue() == 1.0f){ + result->set(*fieldExpl); + _CLDELETE(fieldExpl); + } + } + + +CL_NS_END diff --git a/src/assistant/3rdparty/clucene/src/CLucene/search/PhraseQuery.h b/src/assistant/3rdparty/clucene/src/CLucene/search/PhraseQuery.h new file mode 100644 index 000000000..6b3255822 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/search/PhraseQuery.h @@ -0,0 +1,127 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _lucene_search_PhraseQuery_ +#define _lucene_search_PhraseQuery_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include "SearchHeader.h" +#include "Scorer.h" +#include "BooleanQuery.h" +#include "TermQuery.h" + +#include "CLucene/index/Term.h" +#include "CLucene/index/Terms.h" +#include "CLucene/index/IndexReader.h" + +#include "CLucene/util/StringBuffer.h" +#include "CLucene/util/VoidList.h" + +#include "ExactPhraseScorer.h" +#include "SloppyPhraseScorer.h" + +CL_NS_DEF(search) + // A Query that matches documents containing a particular sequence of terms. + // This may be combined with other terms with a {@link BooleanQuery}. + class PhraseQuery: public Query { + private: + CL_NS(util)::CLVector<int32_t,CL_NS(util)::Deletor::DummyInt32> positions; + int32_t slop; + + const TCHAR* field; + CL_NS(util)::CLVector<CL_NS(index)::Term*> terms; + + + class PhraseWeight: public Weight { + private: + Searcher* searcher; + qreal value; + qreal idf; + qreal queryNorm; + qreal queryWeight; + PhraseQuery* _this; + public: + PhraseWeight(Searcher* searcher, PhraseQuery* _this); + ~PhraseWeight(); + TCHAR* toString(); + + Query* getQuery(); + qreal getValue(); + + qreal sumOfSquaredWeights(); + void normalize(qreal queryNorm); + Scorer* scorer(CL_NS(index)::IndexReader* reader); + void explain(CL_NS(index)::IndexReader* reader, int32_t doc, Explanation* ret); + TCHAR* toString(TCHAR* f); + bool equals(PhraseWeight* o); + }; + friend class PhraseWeight; + protected: + Weight* _createWeight(Searcher* searcher); + PhraseQuery(const PhraseQuery& clone); + public: + //Constructor + PhraseQuery(); + + //Destructor + ~PhraseQuery(); + + //Returns the string "PhraseQuery" + const TCHAR* getQueryName() const; + static const TCHAR* getClassName(); + + //Sets the number of other words permitted between words in query phrase. + //If zero, then this is an exact phrase search. For larger values this works + //like a WITHIN or NEAR operator. + // + //The slop is in fact an edit-distance, where the units correspond to + //moves of terms in the query phrase out of position. For example, to switch + //the order of two words requires two moves (the first move places the words + //atop one another), so to permit re-orderings of phrases, the slop must be + //at least two. + // + //More exact matches are scored higher than sloppier matches, thus search + //results are sorted by exactness. + // + //The slop is zero by default, requiring exact matches. + void setSlop(const int32_t s) { slop = s; } + + //Returns the slop. See setSlop(). + int32_t getSlop() const { return slop; } + + //Adds a term to the end of the query phrase. + void add(CL_NS(index)::Term* term); + void add(CL_NS(index)::Term* term, int32_t position); + + + + //Returns the sum of squared weights + qreal sumOfSquaredWeights(Searcher* searcher); + + //Normalizes the Weight + void normalize(const qreal norm); + + Scorer* scorer(CL_NS(index)::IndexReader* reader); + + //added by search highlighter + CL_NS(index)::Term** getTerms() const; + _CL_DEPRECATED( deleteDocuments ) int32_t* getPositions() const; ///@deprecated. use getPositions(Array<int32_t>& result) + void getPositions(Array<int32_t>& result) const; + const TCHAR* getFieldName() const{ return field; } + + //Prints a user-readable version of this query. + TCHAR* toString(const TCHAR* f) const; + + Query* clone() const; + bool equals(CL_NS(search)::Query *) const; + + size_t hashCode() const; + }; +CL_NS_END +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/search/PhraseQueue.h b/src/assistant/3rdparty/clucene/src/CLucene/search/PhraseQueue.h new file mode 100644 index 000000000..c0682fcaf --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/search/PhraseQueue.h @@ -0,0 +1,36 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _lucene_search_PriorityQueue_ +#define _lucene_search_PriorityQueue_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include "CLucene/util/PriorityQueue.h" +#include "PhrasePositions.h" + +CL_NS_DEF(search) + class PhraseQueue: public CL_NS(util)::PriorityQueue<PhrasePositions*, + CL_NS(util)::Deletor::Object<PhrasePositions> > { + public: + PhraseQueue(const int32_t size) { + initialize(size,false); + } + ~PhraseQueue(){ + } + + protected: + bool lessThan(PhrasePositions* pp1, PhrasePositions* pp2) { + if (pp1->doc == pp2->doc) + return pp1->position < pp2->position; + else + return pp1->doc < pp2->doc; + } + }; +CL_NS_END +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/search/PhraseScorer.cpp b/src/assistant/3rdparty/clucene/src/CLucene/search/PhraseScorer.cpp new file mode 100644 index 000000000..b2da2316a --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/search/PhraseScorer.cpp @@ -0,0 +1,225 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#include "CLucene/StdHeader.h" +#include "PhraseScorer.h" + +#include "PhraseQueue.h" +#include "PhrasePositions.h" +#include "Scorer.h" +#include "Similarity.h" + +CL_NS_USE(index) +CL_NS_USE(util) +CL_NS_DEF(search) + + + PhraseScorer::PhraseScorer(Weight* weight, TermPositions** tps, + int32_t* positions, Similarity* similarity, uint8_t* norms): + Scorer(similarity) + { + //Func - Constructor + //Pre - tps != NULL and is an array of TermPositions + // tpsLength >= 0 + // n != NULL + //Post - The instance has been created + + CND_PRECONDITION(tps != NULL,"tps is NULL"); + + //norms are only used if phraseFreq returns more than 0.0 + //phraseFreq should only return more than 0.0 if norms != NULL + //CND_PRECONDITION(n != NULL,"n is NULL"); + + firstTime = true; + more = true; + this->norms = norms; + this->weight = weight; + this->value = weight->getValue(); + + //reset internal pointers + first = NULL; + last = NULL; + + //use pq to build a sorted list of PhrasePositions + int32_t i = 0; + while(tps[i] != NULL){ + PhrasePositions *pp = _CLNEW PhrasePositions(tps[i], positions[i]); + CND_CONDITION(pp != NULL,"Could not allocate memory for pp"); + + //Store PhrasePos into the PhrasePos pq + if (last != NULL) { // add next to end of list + last->_next = pp; + } else + first = pp; + last = pp; + + i++; + } + + pq = _CLNEW PhraseQueue(i); //i==tps.length + CND_CONDITION(pq != NULL,"Could not allocate memory for pq"); + } + + PhraseScorer::~PhraseScorer() { + //Func - Destructor + //Pre - true + //Post - The instance has been destroyed + + //The PhraseQueue pq (which is a PriorityQueue) pq is actually empty at present, the elements + //having been transferred by pqToList() to the linked list starting with + //first. The nodes of that linked list are deleted by the destructor of + //first, rather than the destructor of pq. + _CLDELETE(first); + _CLDELETE(pq); + } + + bool PhraseScorer::next(){ + if (firstTime) { + init(); + firstTime = false; + } else if (more) { + more = last->next(); // trigger further scanning + } + return doNext(); + } + + // next without initial increment + bool PhraseScorer::doNext() { + while (more) { + while (more && first->doc < last->doc) { // find doc w/ all the terms + more = first->skipTo(last->doc); // skip first upto last + firstToLast(); // and move it to the end + } + + if (more) { + // found a doc with all of the terms + freq = phraseFreq(); // check for phrase + if (freq == 0.0f) // no match + more = last->next(); // trigger further scanning + else + return true; // found a match + } + } + return false; // no more matches + } + + qreal PhraseScorer::score(){ + //System.out.println("scoring " + first.doc); + qreal raw = getSimilarity()->tf(freq) * value; // raw score + return raw * Similarity::decodeNorm(norms[first->doc]); // normalize + } + + bool PhraseScorer::skipTo(int32_t target) { + for (PhrasePositions* pp = first; more && pp != NULL; pp = pp->_next) { + more = pp->skipTo(target); + } + if (more) + sort(); // re-sort + return doNext(); + } + + void PhraseScorer::init() { + for (PhrasePositions* pp = first; more && pp != NULL; pp = pp->_next) + more = pp->next(); + if(more) + sort(); + } + + void PhraseScorer::sort() { + pq->clear(); + for (PhrasePositions* pp = first; pp != NULL; pp = pp->_next) + pq->put(pp); + pqToList(); + } + + + + void PhraseScorer::pqToList(){ + //Func - Transfers the PhrasePositions from the PhraseQueue pq to + // the PhrasePositions list with first as its first element + //Pre - pq != NULL + // first = NULL + // last = NULL + //Post - All PhrasePositions have been transfered to the list + // of PhrasePositions of which the first element is pointed to by first + // and the last element is pointed to by last + + CND_PRECONDITION(pq != NULL,"pq is NULL"); + + last = first = NULL; + + PhrasePositions* PhrasePos = NULL; + + //As long pq is not empty + while (pq->top() != NULL){ + //Pop a PhrasePositions instance + PhrasePos = pq->pop(); + + // add next to end of list + if (last != NULL) { + last->_next = PhrasePos; + } else { + first = PhrasePos; + } + + //Let last point to the new last PhrasePositions instance just added + last = PhrasePos; + //Reset the next of last to NULL + last->_next = NULL; + } + + //Check to see that pq is empty now + CND_CONDITION(pq->size()==0, "pq is not empty while it should be"); + } + + void PhraseScorer::firstToLast(){ + //Func - Moves first to the end of the list + //Pre - first is NULL or points to an PhrasePositions Instance + // last is NULL or points to an PhrasePositions Instance + // first and last both are NULL or both are not NULL + //Post - The first element has become the last element in the list + + CND_PRECONDITION(((first==NULL && last==NULL) ||(first !=NULL && last != NULL)), + "Either first or last is NULL but not both"); + + //Check if first and last are valid pointers + if(first && last){ + last->_next = first; + last = first; + first = first->_next; + last->_next = NULL; + } + } + + + void PhraseScorer::explain(int32_t _doc, Explanation* tfExplanation) { + while (next() && doc() < _doc){ + } + + qreal phraseFreq = (doc() == _doc) ? freq : 0.0f; + tfExplanation->setValue(getSimilarity()->tf(phraseFreq)); + + StringBuffer buf; + buf.append(_T("tf(phraseFreq=")); + buf.appendFloat(phraseFreq,2); + buf.append(_T(")")); + tfExplanation->setDescription(buf.getBuffer()); + } + + TCHAR* PhraseScorer::toString() { + StringBuffer buf; + buf.append(_T("scorer(")); + + TCHAR* tmp = weight->toString(); + buf.append(tmp); + _CLDELETE_CARRAY(tmp); + + buf.append(_T(")")); + + return buf.toString(); + } + +CL_NS_END diff --git a/src/assistant/3rdparty/clucene/src/CLucene/search/PhraseScorer.h b/src/assistant/3rdparty/clucene/src/CLucene/search/PhraseScorer.h new file mode 100644 index 000000000..89f7a1fbb --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/search/PhraseScorer.h @@ -0,0 +1,65 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _lucene_search_PhraseScorer_ +#define _lucene_search_PhraseScorer_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include "PhraseQueue.h" +#include "PhrasePositions.h" +#include "Scorer.h" +#include "Similarity.h" + +CL_NS_DEF(search) + + class PhraseScorer: public Scorer { + private: + Weight* weight; + qreal freq; + bool firstTime; + bool more; + + protected: + uint8_t* norms; + qreal value; + + PhraseQueue* pq; //is used to order the list point to by first and last + PhrasePositions* first; //Points to the first in the list of PhrasePositions + PhrasePositions* last; //Points to the last in the list of PhrasePositions + + public: + //Constructor + PhraseScorer(Weight* weight, CL_NS(index)::TermPositions** tps, + int32_t* positions, Similarity* similarity, uint8_t* norms); + virtual ~PhraseScorer(); + + int32_t doc() const { return first->doc; } + bool next(); + qreal score(); + bool skipTo(int32_t target); + + + void explain(int32_t doc, Explanation* ret); + TCHAR* toString(); + protected: + virtual qreal phraseFreq() =0; + + //Transfers the PhrasePositions from the PhraseQueue pq to + //the PhrasePositions list with first as its first element + void pqToList(); + + //Moves first to the end of the list + void firstToLast(); + private: + bool doNext(); + void init(); + void sort(); + }; +CL_NS_END +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/search/PrefixQuery.cpp b/src/assistant/3rdparty/clucene/src/CLucene/search/PrefixQuery.cpp new file mode 100644 index 000000000..6bb27d1ca --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/search/PrefixQuery.cpp @@ -0,0 +1,273 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#include "CLucene/StdHeader.h" +#include "PrefixQuery.h" +#include "CLucene/util/BitSet.h" + +CL_NS_USE(util) +CL_NS_USE(index) +CL_NS_DEF(search) + + PrefixQuery::PrefixQuery(Term* Prefix){ + //Func - Constructor. + // Constructs a query for terms starting with prefix + //Pre - Prefix != NULL + //Post - The instance has been created + + //Get a pointer to Prefix + prefix = _CL_POINTER(Prefix); + } + + PrefixQuery::PrefixQuery(const PrefixQuery& clone):Query(clone){ + prefix = _CL_POINTER(clone.prefix); + } + Query* PrefixQuery::clone() const{ + return _CLNEW PrefixQuery(*this); + } + + Term* PrefixQuery::getPrefix(bool pointer){ + if ( pointer ) + return _CL_POINTER(prefix); + else + return prefix; + } + + PrefixQuery::~PrefixQuery(){ + //Func - Destructor + //Pre - true + //Post - The instance has been destroyed. + + //Delete prefix by finalizing it + _CLDECDELETE(prefix); + } + + + /** Returns a hash code value for this object.*/ + size_t PrefixQuery::hashCode() const { + return Similarity::floatToByte(getBoost()) ^ prefix->hashCode(); + } + + const TCHAR* PrefixQuery::getQueryName()const{ + //Func - Returns the name "PrefixQuery" + //Pre - true + //Post - The string "PrefixQuery" has been returned + + return getClassName(); + } + const TCHAR* PrefixQuery::getClassName(){ + //Func - Returns the name "PrefixQuery" + //Pre - true + //Post - The string "PrefixQuery" has been returned + + return _T("PrefixQuery"); + } + + bool PrefixQuery::equals(Query * other) const{ + if (!(other->instanceOf(PrefixQuery::getClassName()))) + return false; + + PrefixQuery* rq = (PrefixQuery*)other; + bool ret = (this->getBoost() == rq->getBoost()) + && (this->prefix->equals(rq->prefix)); + + return ret; + } + + Query* PrefixQuery::rewrite(IndexReader* reader){ + BooleanQuery* query = _CLNEW BooleanQuery(); + TermEnum* enumerator = reader->terms(prefix); + Term* lastTerm = NULL; + try { + const TCHAR* prefixText = prefix->text(); + const TCHAR* prefixField = prefix->field(); + const TCHAR* tmp; + size_t i; + int32_t prefixLen = prefix->textLength(); + do { + lastTerm = enumerator->term(); + if (lastTerm != NULL && lastTerm->field() == prefixField ){ + + //now see if term->text() starts with prefixText + int32_t termLen = lastTerm->textLength(); + if ( prefixLen>termLen ) + break; //the prefix is longer than the term, can't be matched + + tmp = lastTerm->text(); + + //check for prefix match in reverse, since most change will be at the end + for ( i=prefixLen-1;i!=-1;--i ){ + if ( tmp[i] != prefixText[i] ){ + tmp=NULL;//signals inequality + break; + } + } + if ( tmp == NULL ) + break; + + TermQuery* tq = _CLNEW TermQuery(lastTerm); // found a match + tq->setBoost(getBoost()); // set the boost + query->add(tq,true,false, false); // add to query + } else + break; + _CLDECDELETE(lastTerm); + } while (enumerator->next()); + }_CLFINALLY( + enumerator->close(); + _CLDELETE(enumerator); + _CLDECDELETE(lastTerm); + ); + _CLDECDELETE(lastTerm); + + + //if we only added one clause and the clause is not prohibited then + //we can just return the query + if (query->getClauseCount() == 1) { // optimize 1-clause queries + BooleanClause* c=0; + query->getClauses(&c); + + if (!c->prohibited) { // just return clause + c->deleteQuery=false; + Query* ret = c->query; + + _CLDELETE(query); + return ret; + } + } + + return query; + } + + Query* PrefixQuery::combine(Query** queries) { + return Query::mergeBooleanQueries(queries); + } + + TCHAR* PrefixQuery::toString(const TCHAR* field) const{ + //Func - Creates a user-readable version of this query and returns it as as string + //Pre - field != NULL + //Post - a user-readable version of this query has been returned as as string + + //Instantiate a stringbuffer buffer to store the readable version temporarily + CL_NS(util)::StringBuffer buffer; + //check if field equal to the field of prefix + if( field==NULL || _tcscmp(prefix->field(),field) != 0 ) { + //Append the field of prefix to the buffer + buffer.append(prefix->field()); + //Append a colon + buffer.append(_T(":") ); + } + //Append the text of the prefix + buffer.append(prefix->text()); + //Append a wildchar character + buffer.append(_T("*")); + //if the boost factor is not eaqual to 1 + if (getBoost() != 1.0f) { + //Append ^ + buffer.append(_T("^")); + //Append the boost factor + buffer.appendFloat( getBoost(),1); + } + //Convert StringBuffer buffer to TCHAR block and return it + return buffer.toString(); + } + + + + + + + + +PrefixFilter::PrefixFilter( Term* prefix ) +{ + this->prefix = _CL_POINTER(prefix); +} + +PrefixFilter::~PrefixFilter() +{ + _CLDECDELETE(prefix); +} + +PrefixFilter::PrefixFilter( const PrefixFilter& copy ) : + prefix( _CL_POINTER(copy.prefix) ) +{ +} + +Filter* PrefixFilter::clone() const { + return _CLNEW PrefixFilter(*this ); +} + +TCHAR* PrefixFilter::toString() +{ + //Instantiate a stringbuffer buffer to store the readable version temporarily + CL_NS(util)::StringBuffer buffer; + //check if field equal to the field of prefix + if( prefix->field() != NULL ) { + //Append the field of prefix to the buffer + buffer.append(prefix->field()); + //Append a colon + buffer.append(_T(":") ); + } + //Append the text of the prefix + buffer.append(prefix->text()); + buffer.append(_T("*")); + + //Convert StringBuffer buffer to TCHAR block and return it + return buffer.toString(); +} + +/** Returns a BitSet with true for documents which should be permitted in +search results, and false for those that should not. */ +BitSet* PrefixFilter::bits( IndexReader* reader ) +{ + BitSet* bts = _CLNEW BitSet( reader->maxDoc() ); + TermEnum* enumerator = reader->terms(prefix); + TermDocs* docs = reader->termDocs(); + const TCHAR* prefixText = prefix->text(); + const TCHAR* prefixField = prefix->field(); + const TCHAR* tmp; + size_t i; + int32_t prefixLen = prefix->textLength(); + Term* lastTerm = NULL; + + try{ + do{ + lastTerm = enumerator->term(false); + if (lastTerm != NULL && lastTerm->field() == prefixField ){ + //now see if term->text() starts with prefixText + int32_t termLen = lastTerm->textLength(); + if ( prefixLen>termLen ) + break; //the prefix is longer than the term, can't be matched + + tmp = lastTerm->text(); + + //check for prefix match in reverse, since most change will be at the end + for ( i=prefixLen-1;i!=-1;--i ){ + if ( tmp[i] != prefixText[i] ){ + tmp=NULL;//signals inequality + break; + } + } + if ( tmp == NULL ) + break; + + docs->seek(enumerator); + while (docs->next()) { + bts->set(docs->doc()); + } + } + }while(enumerator->next()); + } _CLFINALLY( + docs->close(); + _CLDELETE(docs); + enumerator->close(); + _CLDELETE(enumerator); + ) + + return bts; +} + +CL_NS_END diff --git a/src/assistant/3rdparty/clucene/src/CLucene/search/PrefixQuery.h b/src/assistant/3rdparty/clucene/src/CLucene/search/PrefixQuery.h new file mode 100644 index 000000000..8e3f41352 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/search/PrefixQuery.h @@ -0,0 +1,75 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _lucene_search_PrefixQuery +#define _lucene_search_PrefixQuery +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include "CLucene/index/Term.h" +#include "CLucene/index/Terms.h" +#include "CLucene/index/IndexReader.h" +#include "SearchHeader.h" +#include "BooleanQuery.h" +#include "TermQuery.h" +#include "CLucene/util/StringBuffer.h" + +CL_NS_DEF(search) + //PrefixQuery is a Query that matches documents containing terms with a specified prefix. + + class PrefixQuery: public Query { + private: + CL_NS(index)::Term* prefix; + protected: + PrefixQuery(const PrefixQuery& clone); + public: + + //Constructor. Constructs a query for terms starting with prefix + PrefixQuery(CL_NS(index)::Term* Prefix); + + //Destructor + ~PrefixQuery(); + + //Returns the name "PrefixQuery" + const TCHAR* getQueryName() const; + static const TCHAR* getClassName(); + + /** Returns the prefix of this query. */ + CL_NS(index)::Term* getPrefix(bool pointer=true); + + Query* combine(Query** queries); + Query* rewrite(CL_NS(index)::IndexReader* reader); + Query* clone() const; + bool equals(Query * other) const; + + //Creates a user-readable version of this query and returns it as as string + TCHAR* toString(const TCHAR* field) const; + + size_t hashCode() const; + }; + + + class PrefixFilter: public Filter + { + private: + CL_NS(index)::Term* prefix; + protected: + PrefixFilter( const PrefixFilter& copy ); + + public: + PrefixFilter(CL_NS(index)::Term* prefix); + ~PrefixFilter(); + + /** Returns a BitSet with true for documents which should be permitted in + search results, and false for those that should not. */ + CL_NS(util)::BitSet* bits( CL_NS(index)::IndexReader* reader ); + + Filter* clone() const; + TCHAR* toString(); + }; +CL_NS_END +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/search/QueryFilter.cpp b/src/assistant/3rdparty/clucene/src/CLucene/search/QueryFilter.cpp new file mode 100644 index 000000000..2dbe2d7cd --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/search/QueryFilter.cpp @@ -0,0 +1,73 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#include "CLucene/StdHeader.h" +#include "QueryFilter.h" +#include "IndexSearcher.h" + +CL_NS_DEF(search) +CL_NS_USE(util) +CL_NS_USE(index) + + +QueryFilter::QueryFilter( const Query* query ) +{ + this->query = query->clone(); +} + + +QueryFilter::~QueryFilter() +{ + _CLDELETE( query ); +} + + +QueryFilter::QueryFilter( const QueryFilter& copy ) +{ + this->query = copy.query->clone(); +} + + +Filter* QueryFilter::clone() const { + return _CLNEW QueryFilter(*this ); +} + + +TCHAR* QueryFilter::toString() +{ + TCHAR* qt = query->toString(); + size_t len = _tcslen(qt) + 14; + TCHAR* ret = _CL_NEWARRAY( TCHAR, len ); + ret[0] = 0; + _sntprintf( ret, len, _T("QueryFilter(%s)"), qt ); + _CLDELETE_CARRAY(qt); + return ret; +} + + +/** Returns a BitSet with true for documents which should be permitted in +search results, and false for those that should not. */ +BitSet* QueryFilter::bits( IndexReader* reader ) +{ + BitSet* bits = _CLNEW BitSet(reader->maxDoc()); + + IndexSearcher s(reader); + QFHitCollector hc(bits); + s._search(query, NULL, &hc); + return bits; +} + + +QueryFilter::QFHitCollector::QFHitCollector(CL_NS(util)::BitSet* bits){ + this->bits = bits; +} + +void QueryFilter::QFHitCollector::collect(const int32_t doc, const qreal score) { + bits->set(doc); // set bit for hit +} + + +CL_NS_END diff --git a/src/assistant/3rdparty/clucene/src/CLucene/search/QueryFilter.h b/src/assistant/3rdparty/clucene/src/CLucene/search/QueryFilter.h new file mode 100644 index 000000000..8d423b2f7 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/search/QueryFilter.h @@ -0,0 +1,44 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _lucene_search_QueryFilter_ +#define _lucene_search_QueryFilter_ + +#include "CLucene/util/BitSet.h" +#include "CLucene/index/IndexReader.h" +#include "SearchHeader.h" +#include "CachingWrapperFilter.h" + +CL_NS_DEF(search) + +class QueryFilter: public Filter +{ +private: + Query* query; + + class QFHitCollector: public HitCollector{ + CL_NS(util)::BitSet* bits; + public: + QFHitCollector(CL_NS(util)::BitSet* bits); + void collect(const int32_t doc, const qreal score); + }; + +protected: + QueryFilter( const QueryFilter& copy ); +public: + QueryFilter( const Query* query ); + + ~QueryFilter(); + + CL_NS(util)::BitSet* bits( CL_NS(index)::IndexReader* reader ); + + Filter *clone() const; + + TCHAR *toString(); +}; + +CL_NS_END +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/search/RangeFilter.cpp b/src/assistant/3rdparty/clucene/src/CLucene/search/RangeFilter.cpp new file mode 100644 index 000000000..66ee5ce55 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/search/RangeFilter.cpp @@ -0,0 +1,150 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#include "CLucene/StdHeader.h" +#include "RangeFilter.h" + +CL_NS_DEF(search) +CL_NS_USE(index) +CL_NS_USE(util) +CL_NS_USE(document) + + +RangeFilter::RangeFilter( const TCHAR* fieldName, const TCHAR* lowerTerm, const TCHAR* upperTerm, bool includeLower, bool includeUpper ) +{ + this->field = STRDUP_TtoT(fieldName); + if ( lowerTerm != NULL ) + this->lowerValue = STRDUP_TtoT(lowerTerm); + else + this->lowerValue = NULL; + if ( upperTerm != NULL ) + this->upperValue = STRDUP_TtoT(upperTerm); + else + this->upperValue = NULL; + this->includeLower = includeLower; + this->includeUpper = includeUpper; +} + + +/** + * Constructs a filter for field <code>fieldName</code> matching + * less than or equal to <code>upperTerm</code>. + */ +RangeFilter* RangeFilter::Less( TCHAR* fieldName, TCHAR* upperTerm ) { + return new RangeFilter( fieldName, NULL, upperTerm, false, true ); +} + + +/** +* Constructs a filter for field <code>fieldName</code> matching +* more than or equal to <code>lowerTerm</code>. +*/ +RangeFilter* RangeFilter::More( TCHAR* fieldName, TCHAR* lowerTerm ) { + return new RangeFilter( fieldName, lowerTerm, NULL, true, false ); +} + + +RangeFilter::~RangeFilter() +{ + _CLDELETE_CARRAY( lowerValue ); + _CLDELETE_CARRAY( field ); + _CLDELETE_CARRAY( upperValue ); +} + + +RangeFilter::RangeFilter( const RangeFilter& copy ) : + field( STRDUP_TtoT(copy.field) ), + lowerValue( STRDUP_TtoT(copy.lowerValue) ), + upperValue( STRDUP_TtoT(copy.upperValue) ), + includeLower( copy.includeLower ), + includeUpper( copy.includeUpper ) +{ +} + + +Filter* RangeFilter::clone() const { + return _CLNEW RangeFilter(*this ); +} + + +TCHAR* RangeFilter::toString() +{ + size_t len = (field ? _tcslen(field) : 0) + (lowerValue ? _tcslen(lowerValue) : 0) + (upperValue ? _tcslen(upperValue) : 0) + 8; + TCHAR* ret = _CL_NEWARRAY( TCHAR, len ); + ret[0] = 0; + _sntprintf( ret, len, _T("%s: [%s-%s]"), field, (lowerValue?lowerValue:_T("")), (upperValue?upperValue:_T("")) ); + + return ret; +} + + +/** Returns a BitSet with true for documents which should be permitted in +search results, and false for those that should not. */ +BitSet* RangeFilter::bits( IndexReader* reader ) +{ + BitSet* bts = _CLNEW BitSet( reader->maxDoc() ); + Term* term = NULL; + + Term* t = _CLNEW Term( field, (lowerValue ? lowerValue : _T("")), false ); + TermEnum* enumerator = reader->terms( t ); // get enumeration of all terms after lowerValue + _CLDECDELETE( t ); + + if( enumerator->term(false) == NULL ) { + _CLDELETE( enumerator ); + return bts; + } + + bool checkLower = false; + if( !includeLower ) // make adjustments to set to exclusive + checkLower = true; + + TermDocs* termDocs = reader->termDocs(); + + try + { + do + { + term = enumerator->term(); + + if( term == NULL || _tcscmp(term->field(), field) ) + break; + + if( !checkLower || lowerValue == NULL || _tcscmp(term->text(), lowerValue) > 0 ) + { + checkLower = false; + if( upperValue != NULL ) + { + int compare = _tcscmp( upperValue, term->text() ); + + /* if beyond the upper term, or is exclusive and + * this is equal to the upper term, break out */ + if( (compare < 0) || (!includeUpper && compare == 0) ) + break; + } + + termDocs->seek( enumerator->term(false) ); + while( termDocs->next() ) { + bts->set( termDocs->doc() ); + } + } + + _CLDECDELETE( term ); + } + while( enumerator->next() ); + } + _CLFINALLY + ( + _CLDECDELETE( term ); + termDocs->close(); + _CLVDELETE( termDocs ); + enumerator->close(); + _CLDELETE( enumerator ); + ); + + return bts; +} + +CL_NS_END diff --git a/src/assistant/3rdparty/clucene/src/CLucene/search/RangeFilter.h b/src/assistant/3rdparty/clucene/src/CLucene/search/RangeFilter.h new file mode 100644 index 000000000..0865e356f --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/search/RangeFilter.h @@ -0,0 +1,51 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ + +#ifndef _lucene_search_RangeFilter_ +#define _lucene_search_RangeFilter_ + +#include "CLucene/document/DateField.h" +#include "CLucene/index/Term.h" +#include "CLucene/index/Terms.h" +#include "CLucene/index/IndexReader.h" +#include "CLucene/util/BitSet.h" +#include "CLucene/search/Filter.h" + +CL_NS_DEF(search) + +class RangeFilter: public Filter +{ +private: + const TCHAR* field; + TCHAR* lowerValue; + TCHAR* upperValue; + bool includeLower; + bool includeUpper; + +protected: + RangeFilter( const RangeFilter& copy ); + +public: + RangeFilter( const TCHAR* fieldName, const TCHAR* lowerValue, const TCHAR* upperValue, bool includeLower, bool includeUpper ); + + static RangeFilter* Less( TCHAR* fieldName, TCHAR* upperTerm ); + + static RangeFilter* More( TCHAR* fieldName, TCHAR* lowerTerm ); + + ~RangeFilter(); + + /** Returns a BitSet with true for documents which should be permitted in + search results, and false for those that should not. */ + CL_NS(util)::BitSet* bits( CL_NS(index)::IndexReader* reader ); + + Filter* clone() const; + + TCHAR* toString(); +}; + +CL_NS_END +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/search/RangeQuery.cpp b/src/assistant/3rdparty/clucene/src/CLucene/search/RangeQuery.cpp new file mode 100644 index 000000000..4fc242089 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/search/RangeQuery.cpp @@ -0,0 +1,204 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#include "CLucene/StdHeader.h" +#include "RangeQuery.h" + +#include "SearchHeader.h" +#include "Scorer.h" +#include "BooleanQuery.h" +#include "TermQuery.h" + +#include "CLucene/index/Term.h" +#include "CLucene/index/Terms.h" +#include "CLucene/index/IndexReader.h" +#include "CLucene/util/StringBuffer.h" + + +CL_NS_USE(index) +CL_NS_USE(util) +CL_NS_DEF(search) + + RangeQuery::RangeQuery(Term* lowerTerm, Term* upperTerm, const bool Inclusive){ + //Func - Constructor + //Pre - (LowerTerm != NULL OR UpperTerm != NULL) AND + // if LowerTerm and UpperTerm are valid pointer then the fieldnames must be the same + //Post - The instance has been created + + if (lowerTerm == NULL && upperTerm == NULL) + { + _CLTHROWA(CL_ERR_IllegalArgument,"At least one term must be non-null"); + } + if (lowerTerm != NULL && upperTerm != NULL && lowerTerm->field() != upperTerm->field()) + { + _CLTHROWA(CL_ERR_IllegalArgument,"Both terms must be for the same field"); + } + + // if we have a lowerTerm, start there. otherwise, start at beginning + if (lowerTerm != NULL) { + this->lowerTerm = _CL_POINTER(lowerTerm); + } + else { + this->lowerTerm = _CLNEW Term(upperTerm, LUCENE_BLANK_STRING); + } + this->upperTerm = (upperTerm != NULL ? _CL_POINTER(upperTerm) : NULL); + this->inclusive = Inclusive; + } + RangeQuery::RangeQuery(const RangeQuery& clone): + Query(clone){ + this->inclusive = clone.inclusive; + this->upperTerm = (clone.upperTerm != NULL ? _CL_POINTER(clone.upperTerm) : NULL ); + this->lowerTerm = (clone.lowerTerm != NULL ? _CL_POINTER(clone.lowerTerm) : NULL ); + } + Query* RangeQuery::clone() const{ + return _CLNEW RangeQuery(*this); + } + + RangeQuery::~RangeQuery() { + //Func - Destructor + //Pre - true + //Post - The instance has been destroyed + + _CLDECDELETE(lowerTerm); + _CLDECDELETE(upperTerm); + } + + /** Returns a hash code value for this object.*/ + size_t RangeQuery::hashCode() const { + return Similarity::floatToByte(getBoost()) ^ + (lowerTerm != NULL ? lowerTerm->hashCode() : 0) ^ + (upperTerm != NULL ? upperTerm->hashCode() : 0) ^ + (this->inclusive ? 1 : 0); + } + + const TCHAR* RangeQuery::getQueryName() const{ + return getClassName(); + } + const TCHAR* RangeQuery::getClassName(){ + return _T("RangeQuery"); + } + + Query* RangeQuery::combine(Query** queries) { + return Query::mergeBooleanQueries(queries); + } + + bool RangeQuery::equals(Query * other) const{ + if (!(other->instanceOf(RangeQuery::getClassName()))) + return false; + + RangeQuery* rq = (RangeQuery*)other; + bool ret = (this->getBoost() == rq->getBoost()) + && (this->isInclusive() == rq->isInclusive()) + && (this->getLowerTerm()->equals(rq->getLowerTerm())) + && (this->getUpperTerm()->equals(rq->getUpperTerm())); + + return ret; + } + + + /** + * FIXME: Describe <code>rewrite</code> method here. + * + * @param reader an <code>IndexReader</code> value + * @return a <code>Query</code> value + * @exception IOException if an error occurs + */ + Query* RangeQuery::rewrite(IndexReader* reader){ + + BooleanQuery* query = _CLNEW BooleanQuery; + TermEnum* enumerator = reader->terms(lowerTerm); + Term* lastTerm = NULL; + try { + bool checkLower = false; + if (!inclusive) // make adjustments to set to exclusive + checkLower = true; + + const TCHAR* testField = getField(); + do { + lastTerm = enumerator->term(); + if (lastTerm != NULL && lastTerm->field() == testField ) { + if (!checkLower || _tcscmp(lastTerm->text(),lowerTerm->text()) > 0) { + checkLower = false; + if (upperTerm != NULL) { + int compare = _tcscmp(upperTerm->text(),lastTerm->text()); + /* if beyond the upper term, or is exclusive and + * this is equal to the upper term, break out */ + if ((compare < 0) || (!inclusive && compare == 0)) + break; + } + TermQuery* tq = _CLNEW TermQuery(lastTerm); // found a match + tq->setBoost(getBoost()); // set the boost + query->add(tq, true, false, false); // add to query + } + }else { + break; + } + _CLDECDELETE(lastTerm); + } + while (enumerator->next()); + }catch(...){ + _CLDECDELETE(lastTerm); //always need to delete this + _CLDELETE(query); //in case of error, delete the query + enumerator->close(); + _CLDELETE(enumerator); + throw; //rethrow + } + _CLDECDELETE(lastTerm); //always need to delete this + enumerator->close(); + _CLDELETE(enumerator); + + return query; + } + + /** Prints a user-readable version of this query. */ + TCHAR* RangeQuery::toString(const TCHAR* field) const + { + StringBuffer buffer; + if ( field==NULL || _tcscmp(getField(),field)!=0 ) + { + buffer.append( getField() ); + buffer.append( _T(":")); + } + buffer.append(inclusive ? _T("[") : _T("{")); + buffer.append(lowerTerm != NULL ? lowerTerm->text() : _T("NULL")); + buffer.append(_T(" TO ")); + buffer.append(upperTerm != NULL ? upperTerm->text() : _T("NULL")); + buffer.append(inclusive ? _T("]") : _T("}")); + if (getBoost() != 1.0f) + { + buffer.append( _T("^")); + buffer.appendFloat( getBoost(),1 ); + } + return buffer.toString(); + } + + + const TCHAR* RangeQuery::getField() const + { + return (lowerTerm != NULL ? lowerTerm->field() : upperTerm->field()); + } + + /** Returns the lower term of this range query */ + Term* RangeQuery::getLowerTerm(bool pointer) const { + if ( pointer ) + return _CL_POINTER(lowerTerm); + else + return lowerTerm; + } + + /** Returns the upper term of this range query */ + Term* RangeQuery::getUpperTerm(bool pointer) const { + if ( pointer ) + return _CL_POINTER(upperTerm); + else + return upperTerm; + } + + /** Returns <code>true</code> if the range query is inclusive */ + bool RangeQuery::isInclusive() const { return inclusive; } + + +CL_NS_END diff --git a/src/assistant/3rdparty/clucene/src/CLucene/search/RangeQuery.h b/src/assistant/3rdparty/clucene/src/CLucene/search/RangeQuery.h new file mode 100644 index 000000000..9a7733c33 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/search/RangeQuery.h @@ -0,0 +1,71 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _lucene_search_RangeQuery_ +#define _lucene_search_RangeQuery_ +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include "SearchHeader.h" +#include "Scorer.h" +#include "TermQuery.h" + +#include "CLucene/index/Term.h" +#include "CLucene/index/Terms.h" + +#include "CLucene/util/StringBuffer.h" + + +CL_NS_DEF(search) + /** Constructs a query selecting all terms greater than + * <code>lowerTerm</code> but less than <code>upperTerm</code>. + * There must be at least one term and either term may be null, + * in which case there is no bound on that side, but if there are + * two terms, both terms <b>must</b> be for the same field. + */ + class RangeQuery: public Query + { + private: + CL_NS(index)::Term* lowerTerm; + CL_NS(index)::Term* upperTerm; + bool inclusive; + protected: + RangeQuery(const RangeQuery& clone); + + public: + // Constructs a query selecting all terms greater than + // <code>lowerTerm</code> but less than <code>upperTerm</code>. + // There must be at least one term and either term may be NULL-- + // in which case there is no bound on that side, but if there are + // two term, both terms <b>must</b> be for the same field. + RangeQuery(CL_NS(index)::Term* LowerTerm, CL_NS(index)::Term* UpperTerm, const bool Inclusive); + ~RangeQuery(); + + const TCHAR* getQueryName() const; + static const TCHAR* getClassName(); + + Query* rewrite(CL_NS(index)::IndexReader* reader); + + Query* combine(Query** queries); + + // Prints a user-readable version of this query. + TCHAR* toString(const TCHAR* field) const; + + Query* clone() const; + + bool equals(Query * other) const; + + CL_NS(index)::Term* getLowerTerm(bool pointer=true) const; + CL_NS(index)::Term* getUpperTerm(bool pointer=true) const; + bool isInclusive() const; + const TCHAR* getField() const; + + size_t hashCode() const; + }; + +CL_NS_END +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/search/Scorer.h b/src/assistant/3rdparty/clucene/src/CLucene/search/Scorer.h new file mode 100644 index 000000000..0d1d4355a --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/search/Scorer.h @@ -0,0 +1,80 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _lucene_search_Scorer_ +#define _lucene_search_Scorer_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include "Similarity.h" +#include "SearchHeader.h" +#include "Explanation.h" + +CL_NS_DEF(search) + /** Expert: Implements scoring for a class of queries. */ +class Scorer: LUCENE_BASE { + private: + Similarity* similarity; + protected: + /** Constructs a Scorer. */ + Scorer(Similarity* similarity) { + this->similarity = similarity; + } + public: + virtual ~Scorer(){ + } + + /** Returns the Similarity implementation used by this scorer. */ + Similarity* getSimilarity() const{ + return this->similarity; + } + + /** Scores all documents and passes them to a collector. */ + void score(HitCollector* hc) { + while (next()) { + hc->collect(doc(), score()); + } + } + + /** Advance to the next document matching the query. Returns true iff there + * is another match. */ + virtual bool next() = 0; + + /** Returns the current document number. Initially invalid, until {@link + * #next()} is called the first time. */ + virtual int32_t doc() const = 0; + + /** Returns the score of the current document. Initially invalid, until + * {@link #next()} is called the first time. */ + virtual qreal score() = 0; + + /** Skips to the first match beyond the current whose document number is + * greater than or equal to <i>target</i>. <p>Returns true iff there is such + * a match. <p>Behaves as if written: <pre> + * boolean skipTo(int32_t target) { + * do { + * if (!next()) + * return false; + * } while (target > doc()); + * return true; + * } + * </pre> + * Most implementations are considerably more efficient than that. + */ + virtual bool skipTo(int32_t target) = 0; + + /** Returns an explanation of the score for <code>doc</code>. */ + virtual void explain(int32_t doc, Explanation* ret) = 0; + + + /** Returns an string which explains the object */ + virtual TCHAR* toString() = 0; + + }; +CL_NS_END +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/search/SearchHeader.cpp b/src/assistant/3rdparty/clucene/src/CLucene/search/SearchHeader.cpp new file mode 100644 index 000000000..56e4ad585 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/search/SearchHeader.cpp @@ -0,0 +1,141 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#include "CLucene/StdHeader.h" +#include "SearchHeader.h" +#include "BooleanQuery.h" +#include "FieldDocSortedHitQueue.h" + +CL_NS_USE(index) +CL_NS_DEF(search) + +CL_NS(document)::Document* Searchable::doc(const int32_t i){ + CL_NS(document)::Document* ret = _CLNEW CL_NS(document)::Document; + if (!doc(i,ret) ) + _CLDELETE(ret); + return ret; +} + +//static +Query* Query::mergeBooleanQueries(Query** queries) { + CL_NS(util)::CLVector<BooleanClause*> allClauses; + int32_t i = 0; + while ( queries[i] != NULL ){ + BooleanQuery* bq = (BooleanQuery*)queries[i]; + + int32_t size = bq->getClauseCount(); + BooleanClause** clauses = _CL_NEWARRAY(BooleanClause*, size); + bq->getClauses(clauses); + + for (int32_t j = 0;j<size;++j ){ + allClauses.push_back(clauses[j]); + j++; + } + _CLDELETE_ARRAY(clauses); + i++; + } + + BooleanQuery* result = _CLNEW BooleanQuery(); + CL_NS(util)::CLVector<BooleanClause*>::iterator itr = allClauses.begin(); + while (itr != allClauses.end() ) { + result->add(*itr); + } + return result; +} + +Query::Query(const Query& clone):boost(clone.boost){ + //constructor +} +Weight* Query::_createWeight(Searcher* searcher){ + _CLTHROWA(CL_ERR_UnsupportedOperation,"UnsupportedOperationException: Query::_createWeight"); +} + +Query::Query(): + boost(1.0f) +{ + //constructor +} +Query::~Query(){ +} + +/** Expert: called to re-write queries into primitive queries. */ +Query* Query::rewrite(CL_NS(index)::IndexReader* reader){ + return this; +} + +Query* Query::combine(Query** queries){ + _CLTHROWA(CL_ERR_UnsupportedOperation,"UnsupportedOperationException: Query::combine"); +} +Similarity* Query::getSimilarity(Searcher* searcher) { + return searcher->getSimilarity(); +} +bool Query::instanceOf(const TCHAR* other) const{ + const TCHAR* t = getQueryName(); + if ( t==other || _tcscmp( t, other )==0 ) + return true; + else + return false; +} +TCHAR* Query::toString() const{ + return toString(LUCENE_BLANK_STRING); +} + +void Query::setBoost(qreal b) { boost = b; } + +qreal Query::getBoost() const { return boost; } + +Weight* Query::weight(Searcher* searcher){ + Query* query = searcher->rewrite(this); + Weight* weight = query->_createWeight(searcher); + qreal sum = weight->sumOfSquaredWeights(); + qreal norm = getSimilarity(searcher)->queryNorm(sum); + weight->normalize(norm); + return weight; +} + +TopFieldDocs::TopFieldDocs (int32_t totalHits, FieldDoc** fieldDocs, int32_t scoreDocsLen, SortField** fields): + TopDocs (totalHits, NULL, scoreDocsLen) +{ + this->fields = fields; + this->fieldDocs = fieldDocs; + this->scoreDocs = _CL_NEWARRAY(ScoreDoc,scoreDocsLen); + for (int32_t i=0;i<scoreDocsLen;i++ ) + this->scoreDocs[i] = this->fieldDocs[i]->scoreDoc; +} +TopFieldDocs::~TopFieldDocs(){ + if ( fieldDocs ){ + for (int32_t i=0;i<scoreDocsLength;i++) + _CLDELETE(fieldDocs[i]); + _CLDELETE_ARRAY(fieldDocs); + } + if ( fields != NULL ){ + for ( int i=0;fields[i]!=NULL;i++ ) + _CLDELETE(fields[i]); + _CLDELETE_ARRAY(fields); + } +} + +TopDocs::TopDocs(const int32_t th, ScoreDoc*sds, int32_t scoreDocsLen): + totalHits(th), + scoreDocsLength(scoreDocsLen), + scoreDocs(sds) +{ +//Func - Constructor +//Pre - sds may or may not be NULL +// sdLength >= 0 +//Post - The instance has been created + +} + +TopDocs::~TopDocs(){ +//Func - Destructor +//Pre - true +//Post - The instance has been destroyed + + _CLDELETE_ARRAY(scoreDocs); +} + +CL_NS_END diff --git a/src/assistant/3rdparty/clucene/src/CLucene/search/SearchHeader.h b/src/assistant/3rdparty/clucene/src/CLucene/search/SearchHeader.h new file mode 100644 index 000000000..4a896a5c5 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/search/SearchHeader.h @@ -0,0 +1,456 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _lucene_search_SearchHeader_ +#define _lucene_search_SearchHeader_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include "CLucene/index/IndexReader.h" +#include "CLucene/index/Term.h" +#include "Filter.h" +#include "CLucene/document/Document.h" +#include "Sort.h" +#include "CLucene/util/VoidList.h" +#include "Explanation.h" +#include "Similarity.h" + +CL_NS_DEF(search) + + //predefine classes + class Scorer; + class Query; + class Hits; + class Sort; + class FieldDoc; + class TopFieldDocs; + + /** Expert: Returned by low-level search implementations. + * @see TopDocs */ + struct ScoreDoc { + /** Expert: A hit document's number. + * @see Searcher#doc(int32_t) + */ + int32_t doc; + + /** Expert: The score of this document for the query. */ + qreal score; + }; + + /** Expert: Returned by low-level search implementations. + * @see Searcher#search(Query,Filter,int32_t) */ + class TopDocs:LUCENE_BASE { + public: + /** Expert: The total number of hits for the query. + * @see Hits#length() + */ + int32_t totalHits; + + /** Expert: The top hits for the query. */ + ScoreDoc* scoreDocs; + int32_t scoreDocsLength; + + /** Expert: Constructs a TopDocs. TopDocs takes ownership of the ScoreDoc array*/ + TopDocs(const int32_t th, ScoreDoc* sds, int32_t scoreDocsLength); + ~TopDocs(); + }; + + // Lower-level search API. + // @see Searcher#search(Query,HitCollector) + class HitCollector: LUCENE_BASE { + public: + /** Called once for every non-zero scoring document, with the document number + * and its score. + * + * <P>If, for example, an application wished to collect all of the hits for a + * query in a BitSet, then it might:<pre> + * Searcher searcher = new IndexSearcher(indexReader); + * final BitSet bits = new BitSet(indexReader.maxDoc()); + * searcher.search(query, new HitCollector() { + * public void collect(int32_t doc, float score) { + * bits.set(doc); + * } + * }); + * </pre> + * + * <p>Note: This is called in an inner search loop. For good search + * performance, implementations of this method should not call + * {@link Searcher#doc(int32_t)} or + * {@link IndexReader#document(int32_t)} on every + * document number encountered. Doing so can slow searches by an order + * of magnitude or more. + * <p>Note: The <code>score</code> passed to this method is a raw score. + * In other words, the score will not necessarily be a float whose value is + * between 0 and 1. + */ + virtual void collect(const int32_t doc, const qreal score) = 0; + virtual ~HitCollector(){} + }; + + /** Expert: Calculate query weights and build query scorers. + * + * <p>A Weight is constructed by a query, given a Searcher ({@link + * Query#_createWeight(Searcher)}). The {@link #sumOfSquaredWeights()} method + * is then called on the top-level query to compute the query normalization + * factor (@link Similarity#queryNorm(qreal)}). This factor is then passed to + * {@link #normalize(qreal)}. At this point the weighting is complete and a + * scorer may be constructed by calling {@link #scorer(IndexReader)}. + */ + class Weight: LUCENE_BASE { + public: + virtual ~Weight(){ + }; + + /** The query that this concerns. */ + virtual Query* getQuery() = 0; + + /** The weight for this query. */ + virtual qreal getValue() = 0; + + /** The sum of squared weights of contained query clauses. */ + virtual qreal sumOfSquaredWeights() = 0; + + /** Assigns the query normalization factor to this. */ + virtual void normalize(qreal norm) = 0; + + /** Constructs a scorer for this. */ + virtual Scorer* scorer(CL_NS(index)::IndexReader* reader) = 0; + + /** An explanation of the score computation for the named document. */ + virtual void explain(CL_NS(index)::IndexReader* reader, int32_t doc, Explanation* ret) = 0; + + virtual TCHAR* toString(){ + return STRDUP_TtoT(_T("Weight")); + } + }; + + class HitDoc:LUCENE_BASE { + public: + qreal score; + int32_t id; + CL_NS(document)::Document* doc; + + HitDoc* next; // in doubly-linked cache + HitDoc* prev; // in doubly-linked cache + + HitDoc(const qreal s, const int32_t i); + ~HitDoc(); + }; + + + + // A ranked list of documents, used to hold search results. + class Hits:LUCENE_BASE { + private: + Query* query; + Searcher* searcher; + Filter* filter; + const Sort* sort; + + size_t _length; // the total number of hits + CL_NS(util)::CLVector<HitDoc*, CL_NS(util)::Deletor::Object<HitDoc> > hitDocs; // cache of hits retrieved + + HitDoc* first; // head of LRU cache + HitDoc* last; // tail of LRU cache + int32_t numDocs; // number cached + int32_t maxDocs; // max to cache + + public: + Hits(Searcher* s, Query* q, Filter* f, const Sort* sort=NULL); + ~Hits(); + + /** Returns the total number of hits available in this set. */ + int32_t length() const; + + /** Returns the stored fields of the n<sup>th</sup> document in this set. + <p>Documents are cached, so that repeated requests for the same element may + return the same Document object. + * + * @memory Memory belongs to the hits object. Don't delete the return value. + */ + CL_NS(document)::Document& doc(const int32_t n); + + /** Returns the id for the nth document in this set. */ + int32_t id (const int32_t n); + + /** Returns the score for the nth document in this set. */ + qreal score(const int32_t n); + + private: + // Tries to add new documents to hitDocs. + // Ensures that the hit numbered <code>_min</code> has been retrieved. + void getMoreDocs(const size_t _min); + + HitDoc* getHitDoc(const size_t n); + + void addToFront(HitDoc* hitDoc); + + void remove(const HitDoc* hitDoc); + + }; + + /** The interface for search implementations. + * + * <p>Implementations provide search over a single index, over multiple + * indices, and over indices on remote servers. + */ + class Searchable: LUCENE_BASE { + public: + virtual ~Searchable(){ + } + + /** Lower-level search API. + * + * <p>{@link HitCollector#collect(int32_t,qreal)} is called for every non-zero + * scoring document. + * + * <p>Applications should only use this if they need <i>all</i> of the + * matching documents. The high-level search API ({@link + * Searcher#search(Query*)}) is usually more efficient, as it skips + * non-high-scoring hits. + * + * @param query to match documents + * @param filter if non-null, a bitset used to eliminate some documents + * @param results to receive hits + */ + virtual void _search(Query* query, Filter* filter, HitCollector* results) = 0; + + /** Frees resources associated with this Searcher. + * Be careful not to call this method while you are still using objects + * like {@link Hits}. + */ + virtual void close() = 0; + + /** Expert: Returns the number of documents containing <code>term</code>. + * Called by search code to compute term weights. + * @see IndexReader#docFreq(Term). + */ + virtual int32_t docFreq(const CL_NS(index)::Term* term) const = 0; + + /** Expert: Returns one greater than the largest possible document number. + * Called by search code to compute term weights. + * @see IndexReader#maxDoc(). + */ + virtual int32_t maxDoc() const = 0; + + /** Expert: Low-level search implementation. Finds the top <code>n</code> + * hits for <code>query</code>, applying <code>filter</code> if non-null. + * + * <p>Called by {@link Hits}. + * + * <p>Applications should usually call {@link Searcher#search(Query*)} or + * {@link Searcher#search(Query*,Filter*)} instead. + */ + virtual TopDocs* _search(Query* query, Filter* filter, const int32_t n) = 0; + + /** Expert: Returns the stored fields of document <code>i</code>. + * Called by {@link HitCollector} implementations. + * @see IndexReader#document(int32_t). + */ + virtual bool doc(int32_t i, CL_NS(document)::Document* d) = 0; + _CL_DEPRECATED( doc(i, document) ) CL_NS(document)::Document* doc(const int32_t i); + + /** Expert: called to re-write queries into primitive queries. */ + virtual Query* rewrite(Query* query) = 0; + + /** Returns an Explanation that describes how <code>doc</code> scored against + * <code>query</code>. + * + * <p>This is intended to be used in developing Similarity implementations, + * and, for good performance, should not be displayed with every hit. + * Computing an explanation is as expensive as executing the query over the + * entire index. + */ + virtual void explain(Query* query, int32_t doc, Explanation* ret) = 0; + + /** Expert: Low-level search implementation with arbitrary sorting. Finds + * the top <code>n</code> hits for <code>query</code>, applying + * <code>filter</code> if non-null, and sorting the hits by the criteria in + * <code>sort</code>. + * + * <p>Applications should usually call {@link + * Searcher#search(Query,Filter,Sort)} instead. + */ + virtual TopFieldDocs* _search(Query* query, Filter* filter, const int32_t n, const Sort* sort) = 0; + }; + + + + /** An abstract base class for search implementations. + * Implements some common utility methods. + */ + class Searcher:public Searchable { + private: + /** The Similarity implementation used by this searcher. */ + Similarity* similarity; + + public: + Searcher(){ + similarity = Similarity::getDefault(); + } + virtual ~Searcher(){ + } + + // Returns the documents matching <code>query</code>. + Hits* search(Query* query) { + return search(query, (Filter*)NULL ); + } + + // Returns the documents matching <code>query</code> and + // <code>filter</code>. + Hits* search(Query* query, Filter* filter) { + return _CLNEW Hits(this, query, filter); + } + + /** Returns documents matching <code>query</code> sorted by + * <code>sort</code>. + */ + Hits* search(Query* query, const Sort* sort){ + return _CLNEW Hits(this, query, NULL, sort); + } + + /** Returns documents matching <code>query</code> and <code>filter</code>, + * sorted by <code>sort</code>. + */ + Hits* search(Query* query, Filter* filter, const Sort* sort){ + return _CLNEW Hits(this, query, filter, sort); + } + + /** Lower-level search API. + * + * <p>{@link HitCollector#collect(int32_t ,qreal)} is called for every non-zero + * scoring document. + * + * <p>Applications should only use this if they need <i>all</i> of the + * matching documents. The high-level search API ({@link + * Searcher#search(Query*)}) is usually more efficient, as it skips + * non-high-scoring hits. + * <p>Note: The <code>score</code> passed to this method is a raw score. + * In other words, the score will not necessarily be a float whose value is + * between 0 and 1. + */ + void _search(Query* query, HitCollector* results) { + Searchable::_search(query, NULL, results); + } + + /** Expert: Set the Similarity implementation used by this Searcher. + * + * @see Similarity#setDefault(Similarity) + */ + void setSimilarity(Similarity* similarity) { + this->similarity = similarity; + } + + /** Expert: Return the Similarity implementation used by this Searcher. + * + * <p>This defaults to the current value of {@link Similarity#getDefault()}. + */ + Similarity* getSimilarity(){ + return this->similarity; + } + }; + + /** The abstract base class for queries. + <p>Instantiable subclasses are: + <ul> + <li> {@link TermQuery} + <li> {@link MultiTermQuery} + <li> {@link BooleanQuery} + <li> {@link WildcardQuery} + <li> {@link PhraseQuery} + <li> {@link PrefixQuery} + <li> {@link PhrasePrefixQuery} + <li> {@link FuzzyQuery} + <li> {@link RangeQuery} + <li> {@link spans.SpanQuery} + </ul> + <p>A parser for queries is contained in: + <ul> + <li>{@link queryParser.QueryParser QueryParser} + </ul> + */ + class Query :LUCENE_BASE { + private: + // query boost factor + qreal boost; + protected: + Query(const Query& clone); + public: + Query(); + virtual ~Query(); + + /** Sets the boost for this query clause to <code>b</code>. Documents + * matching this clause will (in addition to the normal weightings) have + * their score multiplied by <code>b</code>. + */ + void setBoost(qreal b); + + /** Gets the boost for this clause. Documents matching + * this clause will (in addition to the normal weightings) have their score + * multiplied by <code>b</code>. The boost is 1.0 by default. + */ + qreal getBoost() const; + + /** Expert: Constructs an initializes a Weight for a top-level query. */ + Weight* weight(Searcher* searcher); + + /** Expert: called to re-write queries into primitive queries. */ + virtual Query* rewrite(CL_NS(index)::IndexReader* reader); + + /** Expert: called when re-writing queries under MultiSearcher. + * + * <p>Only implemented by derived queries, with no + * {@link #_createWeight(Searcher)} implementatation. + */ + virtual Query* combine(Query** queries); + + /** Expert: merges the clauses of a set of BooleanQuery's into a single + * BooleanQuery. + * + *<p>A utility for use by {@link #combine(Query[])} implementations. + */ + static Query* mergeBooleanQueries(Query** queries); + + /** Expert: Returns the Similarity implementation to be used for this query. + * Subclasses may override this method to specify their own Similarity + * implementation, perhaps one that delegates through that of the Searcher. + * By default the Searcher's Similarity implementation is returned.*/ + Similarity* getSimilarity(Searcher* searcher); + + /** Returns a clone of this query. */ + virtual Query* clone() const = 0; + virtual const TCHAR* getQueryName() const = 0; + bool instanceOf(const TCHAR* other) const; + + /** Prints a query to a string, with <code>field</code> as the default field + * for terms. <p>The representation used is one that is readable by + * {@link queryParser.QueryParser QueryParser} + * (although, if the query was created by the parser, the printed + * representation may not be exactly what was parsed). + */ + virtual TCHAR* toString(const TCHAR* field) const = 0; + + virtual bool equals(Query* other) const = 0; + virtual size_t hashCode() const = 0; + + /** Prints a query to a string. */ + TCHAR* toString() const; + + + /** Expert: Constructs an appropriate Weight implementation for this query. + * + * <p>Only implemented by primitive queries, which re-write to themselves. + * <i>This is an Internal function</i> + */ + virtual Weight* _createWeight(Searcher* searcher); + + }; + + +CL_NS_END +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/search/Similarity.cpp b/src/assistant/3rdparty/clucene/src/CLucene/search/Similarity.cpp new file mode 100644 index 000000000..d33a036e4 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/search/Similarity.cpp @@ -0,0 +1,233 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#include "CLucene/StdHeader.h" +#include "Similarity.h" + +#include "CLucene/index/Term.h" +#include "SearchHeader.h" + +CL_NS_USE(index) +CL_NS_DEF(search) + +#ifdef _CL_HAVE_NO_FLOAT_BYTE + #if defined(_LUCENE_PRAGMA_WARNINGS) + #pragma message ("==================Using fallback float<->byte encodings!!!==================") + #else + #warning "==================Using fallback float<->byte encodings!!!==================" + #endif + + //if the autoconf figured out that we can't do the conversions properly, then + //we fall back on the old, inaccurate way of doing things. + qreal NORM_TABLE[] = { + 0.0,5.820766E-10,6.9849193E-10,8.1490725E-10,9.313226E-10,1.1641532E-9,1.3969839E-9, + 1.6298145E-9,1.8626451E-9,2.3283064E-9,2.7939677E-9,3.259629E-9,3.7252903E-9, + 4.656613E-9,5.5879354E-9,6.519258E-9,7.4505806E-9,9.313226E-9,1.1175871E-8,1.3038516E-8, + 1.4901161E-8,1.8626451E-8,2.2351742E-8,2.6077032E-8,2.9802322E-8,3.7252903E-8,4.4703484E-8, + 5.2154064E-8,5.9604645E-8,7.4505806E-8,8.940697E-8,1.0430813E-7,1.1920929E-7,1.4901161E-7, + 1.7881393E-7,2.0861626E-7,2.3841858E-7,2.9802322E-7,3.5762787E-7,4.172325E-7,4.7683716E-7, + 5.9604645E-7,7.1525574E-7,8.34465E-7,9.536743E-7,1.1920929E-6,1.4305115E-6,1.66893E-6, + 1.9073486E-6,2.3841858E-6,2.861023E-6,3.33786E-6,3.8146973E-6,4.7683716E-6,5.722046E-6, + 6.67572E-6,7.6293945E-6,9.536743E-6,1.1444092E-5,1.335144E-5,1.5258789E-5,1.9073486E-5, + 2.2888184E-5,2.670288E-5,3.0517578E-5,3.8146973E-5,4.5776367E-5,5.340576E-5,6.1035156E-5, + 7.6293945E-5,9.1552734E-5,1.0681152E-4,1.2207031E-4,1.5258789E-4,1.8310547E-4,2.1362305E-4, + 2.4414062E-4,3.0517578E-4,3.6621094E-4,4.272461E-4,4.8828125E-4,6.1035156E-4,7.324219E-4, + 8.544922E-4,9.765625E-4,0.0012207031,0.0014648438,0.0017089844,0.001953125,0.0024414062, + 0.0029296875,0.0034179688,0.00390625,0.0048828125,0.005859375,0.0068359375, + 0.0078125,0.009765625,0.01171875,0.013671875,0.015625,0.01953125,0.0234375, + 0.02734375,0.03125,0.0390625,0.046875,0.0546875,0.0625,0.078125,0.09375,0.109375, + 0.125,0.15625,0.1875,0.21875,0.25,0.3125,0.375,0.4375,0.5,0.625,0.75, + 0.875,1.0,1.25,1.5,1.75,2,2.5,3,3.5,4.0,5.0,6.0,7.0,8.0,10.0,12.0,14.0,16.0,20.0,24.0,28.0,32.0,40.0,48.0,56.0, + 64.0,80.0,96.0,112.0,128.0,160.0,192.0,224.0,256.0,320.0,384.0,448.0,512.0,640.0,768.0,896.0,1024.0,1280.0,1536.0,1792.0, + 2048.0,2560.0,3072.0,3584.0,4096.0,5120.0,6144.0,7168.0,8192.0,10240.0,12288.0,14336.0,16384.0,20480.0,24576.0, + 28672.0,32768.0,40960.0,49152.0,57344.0,65536.0,81920.0,98304.0,114688.0,131072.0,163840.0,196608.0, + 229376.0,262144.0,327680.0,393216.0,458752.0,524288.0,655360.0,786432.0,917504.0,1048576.0,1310720.0, + 1572864.0,1835008.0,2097152.0,2621440.0,3145728.0,3670016.0,4194304.0,5242880.0,6291456.0,7340032.0, + 8388608.0,10485760.0,12582912.0,14680064.0,16777216.0,20971520.0,25165824.0,29360128.0,33554432.0, + 41943040.0,50331648.0,58720256.0,67108864.0,83886080.0,100663296.0,117440512.0,134217728.0, + 167772160.0,201326592.0,234881024.0,268435456.0,335544320.0,402653184.0,469762048.0,536870912.0, + 671088640.0,805306368.0,939524096.0,1073741824.0,1342177280.0,1610612736.0,1879048192.0, + 2147483648.0,2684354560.0,3221225472.0,3758096384.0,4294967296.0,5368709120.0,6442450944.0,7516192768.0 + }; + + qreal Similarity::byteToFloat(uint8_t b) { + return NORM_TABLE[b]; + } + + uint8_t Similarity::floatToByte(qreal f) { + return Similarity::encodeNorm(f); + } + +#else + + /** Cache of decoded bytes. */ + qreal NORM_TABLE[256]; + bool NORM_TABLE_initd=false; + + //float to bits conversion utilities... + union clvalue { + int32_t i; + float f; //must use a float type, else types dont match up + }; + + int32_t floatToIntBits(qreal value) + { + clvalue u; + int32_t e, f; + u.f = (float)value; + e = u.i & 0x7f800000; + f = u.i & 0x007fffff; + + if (e == 0x7f800000 && f != 0) + u.i = 0x7fc00000; + + return u.i; + } + + qreal intBitsToFloat(int32_t bits) + { + clvalue u; + u.i = bits; + return u.f; + } + + + qreal Similarity::byteToFloat(uint8_t b) { + if (b == 0) // zero is a special case + return 0.0f; + int32_t mantissa = b & 7; + int32_t exponent = (b >> 3) & 31; + int32_t bits = ((exponent+(63-15)) << 24) | (mantissa << 21); + return intBitsToFloat(bits); + } + + uint8_t Similarity::floatToByte(qreal f) { + if (f < 0.0f) // round negatives up to zero + f = 0.0f; + + if (f == 0.0f) // zero is a special case + return 0; + + int32_t bits = floatToIntBits(f); // parse qreal into parts + int32_t mantissa = (bits & 0xffffff) >> 21; + int32_t exponent = (((bits >> 24) & 0x7f) - 63) + 15; + + if (exponent > 31) { // overflow: use max value + exponent = 31; + mantissa = 7; + } + + if (exponent < 0) { // underflow: use min value + exponent = 0; + mantissa = 1; + } + + return (uint8_t)((exponent << 3) | mantissa); // pack into a uint8_t + } +#endif + + /** The Similarity implementation used by default. */ + Similarity* _defaultImpl=NULL; + + void Similarity::setDefault(Similarity* similarity) { + _defaultImpl = similarity; + } + + Similarity* Similarity::getDefault() { + if ( _defaultImpl == NULL ){ + _defaultImpl = _CLNEW DefaultSimilarity(); + } + return _defaultImpl; + } + + qreal Similarity::decodeNorm(uint8_t b) { +#ifndef _CL_HAVE_NO_FLOAT_BYTE + if ( !NORM_TABLE_initd ){ + for (int i = 0; i < 256; i++) + NORM_TABLE[i] = byteToFloat(i); + NORM_TABLE_initd=true; + } +#endif + return NORM_TABLE[b]; + } + + uint8_t Similarity::encodeNorm(qreal f) { +#ifdef _CL_HAVE_NO_FLOAT_BYTE + int32_t i=0; + if ( f <= 0 ) + return 0; + + while ( i<256 && f > NORM_TABLE[i] ){ + i++; + } + if ( i == 0 ) + return 0; + else if ( i == 255 && f>NORM_TABLE[255] ) + return 255; + else + return i; +#else + return floatToByte(f); +#endif + } + + + qreal Similarity::idf(Term* term, Searcher* searcher) { + return idf(searcher->docFreq(term), searcher->maxDoc()); + } + + + qreal Similarity::idf(CL_NS(util)::CLVector<Term*>* terms, Searcher* searcher) { + qreal _idf = 0.0f; + for (CL_NS(util)::CLVector<Term*>::iterator i = terms->begin(); i != terms->end(); i++ ) { + _idf += idf((Term*)*i, searcher); + } + return _idf; + } + + Similarity::~Similarity(){ + } + + + + + DefaultSimilarity::DefaultSimilarity(){ + } + DefaultSimilarity::~DefaultSimilarity(){ + } + + qreal DefaultSimilarity::lengthNorm(const TCHAR* fieldName, int32_t numTerms) { + if ( numTerms == 0 ) //prevent div by zero + return 0; + qreal ret = (qreal)(1.0 / sqrt((qreal)numTerms)); + return ret; + } + + qreal DefaultSimilarity::queryNorm(qreal sumOfSquaredWeights) { + if ( sumOfSquaredWeights == 0 ) //prevent div by zero + return 0.0f; + qreal ret = (qreal)(1.0 / sqrt(sumOfSquaredWeights)); + return ret; + } + + qreal DefaultSimilarity::tf(qreal freq) { + return sqrt(freq); + } + + qreal DefaultSimilarity::sloppyFreq(int32_t distance) { + return 1.0f / (distance + 1); + } + + qreal DefaultSimilarity::idf(int32_t docFreq, int32_t numDocs) { + return (qreal)(log(numDocs/(qreal)(docFreq+1)) + 1.0); + } + + qreal DefaultSimilarity::coord(int32_t overlap, int32_t maxOverlap) { + if ( maxOverlap == 0 ) + return 0.0f; + return overlap / (qreal)maxOverlap; + } +CL_NS_END diff --git a/src/assistant/3rdparty/clucene/src/CLucene/search/Similarity.h b/src/assistant/3rdparty/clucene/src/CLucene/search/Similarity.h new file mode 100644 index 000000000..426af6952 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/search/Similarity.h @@ -0,0 +1,268 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _lucene_search_Similarity_ +#define _lucene_search_Similarity_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include "CLucene/index/Term.h" + +CL_NS_DEF(search) + +class Searcher;//save including the searchheader.h +class DefaultSimilarity; + +/** Expert: Scoring API. +* <p>Subclasses implement search scoring. +* +* <p>The score of query <code>q</code> for document <code>d</code> is defined +* in terms of these methods as follows: +* +* <table cellpadding="0" cellspacing="0" border="0"> +* <tr> +* <td valign="middle" align="right" rowspan="2">score(q,d) =<br></td> +* <td valign="middle" align="center"> +* <big><big><big><big><big>Σ</big></big></big></big></big></td> +* <td valign="middle"><small> +* {@link #tf(int32_t) tf}(t in d) * +* {@link #idf(Term,Searcher) idf}(t) * +* {@link Field#getBoost getBoost}(t.field in d) * +* {@link #lengthNorm(TCHAR*,int32_t) lengthNorm}(t.field in d) +* </small></td> +* <td valign="middle" rowspan="2"> * +* {@link #coord(int32_t,int32_t) coord}(q,d) * +* {@link #queryNorm(qreal) queryNorm}(q) +* </td> +* </tr> +* <tr> +* <td valign="top" align="right"> +* <small>t in q</small> +* </td> +* </tr> +* </table> +* +* @see #setDefault(Similarity) +* @see IndexWriter#setSimilarity(Similarity) +* @see Searcher#setSimilarity(Similarity) +*/ +class Similarity:LUCENE_BASE { +public: + virtual ~Similarity(); + + /** Set the default Similarity implementation used by indexing and search + * code. + * + * @see Searcher#setSimilarity(Similarity) + * @see IndexWriter#setSimilarity(Similarity) + */ + static void setDefault(Similarity* similarity); + + /** Return the default Similarity implementation used by indexing and search + * code. + * + * <p>This is initially an instance of {@link DefaultSimilarity}. + * + * @see Searcher#setSimilarity(Similarity) + * @see IndexWriter#setSimilarity(Similarity) + */ + static Similarity* getDefault(); + + /** Encodes a normalization factor for storage in an index. + * + * <p>The encoding uses a five-bit exponent and three-bit mantissa, thus + * representing values from around 7x10^9 to 2x10^-9 with about one + * significant decimal digit of accuracy. Zero is also represented. + * Negative numbers are rounded up to zero. Values too large to represent + * are rounded down to the largest representable value. Positive values too + * small to represent are rounded up to the smallest positive representable + * value. + * + * @see Field#setBoost(qreal) + */ + static uint8_t encodeNorm(qreal f); + + /** Decodes a normalization factor stored in an index. + * @see #encodeNorm(qreal) + */ + static qreal decodeNorm(uint8_t b); + + static uint8_t floatToByte(qreal f); + static qreal byteToFloat(uint8_t b); + + /** Computes a score factor for a phrase. + * + * <p>The default implementation sums the {@link #idf(Term,Searcher)} factor + * for each term in the phrase. + * + * @param terms the terms in the phrase + * @param searcher the document collection being searched + * @return a score factor for the phrase + */ + qreal idf(CL_NS(util)::CLVector<CL_NS(index)::Term*>* terms, Searcher* searcher); + //qreal idf(Term** terms, Searcher* searcher); + + + /** Computes a score factor for a simple term. + * + * <p>The default implementation is:<pre> + * return idf(searcher.docFreq(term), searcher.maxDoc()); + * </pre> + * + * Note that {@link Searcher#maxDoc()} is used instead of + * {@link IndexReader#numDocs()} because it is proportional to + * {@link Searcher#docFreq(Term)} , i.e., when one is inaccurate, + * so is the other, and in the same direction. + * + * @param term the term in question + * @param searcher the document collection being searched + * @return a score factor for the term + */ + qreal idf(CL_NS(index)::Term* term, Searcher* searcher); + + + /** Computes a score factor based on a term or phrase's frequency in a + * document. This value is multiplied by the {@link #idf(Term, Searcher)} + * factor for each term in the query and these products are then summed to + * form the initial score for a document. + * + * <p>Terms and phrases repeated in a document indicate the topic of the + * document, so implementations of this method usually return larger values + * when <code>freq</code> is large, and smaller values when <code>freq</code> + * is small. + * + * <p>The default implementation calls {@link #tf(qreal)}. + * + * @param freq the frequency of a term within a document + * @return a score factor based on a term's within-document frequency + */ + inline qreal tf(int32_t freq){ return tf((qreal)freq); } + + /** Computes the normalization value for a field given the total number of + * terms contained in a field. These values, together with field boosts, are + * stored in an index and multipled into scores for hits on each field by the + * search code. + * + * <p>Matches in longer fields are less precise, so implemenations of this + * method usually return smaller values when <code>numTokens</code> is large, + * and larger values when <code>numTokens</code> is small. + * + * <p>That these values are computed under {@link + * IndexWriter#addDocument(Document)} and stored then using + * {#encodeNorm(qreal)}. Thus they have limited precision, and documents + * must be re-indexed if this method is altered. + * + * @param fieldName the name of the field + * @param numTokens the total number of tokens contained in fields named + * <i>fieldName</i> of <i>doc</i>. + * @return a normalization factor for hits on this field of this document + * + * @see Field#setBoost(qreal) + */ + virtual qreal lengthNorm(const TCHAR* fieldName, int32_t numTokens) = 0; + + /** Computes the normalization value for a query given the sum of the squared + * weights of each of the query terms. This value is then multipled into the + * weight of each query term. + * + * <p>This does not affect ranking, but rather just attempts to make scores + * from different queries comparable. + * + * @param sumOfSquaredWeights the sum of the squares of query term weights + * @return a normalization factor for query weights + */ + virtual qreal queryNorm(qreal sumOfSquaredWeights) = 0; + + /** Computes the amount of a sloppy phrase match, based on an edit distance. + * This value is summed for each sloppy phrase match in a document to form + * the frequency that is passed to {@link #tf(qreal)}. + * + * <p>A phrase match with a small edit distance to a document passage more + * closely matches the document, so implementations of this method usually + * return larger values when the edit distance is small and smaller values + * when it is large. + * + * @see PhraseQuery#setSlop(int32_t) + * @param distance the edit distance of this sloppy phrase match + * @return the frequency increment for this match + */ + virtual qreal sloppyFreq(int32_t distance) = 0; + + /** Computes a score factor based on a term or phrase's frequency in a + * document. This value is multiplied by the {@link #idf(Term, Searcher)} + * factor for each term in the query and these products are then summed to + * form the initial score for a document. + * + * <p>Terms and phrases repeated in a document indicate the topic of the + * document, so implemenations of this method usually return larger values + * when <code>freq</code> is large, and smaller values when <code>freq</code> + * is small. + * + * @param freq the frequency of a term within a document + * @return a score factor based on a term's within-document frequency + */ + virtual qreal tf(qreal freq) = 0; + + /** Computes a score factor based on a term's document frequency (the number + * of documents which contain the term). This value is multiplied by the + * {@link #tf(int32_t)} factor for each term in the query and these products are + * then summed to form the initial score for a document. + * + * <p>Terms that occur in fewer documents are better indicators of topic, so + * implemenations of this method usually return larger values for rare terms, + * and smaller values for common terms. + * + * @param docFreq the number of documents which contain the term + * @param numDocs the total number of documents in the collection + * @return a score factor based on the term's document frequency + */ + virtual qreal idf(int32_t docFreq, int32_t numDocs) = 0; + + /** Computes a score factor based on the fraction of all query terms that a + * document contains. This value is multiplied into scores. + * + * <p>The presence of a large portion of the query terms indicates a better + * match with the query, so implemenations of this method usually return + * larger values when the ratio between these parameters is large and smaller + * values when the ratio between them is small. + * + * @param overlap the number of query terms matched in the document + * @param maxOverlap the total number of terms in the query + * @return a score factor based on term overlap with the query + */ + virtual qreal coord(int32_t overlap, int32_t maxOverlap) = 0; +}; + + +/** Expert: Default scoring implementation. */ +class DefaultSimilarity: public Similarity { +public: + DefaultSimilarity(); + ~DefaultSimilarity(); + + /** Implemented as <code>1/sqrt(numTerms)</code>. */ + qreal lengthNorm(const TCHAR* fieldName, int32_t numTerms); + + /** Implemented as <code>1/sqrt(sumOfSquaredWeights)</code>. */ + qreal queryNorm(qreal sumOfSquaredWeights); + + /** Implemented as <code>sqrt(freq)</code>. */ + inline qreal tf(qreal freq); + + /** Implemented as <code>1 / (distance + 1)</code>. */ + qreal sloppyFreq(int32_t distance); + + /** Implemented as <code>log(numDocs/(docFreq+1)) + 1</code>. */ + qreal idf(int32_t docFreq, int32_t numDocs); + + /** Implemented as <code>overlap / maxOverlap</code>. */ + qreal coord(int32_t overlap, int32_t maxOverlap); +}; + +CL_NS_END +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/search/SloppyPhraseScorer.cpp b/src/assistant/3rdparty/clucene/src/CLucene/search/SloppyPhraseScorer.cpp new file mode 100644 index 000000000..b7683b018 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/search/SloppyPhraseScorer.cpp @@ -0,0 +1,106 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#include "CLucene/StdHeader.h" +#include "SloppyPhraseScorer.h" + +#include "PhraseScorer.h" +#include "CLucene/index/Terms.h" + +CL_NS_USE(index) +CL_NS_DEF(search) + + SloppyPhraseScorer::SloppyPhraseScorer(Weight* weight, CL_NS(index)::TermPositions** tps, + int32_t* positions, Similarity* similarity, + int32_t slop, uint8_t* norms): + PhraseScorer(weight,tps,positions,similarity,norms){ + //Func - Constructor + //Pre - tps != NULL + // tpsLength >= 0 + // n != NULL + //Post - Instance has been created + + CND_PRECONDITION(tps != NULL, "tps is NULL"); + //CND_PRECONDITION(n != NULL, _T("n is NULL")) = checked in PhraseScorer; + + this->slop = slop; + } + + qreal SloppyPhraseScorer::phraseFreq() { + //Func - Returns the freqency of the phrase + //Pre - first != NULL + // last != NULL + // pq != NULL + //Post - The frequency of the phrase has been returned + + CND_PRECONDITION(first != NULL,"first is NULL"); + CND_PRECONDITION(last != NULL,"last is NULL"); + CND_PRECONDITION(pq != NULL,"pq is NULL"); + + //Clear the PhraseQueue pq; + pq->clear(); + + int32_t end = 0; + + //declare iterator + PhrasePositions* pp = NULL; + + // build pq from list + + //Sort the list of PhrasePositions using pq + for (pp = first; pp != NULL; pp = pp->_next) { + //Read the first TermPosition of the current PhrasePositions pp + pp->firstPosition(); + //Check if the position of the pp is bigger than end + if (pp->position > end){ + end = pp->position; + } + //Store the current PhrasePositions pp into the PhraseQueue pp + pq->put(pp); + } + + qreal freq = 0.0f; + + bool done = false; + + do { + //Pop a PhrasePositions pp from the PhraseQueue pp + pp = pq->pop(); + //Get start position + int32_t start = pp->position; + //Get next position + int32_t next = pq->top()->position; + + for (int32_t pos = start; pos <= next; pos = pp->position) { + //advance pp to min window + start = pos; + + if (!pp->nextPosition()) { + //ran out of a term -- done + done = true; + break; + } + } + + //Calculate matchLength + int32_t matchLength = end - start; + //Check if matchLength is smaller than slop + if (matchLength <= slop){ + // penalize longer matches + freq += 1.0 / (matchLength + 1); + } + + if (pp->position > end){ + end = pp->position; + } + + //restore pq + pq->put(pp); + }while (!done); + + return freq; + } +CL_NS_END diff --git a/src/assistant/3rdparty/clucene/src/CLucene/search/SloppyPhraseScorer.h b/src/assistant/3rdparty/clucene/src/CLucene/search/SloppyPhraseScorer.h new file mode 100644 index 000000000..31516e393 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/search/SloppyPhraseScorer.h @@ -0,0 +1,34 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _lucene_search_SloppyPhraseScorer_ +#define _lucene_search_SloppyPhraseScorer_ +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include "PhraseScorer.h" +#include "CLucene/index/Terms.h" + + +CL_NS_DEF(search) + class SloppyPhraseScorer: public PhraseScorer { + private: + int32_t slop; + + public: + SloppyPhraseScorer(Weight* weight, CL_NS(index)::TermPositions** tps, + int32_t* positions, Similarity* similarity, + int32_t slop, uint8_t* norms); + ~SloppyPhraseScorer(){ + } + + protected: + qreal phraseFreq(); + }; +CL_NS_END +#endif + diff --git a/src/assistant/3rdparty/clucene/src/CLucene/search/Sort.cpp b/src/assistant/3rdparty/clucene/src/CLucene/search/Sort.cpp new file mode 100644 index 000000000..5a17a784d --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/search/Sort.cpp @@ -0,0 +1,345 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#include "CLucene/StdHeader.h" +#include "Sort.h" +#include "Compare.h" + +CL_NS_USE(util) +CL_NS_DEF(search) + + + + /** Represents sorting by document score (relevancy). */ + SortField* SortField::FIELD_SCORE = _CLNEW SortField (NULL, DOCSCORE,false); + + /** Represents sorting by document number (index order). */ + SortField* SortField::FIELD_DOC = _CLNEW SortField (NULL, DOC,false); + + + /** Represents sorting by computed relevance. Using this sort criteria + * returns the same results as calling {@link Searcher#search(Query) Searcher#search()} + * without a sort criteria, only with slightly more overhead. */ + Sort* Sort::RELEVANCE = _CLNEW Sort(); + + /** Represents sorting by index order. */ + Sort* Sort::INDEXORDER = _CLNEW Sort (SortField::FIELD_DOC); + + + + + /** Creates a sort by terms in the given field where the type of term value + * is determined dynamically ({@link #AUTO AUTO}). + * @param field Name of field to sort by, cannot be <code>null</code>. + */ + SortField::SortField (const TCHAR* field) { + this->type = AUTO; + this->reverse = false; + this->field = CLStringIntern::intern(field CL_FILELINE); + this->factory = NULL; + } + + /** Creates a sort, possibly in reverse, by terms in the given field where + * the type of term value is determined dynamically ({@link #AUTO AUTO}). + * @param field Name of field to sort by, cannot be <code>null</code>. + * @param reverse True if natural order should be reversed. + + SortField::SortField (const TCHAR* field, bool reverse) { + this->field = CLStringIntern::intern(field CL_FILELINE); + this->reverse = reverse; + this->type = AUTO; + this->factory = NULL; + }*/ + + + /** Creates a sort, possibly in reverse, by terms in the given field with the + * type of term values explicitly given. + * @param field Name of field to sort by. Can be <code>null</code> if + * <code>type</code> is SCORE or DOC. + * @param type Type of values in the terms. + * @param reverse True if natural order should be reversed (default=false). + */ + SortField::SortField (const TCHAR* field, int32_t type, bool reverse) { + this->field = (field != NULL) ? CLStringIntern::intern(field CL_FILELINE) : field; + this->type = type; + this->reverse = reverse; + this->factory = NULL; + } + + SortField::SortField(const SortField& clone){ + this->field = (clone.field != NULL) ? CLStringIntern::intern(clone.field CL_FILELINE) : clone.field; + this->type = clone.type; + this->reverse = clone.reverse; + this->factory = clone.factory; + } + SortField* SortField::clone() const{ + return _CLNEW SortField(*this); + } + + /** Creates a sort by terms in the given field sorted + * according to the given locale. + * @param field Name of field to sort by, cannot be <code>null</code>. + * @param locale Locale of values in the field. + */ + /*SortField::SortField (TCHAR* field, Locale* locale) { + this->field = (field != NULL) ? CLStringIntern::intern(field): field; + this->type = STRING; + this->locale = locale; + }*/ + + /** Creates a sort, possibly in reverse, by terms in the given field sorted + * according to the given locale. + * @param field Name of field to sort by, cannot be <code>null</code>. + * @param locale Locale of values in the field. + */ + /*SortField::SortField (TCHAR* field, Locale* locale, bool reverse) { + this->field = (field != NULL) ? CLStringIntern::intern(field): field; + this->type = STRING; + this->locale = locale; + this->reverse = reverse; + }*/ + + + /** Creates a sort, possibly in reverse, with a custom comparison function. + * @param field Name of field to sort by; cannot be <code>null</code>. + * @param comparator Returns a comparator for sorting hits. + * @param reverse True if natural order should be reversed (default=false). + */ + SortField::SortField (const TCHAR* field, SortComparatorSource* comparator, bool reverse) { + this->field = (field != NULL) ? CLStringIntern::intern(field CL_FILELINE): field; + this->type = CUSTOM; + this->reverse = reverse; + this->factory = comparator; + } + + SortField::~SortField(){ + CLStringIntern::unintern(field); + } + + TCHAR* SortField::toString() const { + CL_NS(util)::StringBuffer buffer; + switch (type) { + case DOCSCORE: buffer.append(_T("<score>")); + break; + + case DOC: buffer.append(_T("<doc>")); + break; + + case CUSTOM: buffer.append (_T("<custom:\"")); + buffer.append( field ); + buffer.append( _T("\": ")); + buffer.append(factory->getName()); + buffer.append(_T(">")); + break; + + default: buffer.append( _T("\"")); + buffer.append( field ); + buffer.append( _T("\"") ); + break; + } + + //if (locale != null) buffer.append ("("+locale+")"); todo: + if (reverse) buffer.appendChar('!'); + + return buffer.toString(); + } + + + + + + + + + + + + + + /** Sorts by computed relevance. This is the same sort criteria as + * calling {@link Searcher#search(Query) Searcher#search()} without a sort criteria, only with + * slightly more overhead. */ + Sort::Sort() { + fields=NULL; + SortField** fields=_CL_NEWARRAY(SortField*,3); + fields[0]=SortField::FIELD_SCORE; + fields[1]=SortField::FIELD_DOC; + fields[2]=NULL; + setSort (fields); + _CLDELETE_ARRAY(fields); + } + + Sort::~Sort(){ + clear(); + } + void Sort::clear(){ + if ( fields != NULL ){ + int32_t i=0; + while ( fields[i] != NULL ){ + if ( fields[i] != SortField::FIELD_SCORE && + fields[i] != SortField::FIELD_DOC ){ + _CLDELETE(fields[i]); + } + i++; + } + _CLDELETE_ARRAY(fields); + } + } + + /** Sorts possibly in reverse by the terms in <code>field</code> then by + * index order (document number). The type of value in <code>field</code> is determined + * automatically. + * @see SortField#AUTO + */ + Sort::Sort (const TCHAR* field, bool reverse) { + this->fields=NULL; + setSort (field, reverse); + } + + + /** Sorts in succession by the terms in each field. + * The type of value in <code>field</code> is determined + * automatically. + * @see SortField#AUTO + */ + Sort::Sort (const TCHAR** fields) { + this->fields=NULL; + setSort (fields); + } + + + /** Sorts by the criteria in the given SortField. */ + Sort::Sort (SortField* field) { + this->fields=NULL; + setSort (field); + } + + + /** Sorts in succession by the criteria in each SortField. */ + Sort::Sort (SortField** fields) { + this->fields=NULL; + setSort (fields); + } + + + /** Sets the sort to the terms in <code>field</code> possibly in reverse, + * then by index order (document number). */ + void Sort::setSort (const TCHAR* field, bool reverse) { + clear(); + fields = _CL_NEWARRAY(SortField*,3); + fields[0] = _CLNEW SortField (field, SortField::AUTO, reverse); + fields[1] = SortField::FIELD_DOC; + fields[2] = NULL; + } + + + /** Sets the sort to the terms in each field in succession. */ + void Sort::setSort (const TCHAR** fieldnames) { + clear(); + + int32_t n = 0; + while ( fieldnames[n] != NULL ) + n++; + + fields = _CL_NEWARRAY(SortField*,n+1); + for (int32_t i = 0; i < n; ++i) { + fields[i] = _CLNEW SortField (fieldnames[i], SortField::AUTO,false); + } + fields[n]=NULL; + } + + + /** Sets the sort to the given criteria. */ + void Sort::setSort (SortField* field) { + clear(); + + this->fields = _CL_NEWARRAY(SortField*,2); + this->fields[0] = field; + this->fields[1] = NULL; + } + + + /** Sets the sort to the given criteria in succession. */ + void Sort::setSort (SortField** fields) { + clear(); + + int n=0; + while ( fields[n] != NULL ) + n++; + this->fields = _CL_NEWARRAY(SortField*,n+1); + for (int i=0;i<n+1;i++) + this->fields[i]=fields[i]; + } + + TCHAR* Sort::toString() const { + CL_NS(util)::StringBuffer buffer; + + int32_t i = 0; + while ( fields[i] != NULL ){ + if (i>0) + buffer.appendChar(','); + + const TCHAR* p = fields[i]->toString(); + buffer.append(p); + _CLDELETE_CARRAY(p); + + i++; + } + + return buffer.toString(); + } + + + + + + ScoreDocComparator* ScoreDocComparator::INDEXORDER = _CLNEW ScoreDocComparators::IndexOrder; + ScoreDocComparator* ScoreDocComparator::RELEVANCE = _CLNEW ScoreDocComparators::Relevance; + + ScoreDocComparator::~ScoreDocComparator(){ + } + + +class ScoreDocComparatorImpl: public ScoreDocComparator{ + Comparable** cachedValues; + FieldCacheAuto* fca; + int32_t cachedValuesLen; +public: + ScoreDocComparatorImpl(FieldCacheAuto* fca){ + this->fca = fca; + if ( fca->contentType != FieldCacheAuto::COMPARABLE_ARRAY ) + _CLTHROWA(CL_ERR_InvalidCast,"Invalid field cache auto type"); + this->cachedValues = fca->comparableArray; + this->cachedValuesLen = fca->contentLen; + } + ~ScoreDocComparatorImpl(){ + } + int32_t compare (struct ScoreDoc* i, struct ScoreDoc* j){ + CND_PRECONDITION(i->doc >= 0 && i->doc < cachedValuesLen, "i->doc out of range") + CND_PRECONDITION(j->doc >= 0 && j->doc < cachedValuesLen, "j->doc out of range") + return cachedValues[i->doc]->compareTo (cachedValues[j->doc]); + } + + CL_NS(util)::Comparable* sortValue (struct ScoreDoc* i){ + CND_PRECONDITION(i->doc >= 0 && i->doc < cachedValuesLen, "i->doc out of range") + return cachedValues[i->doc]; + } + + int32_t sortType(){ + return SortField::CUSTOM; + } +}; + +ScoreDocComparator* SortComparator::newComparator (CL_NS(index)::IndexReader* reader, const TCHAR* fieldname){ + return _CLNEW ScoreDocComparatorImpl(FieldCache::DEFAULT->getCustom (reader, fieldname, this)); +} +SortComparator::SortComparator(){ +} +SortComparator::~SortComparator(){ +} + + +CL_NS_END diff --git a/src/assistant/3rdparty/clucene/src/CLucene/search/Sort.h b/src/assistant/3rdparty/clucene/src/CLucene/search/Sort.h new file mode 100644 index 000000000..cfe96d56c --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/search/Sort.h @@ -0,0 +1,356 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _lucene_search_Sort_ +#define _lucene_search_Sort_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include "CLucene/index/IndexReader.h" +#include "SearchHeader.h" + +CL_NS_DEF(search) + + class SortField; //predefine + class Sort; + +/** + * Expert: Compares two ScoreDoc objects for sorting. + * + */ + class ScoreDocComparator:LUCENE_BASE { + protected: + ScoreDocComparator(){} + public: + virtual ~ScoreDocComparator(); +// CL_NS(util)::Comparable** cachedValues; +// ScoreDocComparator(CL_NS(util)::Comparable** cachedValues); + + /** + * Compares two ScoreDoc objects and returns a result indicating their + * sort order. + * @param i First ScoreDoc + * @param j Second ScoreDoc + * @return <code>-1</code> if <code>i</code> should come before <code>j</code><br><code>1</code> if <code>i</code> should come after <code>j</code><br><code>0</code> if they are equal + * @see java.util.Comparator + */ + virtual int32_t compare (struct ScoreDoc* i, struct ScoreDoc* j) = 0; + + /** + * Returns the value used to sort the given document. The + * object returned must implement the java.io.Serializable + * interface. This is used by multisearchers to determine how to collate results from their searchers. + * @see FieldDoc + * @param i Document + * @return Serializable object + */ + virtual CL_NS(util)::Comparable* sortValue (struct ScoreDoc* i) = 0; + + + /** + * Returns the type of sort. Should return <code>SortField.SCORE</code>, <code>SortField.DOC</code>, <code>SortField.STRING</code>, <code>SortField.INTEGER</code>, + * <code>SortField::FLOAT</code> or <code>SortField.CUSTOM</code>. It is not valid to return <code>SortField.AUTO</code>. + * This is used by multisearchers to determine how to collate results from their searchers. + * @return One of the constants in SortField. + * @see SortField + */ + virtual int32_t sortType() = 0; + + /** Special comparator for sorting hits according to computed relevance (document score). */ + static ScoreDocComparator* RELEVANCE; + + /** Special comparator for sorting hits according to index order (document number). */ + static ScoreDocComparator* INDEXORDER; + }; + +/** + * Expert: returns a comparator for sorting ScoreDocs. + * + */ +class SortComparatorSource:LUCENE_BASE { +public: + virtual ~SortComparatorSource(){ + } + + /** + * return a reference to a string describing the name of the comparator + * this is used in the explanation + */ + virtual TCHAR* getName() = 0; + + virtual size_t hashCode() = 0; + + /** + * Creates a comparator for the field in the given index. + * @param reader Index to create comparator for. + * @param fieldname Field to create comparator for. + * @return Comparator of ScoreDoc objects. + * @throws IOException If an error occurs reading the index. + */ + virtual ScoreDocComparator* newComparator (CL_NS(index)::IndexReader* reader, const TCHAR* fieldname) = 0; +}; + + +/** + * Abstract base class for sorting hits returned by a Query. + * + * <p>This class should only be used if the other SortField + * types (SCORE, DOC, STRING, INT, FLOAT) do not provide an + * adequate sorting. It maintains an internal cache of values which + * could be quite large. The cache is an array of Comparable, + * one for each document in the index. There is a distinct + * Comparable for each unique term in the field - if + * some documents have the same term in the field, the cache + * array will have entries which reference the same Comparable. + * + */ +class SortComparator: public SortComparatorSource { +public: + virtual ScoreDocComparator* newComparator (CL_NS(index)::IndexReader* reader, const TCHAR* fieldname); + + SortComparator(); + virtual ~SortComparator(); + + /** + * Returns an object which, when sorted according to natural order, + * will order the Term values in the correct order. + * <p>For example, if the Terms contained integer values, this method + * would return <code>new Integer(termtext)</code>. Note that this + * might not always be the most efficient implementation - for this + * particular example, a better implementation might be to make a + * ScoreDocLookupComparator that uses an internal lookup table of int. + * @param termtext The textual value of the term. + * @return An object representing <code>termtext</code> that sorts + * according to the natural order of <code>termtext</code>. + * @see Comparable + * @see ScoreDocComparator + */ + virtual CL_NS(util)::Comparable* getComparable (const TCHAR* termtext) = 0; + +}; + + +/** + * Stores information about how to sort documents by terms in an individual + * field. Fields must be indexed in order to sort by them. + * + */ +class SortField:LUCENE_BASE { +private: + const TCHAR* field; + int32_t type; // defaults to determining type dynamically + //Locale* locale; // defaults to "natural order" (no Locale) + bool reverse; // defaults to natural order + SortComparatorSource* factory; + +protected: + SortField (const SortField& clone); +public: + virtual ~SortField(); + + /** Sort by document score (relevancy). Sort values are Float and higher + * values are at the front. + * PORTING: this is the same as SCORE in java, it had to be renamed because + * SCORE is a system macro on some platforms (AIX). + */ + LUCENE_STATIC_CONSTANT(int32_t, DOCSCORE=0); + + /** Sort by document number (index order). Sort values are Integer and lower + * values are at the front. */ + LUCENE_STATIC_CONSTANT(int32_t, DOC=1); + + /** Guess type of sort based on field contents. A regular expression is used + * to look at the first term indexed for the field and determine if it + * represents an integer number, a floating point number, or just arbitrary + * string characters. */ + LUCENE_STATIC_CONSTANT(int32_t, AUTO=2); + + /** Sort using term values as Strings. Sort values are String and lower + * values are at the front. */ + LUCENE_STATIC_CONSTANT(int32_t, STRING=3); + + /** Sort using term values as encoded Integers. Sort values are Integer and + * lower values are at the front. */ + LUCENE_STATIC_CONSTANT(int32_t, INT=4); + + /** Sort using term values as encoded Floats. Sort values are Float and + * lower values are at the front. */ + LUCENE_STATIC_CONSTANT(int32_t, FLOAT=5); + + /** Sort using a custom Comparator. Sort values are any Comparable and + * sorting is done according to natural order. */ + LUCENE_STATIC_CONSTANT(int32_t, CUSTOM=9); + + // IMPLEMENTATION NOTE: the FieldCache.STRING_INDEX is in the same "namespace" + // as the above static int values. Any new values must not have the same value + // as FieldCache.STRING_INDEX. + + /** Represents sorting by document score (relevancy). */ + static SortField* FIELD_SCORE; + + /** Represents sorting by document number (index order). */ + static SortField* FIELD_DOC; + + SortField (const TCHAR* field); + //SortField (const TCHAR* field, bool reverse); + //todo: we cannot make reverse use default field of =false. + //because bool and int are the same type in c, overloading is not possible + SortField (const TCHAR* field, int32_t type, bool reverse); + + /* + SortField (TCHAR* field, Locale* locale) { + SortField (TCHAR* field, Locale* locale, bool reverse);*/ + + SortField (const TCHAR* field, SortComparatorSource* comparator, bool reverse=false); + + /** Returns the name of the field. Could return <code>null</code> + * if the sort is by SCORE or DOC. + * @return Name of field, possibly <code>null</code>. + */ + const TCHAR* getField() const { return field; } + + SortField* clone() const; + + /** Returns the type of contents in the field. + * @return One of the constants SCORE, DOC, AUTO, STRING, INT or FLOAT. + */ + int32_t getType() const { return type; } + + /** Returns the Locale by which term values are interpreted. + * May return <code>null</code> if no Locale was specified. + * @return Locale, or <code>null</code>. + */ + /*Locale getLocale() { + return locale; + }*/ + + /** Returns whether the sort should be reversed. + * @return True if natural order should be reversed. + */ + bool getReverse() const { return reverse; } + + SortComparatorSource* getFactory() const { return factory; } + + TCHAR* toString() const; +}; + + + +/** + * Encapsulates sort criteria for returned hits. + * + * <p>The fields used to determine sort order must be carefully chosen. + * Documents must contain a single term in such a field, + * and the value of the term should indicate the document's relative position in + * a given sort order. The field must be indexed, but should not be tokenized, + * and does not need to be stored (unless you happen to want it back with the + * rest of your document data). In other words: + * + * <dl><dd><code>document.add (new Field ("byNumber", Integer.toString(x), false, true, false));</code> + * </dd></dl> + * + * <p><h3>Valid Types of Values</h3> + * + * <p>There are three possible kinds of term values which may be put into + * sorting fields: Integers, Floats, or Strings. Unless + * {@link SortField SortField} objects are specified, the type of value + * in the field is determined by parsing the first term in the field. + * + * <p>Integer term values should contain only digits and an optional + * preceeding negative sign. Values must be base 10 and in the range + * <code>Integer.MIN_VALUE</code> and <code>Integer.MAX_VALUE</code> inclusive. + * Documents which should appear first in the sort + * should have low value integers, later documents high values + * (i.e. the documents should be numbered <code>1..n</code> where + * <code>1</code> is the first and <code>n</code> the last). + * + * <p>Float term values should conform to values accepted by + * {@link Float Float.valueOf(String)} (except that <code>NaN</code> + * and <code>Infinity</code> are not supported). + * Documents which should appear first in the sort + * should have low values, later documents high values. + * + * <p>String term values can contain any valid String, but should + * not be tokenized. The values are sorted according to their + * {@link Comparable natural order}. Note that using this type + * of term value has higher memory requirements than the other + * two types. + * + * <p><h3>Object Reuse</h3> + * + * <p>One of these objects can be + * used multiple times and the sort order changed between usages. + * + * <p>This class is thread safe. + * + * <p><h3>Memory Usage</h3> + * + * <p>Sorting uses of caches of term values maintained by the + * internal HitQueue(s). The cache is static and contains an integer + * or float array of length <code>IndexReader.maxDoc()</code> for each field + * name for which a sort is performed. In other words, the size of the + * cache in bytes is: + * + * <p><code>4 * IndexReader.maxDoc() * (# of different fields actually used to sort)</code> + * + * <p>For String fields, the cache is larger: in addition to the + * above array, the value of every term in the field is kept in memory. + * If there are many unique terms in the field, this could + * be quite large. + * + * <p>Note that the size of the cache is not affected by how many + * fields are in the index and <i>might</i> be used to sort - only by + * the ones actually used to sort a result set. + * + * <p>The cache is cleared each time a new <code>IndexReader</code> is + * passed in, or if the value returned by <code>maxDoc()</code> + * changes for the current IndexReader. This class is not set up to + * be able to efficiently sort hits from more than one index + * simultaneously. + * + */ +class Sort:LUCENE_BASE { + // internal representation of the sort criteria + SortField** fields; + void clear(); +public: + ~Sort(); + + /** Represents sorting by computed relevance. Using this sort criteria + * returns the same results as calling {@link Searcher#search(Query) Searcher#search()} + * without a sort criteria, only with slightly more overhead. */ + static Sort* RELEVANCE; + + /** Represents sorting by index order. */ + static Sort* INDEXORDER; + + Sort(); + Sort (const TCHAR* field, bool reverse=false); + Sort (const TCHAR** fields); + Sort (SortField* field); + Sort (SortField** fields); + void setSort (const TCHAR* field, bool reverse=false); + void setSort (const TCHAR** fieldnames); + void setSort (SortField* field); + void setSort (SortField** fields); + + TCHAR* toString() const; + + /** + * Representation of the sort criteria. + * @return a pointer to the of SortField array used in this sort criteria + */ + SortField** getSort() const{ return fields; } +}; + + + + + +CL_NS_END +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/search/TermQuery.cpp b/src/assistant/3rdparty/clucene/src/CLucene/search/TermQuery.cpp new file mode 100644 index 000000000..a04c20fec --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/search/TermQuery.cpp @@ -0,0 +1,213 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#include "CLucene/StdHeader.h" +#include "TermQuery.h" + +#include "SearchHeader.h" +#include "Scorer.h" +#include "CLucene/index/Term.h" +#include "TermScorer.h" +#include "CLucene/index/IndexReader.h" +#include "CLucene/util/StringBuffer.h" +#include "CLucene/index/Terms.h" + +CL_NS_USE(index) +CL_NS_DEF(search) + + + /** Constructs a query for the term <code>t</code>. */ + TermQuery::TermQuery(Term* t): + term( _CL_POINTER(t) ) + { + } + TermQuery::TermQuery(const TermQuery& clone): + Query(clone){ + this->term=_CL_POINTER(clone.term); + } + TermQuery::~TermQuery(){ + _CLDECDELETE(term); + } + + Query* TermQuery::clone() const{ + return _CLNEW TermQuery(*this); + } + + const TCHAR* TermQuery::getClassName(){ + return _T("TermQuery"); + } + const TCHAR* TermQuery::getQueryName() const{ + return getClassName(); + } + size_t TermQuery::hashCode() const { + return Similarity::floatToByte(getBoost()) ^ term->hashCode(); + } + + + //added by search highlighter + Term* TermQuery::getTerm(bool pointer) const + { + if ( pointer ) + return _CL_POINTER(term); + else + return term; + } + + + /** Prints a user-readable version of this query. */ + TCHAR* TermQuery::toString(const TCHAR* field) const{ + CL_NS(util)::StringBuffer buffer; + if ( field==NULL || _tcscmp(term->field(),field)!= 0 ) { + buffer.append(term->field()); + buffer.append(_T(":")); + } + buffer.append(term->text()); + if (getBoost() != 1.0f) { + buffer.append(_T("^")); + buffer.appendFloat( getBoost(),1 ); + } + return buffer.toString(); + } + + /** Returns true iff <code>o</code> is equal to this. */ + bool TermQuery::equals(Query* other) const { + if (!(other->instanceOf(TermQuery::getClassName()))) + return false; + + TermQuery* tq = (TermQuery*)other; + return (this->getBoost() == tq->getBoost()) + && this->term->equals(tq->term); + } + + + TermQuery::TermWeight::TermWeight(Searcher* searcher, TermQuery* _this, Term* _term) { + this->_this = _this; + this->_term = _term; + this->searcher = searcher; + value=0; + idf=0; + queryNorm=0; + queryWeight=0; + } + TermQuery::TermWeight::~TermWeight(){ + } + + //return a *new* string describing this object + TCHAR* TermQuery::TermWeight::toString() { + int32_t size=_tcslen(_this->getQueryName()) + 10; + TCHAR* tmp = _CL_NEWARRAY(TCHAR, size);//_tcslen(weight()) + _sntprintf(tmp,size,_T("weight(%s)"),_this->getQueryName()); + return tmp; + } + + qreal TermQuery::TermWeight::sumOfSquaredWeights() { + idf = _this->getSimilarity(searcher)->idf(_term, searcher); // compute idf + queryWeight = idf * _this->getBoost(); // compute query weight + return queryWeight * queryWeight; // square it + } + + void TermQuery::TermWeight::normalize(qreal queryNorm) { + this->queryNorm = queryNorm; + queryWeight *= queryNorm; // normalize query weight + value = queryWeight * idf; // idf for document + } + + Scorer* TermQuery::TermWeight::scorer(IndexReader* reader) { + TermDocs* termDocs = reader->termDocs(_term); + + if (termDocs == NULL) + return NULL; + + return _CLNEW TermScorer(this, termDocs, _this->getSimilarity(searcher), + reader->norms(_term->field())); + } + + void TermQuery::TermWeight::explain(IndexReader* reader, int32_t doc, Explanation* result){ + TCHAR buf[LUCENE_SEARCH_EXPLANATION_DESC_LEN]; + TCHAR* tmp; + + tmp = getQuery()->toString(); + _sntprintf(buf,LUCENE_SEARCH_EXPLANATION_DESC_LEN, + _T("weight(%s in %d), product of:"),tmp,doc); + _CLDELETE_CARRAY(tmp); + result->setDescription(buf); + + _sntprintf(buf,LUCENE_SEARCH_EXPLANATION_DESC_LEN, + _T("idf(docFreq=%d)"), searcher->docFreq(_term) ); + Explanation* idfExpl = _CLNEW Explanation(idf, buf); + + // explain query weight + Explanation* queryExpl = _CLNEW Explanation(); + tmp = getQuery()->toString(); + _sntprintf(buf,LUCENE_SEARCH_EXPLANATION_DESC_LEN, + _T("queryWeight(%s), product of:"), tmp); + _CLDELETE_CARRAY(tmp); + queryExpl->setDescription(buf); + + Explanation* boostExpl = _CLNEW Explanation(_this->getBoost(), _T("boost")); + if (_this->getBoost() != 1.0f) + queryExpl->addDetail(boostExpl); + else + _CLDELETE(boostExpl); + + queryExpl->addDetail(idfExpl->clone()); + + Explanation* queryNormExpl = _CLNEW Explanation(queryNorm,_T("queryNorm")); + queryExpl->addDetail(queryNormExpl); + + queryExpl->setValue(_this->getBoost()* // always 1.0 + idfExpl->getValue() * + queryNormExpl->getValue()); + + // explain field weight + const TCHAR* field = _term->field(); + Explanation* fieldExpl = _CLNEW Explanation(); + + tmp = _term->toString(); + _sntprintf(buf,LUCENE_SEARCH_EXPLANATION_DESC_LEN, + _T("fieldWeight(%s in %d), product of:"),tmp,doc); + _CLDELETE_CARRAY(tmp); + fieldExpl->setDescription(buf); + + Scorer* sc = scorer(reader); + Explanation* tfExpl = _CLNEW Explanation; + sc->explain(doc, tfExpl); + _CLDELETE(sc); + fieldExpl->addDetail(tfExpl); + fieldExpl->addDetail(idfExpl); + + Explanation* fieldNormExpl = _CLNEW Explanation(); + uint8_t* fieldNorms = reader->norms(field); + qreal fieldNorm = + fieldNorms!=NULL ? Similarity::decodeNorm(fieldNorms[doc]) : 0.0f; + fieldNormExpl->setValue(fieldNorm); + + _sntprintf(buf,LUCENE_SEARCH_EXPLANATION_DESC_LEN, + _T("fieldNorm(field=%s, doc=%d)"),field,doc); + fieldNormExpl->setDescription(buf); + fieldExpl->addDetail(fieldNormExpl); + + fieldExpl->setValue(tfExpl->getValue() * + idfExpl->getValue() * + fieldNormExpl->getValue()); + + /*if (queryExpl->getValue() == 1.0f){ + _CLDELETE(result); + return fieldExpl; + }else{*/ + result->addDetail(queryExpl); + result->addDetail(fieldExpl); + + // combine them + result->setValue(queryExpl->getValue() * fieldExpl->getValue()); + //} + } + + Weight* TermQuery::_createWeight(Searcher* searcher) { + return _CLNEW TermWeight(searcher,this,term); + } +CL_NS_END + diff --git a/src/assistant/3rdparty/clucene/src/CLucene/search/TermQuery.h b/src/assistant/3rdparty/clucene/src/CLucene/search/TermQuery.h new file mode 100644 index 000000000..a7dd8039b --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/search/TermQuery.h @@ -0,0 +1,81 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _lucene_search_TermQuery_ +#define _lucene_search_TermQuery_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include "SearchHeader.h" +#include "Scorer.h" +#include "CLucene/index/Term.h" +#include "TermScorer.h" +#include "CLucene/index/IndexReader.h" +#include "CLucene/util/StringBuffer.h" +#include "CLucene/index/Terms.h" + +CL_NS_DEF(search) + + + /** A Query that matches documents containing a term. + This may be combined with other terms with a {@link BooleanQuery}. + */ + class TermQuery: public Query { + private: + CL_NS(index)::Term* term; + + + class TermWeight: public Weight { + private: + Searcher* searcher; + qreal value; + qreal idf; + qreal queryNorm; + qreal queryWeight; + TermQuery* _this; + CL_NS(index)::Term* _term; + + public: + TermWeight(Searcher* searcher, TermQuery* _this, CL_NS(index)::Term* _term); + ~TermWeight(); + TCHAR* toString(); + Query* getQuery() { return (Query*)_this; } + qreal getValue() { return value; } + + qreal sumOfSquaredWeights(); + void normalize(qreal queryNorm); + Scorer* scorer(CL_NS(index)::IndexReader* reader); + void explain(CL_NS(index)::IndexReader* reader, int32_t doc, Explanation* ret); + }; + + protected: + Weight* _createWeight(Searcher* searcher); + TermQuery(const TermQuery& clone); + public: + // Constructs a query for the term <code>t</code>. + TermQuery(CL_NS(index)::Term* t); + ~TermQuery(); + + static const TCHAR* getClassName(); + const TCHAR* getQueryName() const; + + //added by search highlighter + CL_NS(index)::Term* getTerm(bool pointer=true) const; + + // Prints a user-readable version of this query. + TCHAR* toString(const TCHAR* field) const; + + bool equals(Query* other) const; + Query* clone() const; + + /** Returns a hash code value for this object.*/ + size_t hashCode() const; + }; +CL_NS_END +#endif + diff --git a/src/assistant/3rdparty/clucene/src/CLucene/search/TermScorer.cpp b/src/assistant/3rdparty/clucene/src/CLucene/search/TermScorer.cpp new file mode 100644 index 000000000..ddd7f74ed --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/search/TermScorer.cpp @@ -0,0 +1,120 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#include "CLucene/StdHeader.h" +#include "TermScorer.h" + +#include "CLucene/index/Terms.h" +#include "TermQuery.h" + +CL_NS_USE(index) +CL_NS_DEF(search) + + //TermScorer takes TermDocs and delets it when TermScorer is cleaned up + TermScorer::TermScorer(Weight* w, CL_NS(index)::TermDocs* td, + Similarity* similarity,uint8_t* _norms): + Scorer(similarity), + termDocs(td), + norms(_norms), + weight(w), + weightValue(w->getValue()), + _doc(0), + pointer(0), + pointerMax(0) + { + memset(docs,0,32*sizeof(int32_t)); + memset(freqs,0,32*sizeof(int32_t)); + + for (int32_t i = 0; i < LUCENE_SCORE_CACHE_SIZE; i++) + scoreCache[i] = getSimilarity()->tf(i) * weightValue; + } + + TermScorer::~TermScorer(){ + _CLDELETE(termDocs); + } + bool TermScorer::next(){ + pointer++; + if (pointer >= pointerMax) { + pointerMax = termDocs->read(docs, freqs, 32); // refill buffer + if (pointerMax != 0) { + pointer = 0; + } else { + termDocs->close(); // close stream + _doc = LUCENE_INT32_MAX_SHOULDBE; // set to sentinel value + return false; + } + } + _doc = docs[pointer]; + return true; + } + + bool TermScorer::skipTo(int32_t target) { + // first scan in cache + for (pointer++; pointer < pointerMax; pointer++) { + if (docs[pointer] >= target) { + _doc = docs[pointer]; + return true; + } + } + + // not found in cache, seek underlying stream + bool result = termDocs->skipTo(target); + if (result) { + pointerMax = 1; + pointer = 0; + docs[pointer] = _doc = termDocs->doc(); + freqs[pointer] = termDocs->freq(); + } else { + _doc = LUCENE_INT32_MAX_SHOULDBE; + } + return result; + } + + void TermScorer::explain(int32_t doc, Explanation* tfExplanation) { + TermQuery* query = (TermQuery*)weight->getQuery(); + int32_t tf = 0; + while (pointer < pointerMax) { + if (docs[pointer] == doc) + tf = freqs[pointer]; + pointer++; + } + if (tf == 0) { + while (termDocs->next()) { + if (termDocs->doc() == doc) { + tf = termDocs->freq(); + } + } + } + termDocs->close(); + tfExplanation->setValue(getSimilarity()->tf(tf)); + + TCHAR buf[LUCENE_SEARCH_EXPLANATION_DESC_LEN+1]; + TCHAR* termToString = query->getTerm(false)->toString(); + _sntprintf(buf,LUCENE_SEARCH_EXPLANATION_DESC_LEN,_T("tf(termFreq(%s)=%d)"), termToString, tf); + _CLDELETE_CARRAY(termToString); + tfExplanation->setDescription(buf); + } + + TCHAR* TermScorer::toString() { + TCHAR* wb = weight->toString(); + int32_t rl = _tcslen(wb) + 9; //9=_tcslen("scorer(" ")") + 1 + TCHAR* ret = _CL_NEWARRAY(TCHAR,rl); + _sntprintf(ret,rl,_T("scorer(%s)"), wb); + _CLDELETE_ARRAY(wb); + return ret; + } + + qreal TermScorer::score(){ + int32_t f = freqs[pointer]; + qreal raw = // compute tf(f)*weight + f < LUCENE_SCORE_CACHE_SIZE // check cache + ? scoreCache[f] // cache hit + : getSimilarity()->tf(f) * weightValue; // cache miss + + return raw * Similarity::decodeNorm(norms[_doc]); // normalize for field + } + +CL_NS_END diff --git a/src/assistant/3rdparty/clucene/src/CLucene/search/TermScorer.h b/src/assistant/3rdparty/clucene/src/CLucene/search/TermScorer.h new file mode 100644 index 000000000..ccbf5f7ec --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/search/TermScorer.h @@ -0,0 +1,53 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _lucene_search_TermScorer_ +#define _lucene_search_TermScorer_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include "Scorer.h" +#include "CLucene/index/Terms.h" +#include "CLucene/search/Similarity.h" +#include "SearchHeader.h" + +CL_NS_DEF(search) + + class TermScorer: public Scorer { + private: + CL_NS(index)::TermDocs* termDocs; + uint8_t* norms; + Weight* weight; + const qreal weightValue; + int32_t _doc; + + int32_t docs[32]; // buffered doc numbers + int32_t freqs[32]; // buffered term freqs + int32_t pointer; + int32_t pointerMax; + + qreal scoreCache[LUCENE_SCORE_CACHE_SIZE]; + public: + + //TermScorer takes TermDocs and delets it when TermScorer is cleaned up + TermScorer(Weight* weight, CL_NS(index)::TermDocs* td, + Similarity* similarity, uint8_t* _norms); + + ~TermScorer(); + + int32_t doc() const { return _doc; } + + bool next(); + bool skipTo(int32_t target); + void explain(int32_t doc, Explanation* ret); + TCHAR* toString(); + + qreal score(); + }; +CL_NS_END +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/search/WildcardQuery.cpp b/src/assistant/3rdparty/clucene/src/CLucene/search/WildcardQuery.cpp new file mode 100644 index 000000000..9373cef0a --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/search/WildcardQuery.cpp @@ -0,0 +1,147 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#include "CLucene/StdHeader.h" +#include "WildcardQuery.h" +#include "CLucene/util/BitSet.h" + +CL_NS_USE(index) +CL_NS_USE(util) +CL_NS_DEF(search) + + + WildcardQuery::WildcardQuery(Term* term): + MultiTermQuery( term ){ + //Func - Constructor + //Pre - term != NULL + //Post - Instance has been created + + } + + WildcardQuery::~WildcardQuery(){ + //Func - Destructor + //Pre - true + //Post - true + + } + + const TCHAR* WildcardQuery::getQueryName() const{ + //Func - Returns the string "WildcardQuery" + //Pre - true + //Post - The string "WildcardQuery" has been returned + return getClassName(); + } + + const TCHAR* WildcardQuery::getClassName(){ + return _T("WildcardQuery"); + } + + + FilteredTermEnum* WildcardQuery::getEnum(IndexReader* reader) { + return _CLNEW WildcardTermEnum(reader, getTerm(false)); + } + + WildcardQuery::WildcardQuery(const WildcardQuery& clone): + MultiTermQuery(clone) + { + } + + Query* WildcardQuery::clone() const{ + return _CLNEW WildcardQuery(*this); + } + size_t WildcardQuery::hashCode() const{ + //todo: we should give the query a seeding value... but + //need to do it for all hascode functions + return Similarity::floatToByte(getBoost()) ^ getTerm()->hashCode(); + } + bool WildcardQuery::equals(Query* other) const{ + if (!(other->instanceOf(WildcardQuery::getClassName()))) + return false; + + WildcardQuery* tq = (WildcardQuery*)other; + return (this->getBoost() == tq->getBoost()) + && getTerm()->equals(tq->getTerm()); + } + + + + + + + + + + + + +WildcardFilter::WildcardFilter( Term* term ) +{ + this->term = _CL_POINTER(term); +} + +WildcardFilter::~WildcardFilter() +{ + _CLDECDELETE(term); +} + +WildcardFilter::WildcardFilter( const WildcardFilter& copy ) : + term( _CL_POINTER(copy.term) ) +{ +} + +Filter* WildcardFilter::clone() const { + return _CLNEW WildcardFilter(*this ); +} + + +TCHAR* WildcardFilter::toString() +{ + //Instantiate a stringbuffer buffer to store the readable version temporarily + CL_NS(util)::StringBuffer buffer; + //check if field equal to the field of prefix + if( term->field() != NULL ) { + //Append the field of prefix to the buffer + buffer.append(term->field()); + //Append a colon + buffer.append(_T(":") ); + } + //Append the text of the prefix + buffer.append(term->text()); + + //Convert StringBuffer buffer to TCHAR block and return it + return buffer.toString(); +} + + +/** Returns a BitSet with true for documents which should be permitted in +search results, and false for those that should not. */ +BitSet* WildcardFilter::bits( IndexReader* reader ) +{ + BitSet* bts = _CLNEW BitSet( reader->maxDoc() ); + + WildcardTermEnum termEnum (reader, term); + if (termEnum.term(false) == NULL) + return bts; + + TermDocs* termDocs = reader->termDocs(); + try{ + do{ + termDocs->seek(&termEnum); + + while (termDocs->next()) { + bts->set(termDocs->doc()); + } + }while(termEnum.next()); + } _CLFINALLY( + termDocs->close(); + _CLDELETE(termDocs); + termEnum.close(); + ) + + return bts; +} + +CL_NS_END diff --git a/src/assistant/3rdparty/clucene/src/CLucene/search/WildcardQuery.h b/src/assistant/3rdparty/clucene/src/CLucene/search/WildcardQuery.h new file mode 100644 index 000000000..cfc38f648 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/search/WildcardQuery.h @@ -0,0 +1,69 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _lucene_search_WildcardQuery_ +#define _lucene_search_WildcardQuery_ +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include "CLucene/index/IndexReader.h" +#include "CLucene/index/Term.h" +#include "MultiTermQuery.h" +#include "WildcardTermEnum.h" + +CL_NS_DEF(search) + + /** Implements the wildcard search query. Supported wildcards are <code>*</code>, which + * matches any character sequence (including the empty one), and <code>?</code>, + * which matches any single character. Note this query can be slow, as it + * needs to iterate over all terms. In order to prevent extremely slow WildcardQueries, + * a Wildcard term must not start with one of the wildcards <code>*</code> or + * <code>?</code>. + * + * @see WildcardTermEnum + */ + class WildcardQuery: public MultiTermQuery { + protected: + FilteredTermEnum* getEnum(CL_NS(index)::IndexReader* reader); + WildcardQuery(const WildcardQuery& clone); + public: + WildcardQuery(CL_NS(index)::Term* term); + ~WildcardQuery(); + + //Returns the string "WildcardQuery" + const TCHAR* getQueryName() const; + static const TCHAR* getClassName(); + + size_t hashCode() const; + bool equals(Query* other) const; + Query* clone() const; + }; + + + +class WildcardFilter: public Filter +{ +private: + CL_NS(index)::Term* term; +protected: + WildcardFilter( const WildcardFilter& copy ); + +public: + WildcardFilter(CL_NS(index)::Term* term); + ~WildcardFilter(); + + /** Returns a BitSet with true for documents which should be permitted in + search results, and false for those that should not. */ + CL_NS(util)::BitSet* bits( CL_NS(index)::IndexReader* reader ); + + Filter* clone() const; + TCHAR* toString(); +}; + + +CL_NS_END +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/search/WildcardTermEnum.cpp b/src/assistant/3rdparty/clucene/src/CLucene/search/WildcardTermEnum.cpp new file mode 100644 index 000000000..bed9e6e0c --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/search/WildcardTermEnum.cpp @@ -0,0 +1,150 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#include "CLucene/StdHeader.h" +#include "WildcardTermEnum.h" + +CL_NS_USE(index) +CL_NS_DEF(search) + + bool WildcardTermEnum::termCompare(Term* term) { + if ( term!=NULL && __term->field() == term->field() ) { + const TCHAR* searchText = term->text(); + const TCHAR* patternText = __term->text(); + if ( _tcsncmp( searchText, pre, preLen ) == 0 ){ + return wildcardEquals(patternText+preLen, __term->textLength()-preLen, 0, searchText, term->textLength(), preLen); + } + } + _endEnum = true; + return false; + } + + /** Creates new WildcardTermEnum */ + WildcardTermEnum::WildcardTermEnum(IndexReader* reader, Term* term): + FilteredTermEnum(), + __term(_CL_POINTER(term)), + fieldMatch(false), + _endEnum(false) + { + + pre = stringDuplicate(term->text()); + + const TCHAR* sidx = _tcschr( pre, LUCENE_WILDCARDTERMENUM_WILDCARD_STRING ); + const TCHAR* cidx = _tcschr( pre, LUCENE_WILDCARDTERMENUM_WILDCARD_CHAR ); + const TCHAR* tidx = sidx; + if (tidx == NULL) + tidx = cidx; + else if ( cidx && cidx > pre) + tidx = min(sidx, cidx); + CND_PRECONDITION(tidx != NULL, "tidx==NULL"); + int32_t idx = (int32_t)(tidx - pre); + preLen = idx; + CND_PRECONDITION(preLen<term->textLength(), "preLen >= term->textLength()"); + pre[preLen]=0; //trim end + + Term* t = _CLNEW Term(__term, pre); + setEnum( reader->terms(t) ); + _CLDECDELETE(t); + } + + void WildcardTermEnum::close() + { + if ( __term != NULL ){ + FilteredTermEnum::close(); + + _CLDECDELETE(__term); + __term = NULL; + + _CLDELETE_CARRAY( pre ); + } + } + WildcardTermEnum::~WildcardTermEnum() { + close(); + } + + qreal WildcardTermEnum::difference() { + return 1.0f; + } + + bool WildcardTermEnum::endEnum() { + return _endEnum; + } + + bool WildcardTermEnum::wildcardEquals(const TCHAR* pattern, int32_t patternLen, int32_t patternIdx, const TCHAR* str, int32_t strLen, int32_t stringIdx) + { + for (int32_t p = patternIdx; ; ++p) + { + for (int32_t s = stringIdx; ; ++p, ++s) + { + // End of str yet? + bool sEnd = (s >= strLen); + // End of pattern yet? + bool pEnd = (p >= patternLen); + + // If we're looking at the end of the str... + if (sEnd) + { + // Assume the only thing left on the pattern is/are wildcards + bool justWildcardsLeft = true; + + // Current wildcard position + int32_t wildcardSearchPos = p; + // While we haven't found the end of the pattern, + // and haven't encountered any non-wildcard characters + while (wildcardSearchPos < patternLen && justWildcardsLeft) + { + // Check the character at the current position + TCHAR wildchar = pattern[wildcardSearchPos]; + // If it's not a wildcard character, then there is more + // pattern information after this/these wildcards. + + if (wildchar != LUCENE_WILDCARDTERMENUM_WILDCARD_CHAR && + wildchar != LUCENE_WILDCARDTERMENUM_WILDCARD_STRING){ + justWildcardsLeft = false; + }else{ + // to prevent "cat" matches "ca??" + if (wildchar == LUCENE_WILDCARDTERMENUM_WILDCARD_CHAR) + return false; + + wildcardSearchPos++; // Look at the next character + } + } + + // This was a prefix wildcard search, and we've matched, so + // return true. + if (justWildcardsLeft) + return true; + } + + // If we've gone past the end of the str, or the pattern, + // return false. + if (sEnd || pEnd) + break; + + // Match a single character, so continue. + if (pattern[p] == LUCENE_WILDCARDTERMENUM_WILDCARD_CHAR) + continue; + + if (pattern[p] == LUCENE_WILDCARDTERMENUM_WILDCARD_STRING) + { + // Look at the character beyond the '*'. + ++p; + // Examine the str, starting at the last character. + for (int32_t i = strLen; i >= s; --i) + { + if (wildcardEquals(pattern, patternLen, p, str, strLen, i)) + return true; + } + break; + } + if (pattern[p] != str[s]) + break; + } + return false; + } + } + +CL_NS_END diff --git a/src/assistant/3rdparty/clucene/src/CLucene/search/WildcardTermEnum.h b/src/assistant/3rdparty/clucene/src/CLucene/search/WildcardTermEnum.h new file mode 100644 index 000000000..2a0373540 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/search/WildcardTermEnum.h @@ -0,0 +1,67 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _lucene_search_WildcardTermEnum_ +#define _lucene_search_WildcardTermEnum_ +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include "CLucene/index/IndexReader.h" +#include "CLucene/index/Term.h" +#include "CLucene/index/Terms.h" +#include "FilteredTermEnum.h" + +CL_NS_DEF(search) + /** + * Subclass of FilteredTermEnum for enumerating all terms that match the + * specified wildcard filter term-> + * <p> + * Term enumerations are always ordered by term->compareTo(). Each term in + * the enumeration is greater than all that precede it. + */ + class WildcardTermEnum: public FilteredTermEnum { + private: + CL_NS(index)::Term* __term; + TCHAR* pre; + int32_t preLen; + bool fieldMatch; + bool _endEnum; + + /******************************************** + * const TCHAR* equality with support for wildcards + ********************************************/ + + protected: + bool termCompare(CL_NS(index)::Term* term) ; + + public: + + /** + * Creates a new <code>WildcardTermEnum</code>. Passing in a + * {@link Term Term} that does not contain a + * <code>LUCENE_WILDCARDTERMENUM_WILDCARD_STRING</code> or + * <code>LUCENE_WILDCARDTERMENUM_WILDCARD_CHAR</code> will cause an exception to be thrown. + */ + WildcardTermEnum(CL_NS(index)::IndexReader* reader, CL_NS(index)::Term* term); + ~WildcardTermEnum(); + + qreal difference() ; + + bool endEnum() ; + + /** + * Determines if a word matches a wildcard pattern. + */ + static bool wildcardEquals(const TCHAR* pattern, int32_t patternLen, int32_t patternIdx, const TCHAR* str, int32_t strLen, int32_t stringIdx); + + void close(); + + const char* getObjectName(){ return WildcardTermEnum::getClassName(); } + static const char* getClassName(){ return "WildcardTermEnum"; } + }; +CL_NS_END +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/store/Directory.h b/src/assistant/3rdparty/clucene/src/CLucene/store/Directory.h new file mode 100644 index 000000000..818bc7af9 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/store/Directory.h @@ -0,0 +1,108 @@ +/* + * Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team + * + * Distributable under the terms of either the Apache License (Version 2.0) or + * the GNU Lesser General Public License, as specified in the COPYING file. + * + * Changes are Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies). +*/ +#ifndef _lucene_store_Directory +#define _lucene_store_Directory + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include <QtCore/QStringList> + +#include "CLucene/store/Lock.h" +#include "CLucene/util/VoidList.h" +#include "CLucene/util/Misc.h" + +#include "IndexInput.h" +#include "IndexOutput.h" + +CL_NS_DEF(store) + +/** A Directory is a flat list of files. Files may be written once, when they +* are created. Once a file is created it may only be opened for read, or +* deleted. Random access is permitted both when reading and writing. +* +* <p> Direct i/o is not used directly, but rather all i/o is +* through this API. This permits things such as: <ul> +* <li> implementation of RAM-based indices; +* <li> implementation indices stored in a database, via a database; +* <li> implementation of an index as a single file; +* </ul> +* +*/ +class Directory : LUCENE_REFBASE +{ +protected: + Directory() {} + // Removes an existing file in the directory. + virtual bool doDeleteFile(const QString& name) = 0; + +public: + DEFINE_MUTEX(THIS_LOCK) + + virtual ~Directory() {}; + + // Returns an array of strings, one for each file in the directory. + virtual QStringList list() const = 0; + + // Returns true iff a file with the given name exists. + virtual bool fileExists(const QString& name) const = 0; + + // Returns the time the named file was last modified. + virtual int64_t fileModified(const QString& name) const = 0; + + // Returns the length of a file in the directory. + virtual int64_t fileLength(const QString& name) const = 0; + + // Returns a stream reading an existing file. + virtual IndexInput* openInput(const QString& name) = 0; + virtual IndexInput* openInput(const QString& name, int32_t bufferSize) + { + // didnt overload bufferSize + return openInput(name); + } + + // Set the modified time of an existing file to now. + virtual void touchFile(const QString& name) = 0; + + // Removes an existing file in the directory. + virtual bool deleteFile(const QString& name, const bool throwError = true) { + bool ret = doDeleteFile(name); + if (!ret && throwError) { + char buffer[200]; + _snprintf(buffer, 200, "couldn't delete file %s", + name.toLocal8Bit().constData()); + _CLTHROWA(CL_ERR_IO, buffer); + } + return ret; + } + + // Renames an existing file in the directory. + // If a file already exists with the new name, then it is replaced. + virtual void renameFile(const QString& from, const QString& to) = 0; + + // Creates a new, empty file in the directory with the given name. + // Returns a stream writing this file. + virtual IndexOutput* createOutput(const QString& name) = 0; + + // Construct a {@link Lock}. + // @param name the name of the lock file + virtual LuceneLock* makeLock(const QString& name) = 0; + + // Closes the store. + virtual void close() = 0; + + virtual QString toString() const = 0; + + virtual QString getDirectoryType() const = 0; +}; + +CL_NS_END + +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/store/FSDirectory.cpp b/src/assistant/3rdparty/clucene/src/CLucene/store/FSDirectory.cpp new file mode 100644 index 000000000..5f96e91cd --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/store/FSDirectory.cpp @@ -0,0 +1,662 @@ +/* + * Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team + * + * Distributable under the terms of either the Apache License (Version 2.0) or + * the GNU Lesser General Public License, as specified in the COPYING file. + * + * Changes are Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies). +*/ +#include <QtCore/QDir> +#include <QtCore/QDateTime> +#include <QtCore/QFileInfo> +#include <QtCore/QByteArray> +#include <QtCore/QCryptographicHash> + +#include "CLucene/StdHeader.h" +#include "FSDirectory.h" +#include "CLucene/index/IndexReader.h" +#include "CLucene/util/Misc.h" +#include "CLucene/debug/condition.h" + +CL_NS_DEF(store) +CL_NS_USE(util) + +bool FSDirectory::disableLocks = false; + +// This cache of directories ensures that there is a unique Directory instance +// per path, so that synchronization on the Directory can be used to synchronize +// access between readers and writers. +static CL_NS(util)::CLHashMap<QString, FSDirectory*, + CL_NS(util)::Compare::Qstring, CL_NS(util)::Equals::Qstring, + CL_NS(util)::Deletor::DummyQString> DIRECTORIES(false, false); + +// # pragma mark -- FSDirectory::FSLock + +FSDirectory::FSLock::FSLock(const QString& _lockDir, const QString& name) + : lockDir(_lockDir) + , lockFile(_lockDir + QDir::separator() + name) +{ +} + +FSDirectory::FSLock::~FSLock() +{ +} + +bool FSDirectory::FSLock::obtain() +{ + if (disableLocks) + return true; + + if (QFile::exists(lockFile)) + return false; + + QDir dir(lockDir); + if (!dir.exists()) { + if (!dir.mkpath(lockDir)) { + // 34: len of "Couldn't create lock directory: " + char* err = _CL_NEWARRAY( + char, 34 + strlen(lockDir.toLocal8Bit().constData()) + 1); + strcpy(err, "Couldn't create lock directory: "); + strcat(err, lockDir.toLocal8Bit().constData()); + _CLTHROWA_DEL(CL_ERR_IO, err); + } + } + + QFile file(lockFile); + return file.open(QIODevice::ReadWrite); +} + +void FSDirectory::FSLock::release() +{ + if (disableLocks) + return; + + QFile file(lockFile); + file.remove(); +} + +bool FSDirectory::FSLock::isLocked() +{ + if (disableLocks) + return false; + return QFile::exists(lockFile); +} + +QString FSDirectory::FSLock::toString() const +{ + QString ret(QLatin1String("Lock@")); + return ret.append(lockFile); +} + +// # pragma mark -- FSDirectory::FSIndexInput + +FSDirectory::FSIndexInput::FSIndexInput(const QString& path, int32_t bufferSize) + : BufferedIndexInput(bufferSize) +{ + CND_PRECONDITION(!path.isEmpty(), "path is NULL"); + + handle = _CLNEW SharedHandle(); + handle->fhandle.setFileName(path); + handle->fhandle.open(QIODevice::ReadOnly); + + if (handle->fhandle.error() != QFile::NoError) { + switch(handle->fhandle.error()) { + case 1: + _CLTHROWA(CL_ERR_IO, "An error occurred when reading from the file"); + break; + case 2: + _CLTHROWA(CL_ERR_IO, "An error occurred when writing to the file."); + break; + case 5: + _CLTHROWA(CL_ERR_IO, "The file could not be opened."); + break; + case 6: + _CLTHROWA(CL_ERR_IO, "The operation was aborted."); + break; + case 7: + _CLTHROWA(CL_ERR_IO, "A timeout occurred."); + break; + case 8: + _CLTHROWA(CL_ERR_IO, "An unspecified error occurred."); + break; + case 9: + _CLTHROWA(CL_ERR_IO, "The file could not be removed."); + break; + case 10: + _CLTHROWA(CL_ERR_IO, "The file could not be renamed."); + break; + case 11: + _CLTHROWA(CL_ERR_IO, "The position in the file could not be changed."); + break; + case 12: + _CLTHROWA(CL_ERR_IO, "The file could not be resized.e"); + break; + case 13: + _CLTHROWA(CL_ERR_IO, "The file could not be accessed."); + break; + case 14: + _CLTHROWA(CL_ERR_IO, "The file could not be copied."); + break; + case 4: + default: + _CLTHROWA(CL_ERR_IO, "A fatal error occurred."); + } + } + + //Store the file length + handle->_length = handle->fhandle.size(); + handle->_fpos = 0; + this->_pos = 0; +} + +FSDirectory::FSIndexInput::FSIndexInput(const FSIndexInput& other) + : BufferedIndexInput(other) +{ + if (other.handle == NULL) + _CLTHROWA(CL_ERR_NullPointer, "other handle is null"); + + SCOPED_LOCK_MUTEX(*other.handle->THIS_LOCK) + + _pos = other.handle->_fpos; + handle = _CL_POINTER(other.handle); +} + +FSDirectory::FSIndexInput::~FSIndexInput() +{ + FSIndexInput::close(); +} + +void FSDirectory::FSIndexInput::close() +{ + BufferedIndexInput::close(); +#ifdef _LUCENE_THREADMUTEX + if (handle != NULL) { + // Here we have a bit of a problem... We need to lock the handle to + // ensure that we can safely delete the handle... But if we delete the + // handle, then the scoped unlock, won't be able to unlock the mutex... + + // take a reference of the lock object... + _LUCENE_THREADMUTEX* mutex = handle->THIS_LOCK; + //lock the mutex + mutex->lock(); + + // determine if we are about to delete the handle... + bool doUnlock = (handle->__cl_refcount > 1); + // decdelete (deletes if refcount is down to 0) + _CLDECDELETE(handle); + + if (doUnlock) + mutex->unlock(); + else + delete mutex; + } +#else + _CLDECDELETE(handle); +#endif +} + +IndexInput* FSDirectory::FSIndexInput::clone() const +{ + return _CLNEW FSDirectory::FSIndexInput(*this); +} + +void FSDirectory::FSIndexInput::seekInternal(const int64_t position) +{ + CND_PRECONDITION(position >= 0 && position < handle->_length, + "Seeking out of range") + _pos = position; +} + +void FSDirectory::FSIndexInput::readInternal(uint8_t* b, const int32_t len) +{ + SCOPED_LOCK_MUTEX(*handle->THIS_LOCK) + + CND_PRECONDITION(handle != NULL, "shared file handle has closed"); + CND_PRECONDITION(handle->fhandle.isOpen(), "file is not open"); + + if (handle->_fpos != _pos) { + handle->fhandle.seek(_pos); + if (handle->fhandle.pos() != _pos) + _CLTHROWA( CL_ERR_IO, "File IO Seek error"); + handle->_fpos = _pos; + } + + bufferLength = (int32_t)handle->fhandle.read((char*)b, len); + if (bufferLength == 0) + _CLTHROWA(CL_ERR_IO, "read past EOF"); + + if (bufferLength == -1) + _CLTHROWA(CL_ERR_IO, "read error"); + + _pos += bufferLength; + handle->_fpos =_pos; +} + +// # pragma mark -- FSDirectory::FSIndexInput::SharedHandle + +FSDirectory::FSIndexInput::SharedHandle::SharedHandle() + : _fpos(0) + , _length(0) +{ +#ifdef _LUCENE_THREADMUTEX + THIS_LOCK = new _LUCENE_THREADMUTEX; +#endif +} + +FSDirectory::FSIndexInput::SharedHandle::~SharedHandle() +{ + if (fhandle.isOpen()) + fhandle.close(); +} + +// # pragma mark -- FSDirectory::FSIndexOutput + +FSDirectory::FSIndexOutput::FSIndexOutput(const QString& path) +{ + //O_BINARY - Opens file in binary (untranslated) mode + //O_CREAT - Creates and opens new file for writing. Has no effect if file specified by filename exists + //O_RANDOM - Specifies that caching is optimized for, but not restricted to, random access from disk. + //O_WRONLY - Opens file for writing only; + fhandle.setFileName(path); + fhandle.open(QIODevice::ReadWrite | QIODevice::Truncate); + + if (fhandle.error() != QFile::NoError) { + switch(fhandle.error()) { + case 1: + _CLTHROWA(CL_ERR_IO, "An error occurred when reading from the file"); + break; + case 2: + _CLTHROWA(CL_ERR_IO, "An error occurred when writing to the file."); + break; + case 5: + _CLTHROWA(CL_ERR_IO, "The file could not be opened."); + break; + case 6: + _CLTHROWA(CL_ERR_IO, "The operation was aborted."); + break; + case 7: + _CLTHROWA(CL_ERR_IO, "A timeout occurred."); + break; + case 8: + _CLTHROWA(CL_ERR_IO, "An unspecified error occurred."); + break; + case 9: + _CLTHROWA(CL_ERR_IO, "The file could not be removed."); + break; + case 10: + _CLTHROWA(CL_ERR_IO, "The file could not be renamed."); + break; + case 11: + _CLTHROWA(CL_ERR_IO, "The position in the file could not be changed."); + break; + case 12: + _CLTHROWA(CL_ERR_IO, "The file could not be resized.e"); + break; + case 13: + _CLTHROWA(CL_ERR_IO, "The file could not be accessed."); + break; + case 14: + _CLTHROWA(CL_ERR_IO, "The file could not be copied."); + break; + case 4: + default: + _CLTHROWA(CL_ERR_IO, "A fatal error occurred."); + } + } +} + +FSDirectory::FSIndexOutput::~FSIndexOutput() +{ + if (fhandle.isOpen()) { + try { + FSIndexOutput::close(); + } catch (CLuceneError& err) { + //ignore IO errors... + if (err.number() != CL_ERR_IO) + throw; + } + } +} + +void FSDirectory::FSIndexOutput::close() +{ + try { + BufferedIndexOutput::close(); + } catch (CLuceneError& err) { + //ignore IO errors... + if (err.number() != CL_ERR_IO) + throw; + } + fhandle.close(); +} + +int64_t FSDirectory::FSIndexOutput::length() +{ + CND_PRECONDITION(fhandle.isOpen(), "file is not open"); + return fhandle.size(); +} + +void FSDirectory::FSIndexOutput::seek(const int64_t pos) +{ + CND_PRECONDITION(fhandle.isOpen(), "file is not open"); + + BufferedIndexOutput::seek(pos); + fhandle.seek(pos); + if (fhandle.pos() != pos) + _CLTHROWA(CL_ERR_IO, "File IO Seek error"); +} + +void FSDirectory::FSIndexOutput::flushBuffer(const uint8_t* b, const int32_t size) +{ + CND_PRECONDITION(fhandle.isOpen(), "file is not open"); + + if (size > 0 && fhandle.write((const char*)b, size) != size) + _CLTHROWA(CL_ERR_IO, "File IO Write error"); +} + +// # pragma mark -- FSDirectory + +FSDirectory::FSDirectory(const QString& path, const bool createDir) + : Directory() + , refCount(0) + , useMMap(false) +{ + //set a realpath so that if we change directory, we can still function + directory = QFileInfo(path).absoluteFilePath(); + lockDir = directory; + + QDir dir(lockDir); + if (!dir.exists()) { + if (!dir.mkpath(lockDir)) + _CLTHROWA_DEL(CL_ERR_IO, "Cannot create temp directory"); + } + + QFileInfo info(lockDir); + if (info.isFile() || info.isSymLink()) + _CLTHROWA(CL_ERR_IO, "Found regular file where directory expected"); + + if (createDir) + create(); + + dir.setPath(directory); + if (!dir.exists()) { + //19: len of " is not a directory" + char* err = + _CL_NEWARRAY(char, 19 + strlen(path.toLocal8Bit().constData()) + 1); + strcpy(err, path.toLocal8Bit().constData()); + strcat(err, " is not a directory"); + _CLTHROWA_DEL(CL_ERR_IO, err); + } +} + +void FSDirectory::create() +{ + SCOPED_LOCK_MUTEX(THIS_LOCK) + + bool clear = false; + QDir dir(directory); + if (!dir.exists()) { + if (!dir.mkpath(directory)) { + char* err = _CL_NEWARRAY( // 27 len of "Couldn't create directory:" + char, 27 + strlen(directory.toLocal8Bit().constData()) + 1); + strcpy(err, "Couldn't create directory: "); + strcat(err, directory.toLocal8Bit().constData()); + _CLTHROWA_DEL(CL_ERR_IO, err); + } + } else { + clear = true; + } + + QFileInfo info(directory); + if (info.isFile() || info.isSymLink()) { + char tmp[1024]; + _snprintf(tmp, 1024, "%s not a directory", + directory.toLocal8Bit().constData()); + _CLTHROWA(CL_ERR_IO, tmp); + } + + if (clear) { + dir.setPath(directory); + // clear probably existing lucene index files + QStringList fileList = dir.entryList(QDir::Files | QDir::Hidden + | QDir::NoSymLinks); + foreach(const QString file, fileList) { + if (CL_NS(index)::IndexReader::isLuceneFile(file)) { + if (!dir.remove(file)) + _CLTHROWA(CL_ERR_IO, "Couldn't delete file "); + } + } + + // clear probably existing file locks + QFileInfo dirInfo(lockDir); + if (dirInfo.exists() && dirInfo.isReadable() && dirInfo.isWritable() + && !dirInfo.isFile() && !dirInfo.isSymLink()) { + QDir lockDirectory(lockDir); + fileList = dir.entryList(QStringList() << getLockPrefix() + + QLatin1Char('*'), QDir::Files | QDir::Hidden | QDir::NoSymLinks); + + foreach(const QString file, fileList) { + if (!lockDirectory.remove(file)) + _CLTHROWA(CL_ERR_IO, "Couldn't delete file "); + } + } + else { + //todo: richer error: + lockDir.getAbsolutePath()); + _CLTHROWA(CL_ERR_IO, "Cannot read lock directory"); + } + } +} + +void FSDirectory::priv_getFN(QString& buffer, const QString& name) const +{ + buffer.clear(); + buffer.append(directory); + buffer.append(QDir::separator()); + buffer.append(name); +} + +FSDirectory::~FSDirectory() +{ +} + +QStringList FSDirectory::list() const +{ + CND_PRECONDITION(!directory.isEmpty(), "directory is not open"); + + QDir dir(directory); + return dir.entryList(QDir::Files | QDir::Hidden); +} + +bool FSDirectory::fileExists(const QString& name) const +{ + CND_PRECONDITION(!directory.isEmpty(), "directory is not open"); + + QDir dir(directory); + return dir.entryList().contains(name); +} + +QString FSDirectory::getDirName() const +{ + return directory; +} + +//static +FSDirectory* FSDirectory::getDirectory(const QString& file, const bool _create) +{ + FSDirectory* dir = NULL; + { + if (file.isEmpty()) + _CLTHROWA(CL_ERR_IO, "Invalid directory"); + + SCOPED_LOCK_MUTEX(DIRECTORIES.THIS_LOCK) + dir = DIRECTORIES.get(file); + if ( dir == NULL ){ + dir = _CLNEW FSDirectory(file, _create); + DIRECTORIES.put(dir->directory, dir); + } else if (_create) { + dir->create(); + } + + { + SCOPED_LOCK_MUTEX(dir->THIS_LOCK) + dir->refCount++; + } + } + + return _CL_POINTER(dir); +} + +int64_t FSDirectory::fileModified(const QString& name) const +{ + CND_PRECONDITION(!directory.isEmpty(), "directory is not open"); + + QFileInfo fInfo(directory + QDir::separator() + name); + return fInfo.lastModified().toTime_t(); +} + +//static +int64_t FSDirectory::fileModified(const QString& dir, const QString& name) +{ + QFileInfo fInfo(dir + QDir::separator() + name); + return fInfo.lastModified().toTime_t(); +} + +void FSDirectory::touchFile(const QString& name) +{ + CND_PRECONDITION(!directory.isEmpty(), "directory is not open"); + + QFile file(directory + QDir::separator() + name); + if (!file.open(QIODevice::ReadWrite)) + _CLTHROWA(CL_ERR_IO, "IO Error while touching file"); +} + +int64_t FSDirectory::fileLength(const QString& name) const +{ + CND_PRECONDITION(!directory.isEmpty(), "directory is not open"); + + QFileInfo fInfo(directory + QDir::separator() + name); + return fInfo.size(); +} + +IndexInput* FSDirectory::openInput(const QString& name) +{ + return openInput(name, CL_NS(store)::BufferedIndexOutput::BUFFER_SIZE); +} + +IndexInput* FSDirectory::openInput(const QString& name, int32_t bufferSize ) +{ + CND_PRECONDITION(directory[0]!=0,"directory is not open") + + return _CLNEW FSIndexInput(directory + QDir::separator() + name, bufferSize); +} + +void FSDirectory::close() +{ + SCOPED_LOCK_MUTEX(DIRECTORIES.THIS_LOCK) + { + SCOPED_LOCK_MUTEX(THIS_LOCK) + + CND_PRECONDITION(!directory.isEmpty(), "directory is not open"); + + //refcount starts at 1 + if (--refCount <= 0) { + Directory* dir = DIRECTORIES.get(getDirName()); + if (dir) { + //this will be removed in ~FSDirectory + DIRECTORIES.remove(getDirName()); + _CLDECDELETE(dir); + } + } + } +} + +QString FSDirectory::getLockPrefix() const +{ + CND_PRECONDITION(!directory.isEmpty(), "directory is not open"); + + QString dirName(QFileInfo(directory).absoluteFilePath()); + if (dirName.isEmpty()) + _CLTHROWA(CL_ERR_Runtime, "Invalid directory path"); + + // to be compatible with jlucene, + // we need to make some changes ... + if (dirName.at(1) == QLatin1Char(':')) + dirName[0] = dirName.at(0).toUpper(); + + TCHAR tBuffer[2048] = { 0 }; + dirName.toWCharArray(tBuffer); + + char aBuffer[4096] = { 0 }; + STRCPY_TtoA(aBuffer, tBuffer, 4096); + + QString string(QLatin1String("lucene-")); + QByteArray hash(QCryptographicHash::hash(aBuffer, QCryptographicHash::Md5)); + + // TODO: verify this !!! + return string.append(QLatin1String(hash.toHex().constData())); +} + +bool FSDirectory::doDeleteFile(const QString& name) +{ + CND_PRECONDITION(!directory.isEmpty(), "directory is not open"); + + QDir dir(directory); + return dir.remove(name); +} + +void FSDirectory::renameFile(const QString& from, const QString& to) +{ + CND_PRECONDITION(!directory.isEmpty(), "directory is not open"); + SCOPED_LOCK_MUTEX(THIS_LOCK) + + if (fileExists(to)) + deleteFile(to, false); + + QFile file(directory + QDir::separator() + from); + QString newFile(directory + QDir::separator() + to); + if (!file.rename(newFile)) { + // try a second time if we fail + if (fileExists(to)) + deleteFile(to, false); + + if (!file.rename(newFile)) { + QString error(QLatin1String("Could not rename: %1 to %2!!!!")); + error.arg(from).arg(newFile); + QByteArray bArray(error.toLocal8Bit()); + _CLTHROWA(CL_ERR_IO, bArray.constData()); + } + } +} + +IndexOutput* FSDirectory::createOutput(const QString& name) +{ + CND_PRECONDITION(!directory.isEmpty(), "directory is not open"); + + QString file = directory + QDir::separator() + name; + if (QFileInfo(file).exists()) { + if (!QFile::remove(file)) { + QByteArray bArray("Cannot overwrite: "); + bArray.append(name.toLocal8Bit()); + _CLTHROWA(CL_ERR_IO, bArray.constData()); + } + } + return _CLNEW FSIndexOutput(file); +} + +LuceneLock* FSDirectory::makeLock(const QString& name) +{ + CND_PRECONDITION(!directory.isEmpty(), "directory is not open"); + + + QString lockFile(getLockPrefix()); + lockFile.append(QLatin1Char('-')).append(name); + + return _CLNEW FSLock(lockDir, lockFile); +} + +QString FSDirectory::toString() const +{ + return QString::fromLatin1("FSDirectory@").append(directory); +} + +CL_NS_END diff --git a/src/assistant/3rdparty/clucene/src/CLucene/store/FSDirectory.h b/src/assistant/3rdparty/clucene/src/CLucene/store/FSDirectory.h new file mode 100644 index 000000000..e967380e0 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/store/FSDirectory.h @@ -0,0 +1,216 @@ +/* + * Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team + * + * Distributable under the terms of either the Apache License (Version 2.0) or + * the GNU Lesser General Public License, as specified in the COPYING file. + * + * Changes are Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies). +*/ +#ifndef _lucene_store_FSDirectory_ +#define _lucene_store_FSDirectory_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include <QtCore/QFile> +#include <QtCore/QString> +#include <QtCore/QStringList> + +#include "Directory.h" +#include "Lock.h" +#include "CLucene/util/VoidMap.h" +#include "CLucene/util/StringBuffer.h" + +CL_NS_DEF(store) + +/** +* Straightforward implementation of {@link Directory} as a directory of files. +* <p>If the system property 'disableLuceneLocks' has the String value of +* "true", lock creation will be disabled. +* +* @see Directory +*/ +class FSDirectory : public Directory +{ +public: + // Destructor - only call this if you are sure the directory + // is not being used anymore. Otherwise use the ref-counting + // facilities of _CLDECDELETE + ~FSDirectory(); + + // Get a list of strings, one for each file in the directory. + QStringList list() const; + + // Returns true iff a file with the given name exists. + bool fileExists(const QString& name) const; + + // Returns the text name of the directory + QString getDirName() const; ///<returns reference + + /** + Returns the directory instance for the named location. + + Do not delete this instance, only use close, otherwise other instances + will lose this instance. + + <p>Directories are cached, so that, for a given canonical path, the same + FSDirectory instance will always be returned. This permits + synchronization on directories. + + @param file the path to the directory. + @param create if true, create, or erase any existing contents. + @return the FSDirectory for the named file. + */ + static FSDirectory* getDirectory(const QString& file, const bool create); + + // Returns the time the named file was last modified. + int64_t fileModified(const QString& name) const; + + //static + // Returns the time the named file was last modified. + static int64_t fileModified(const QString& dir, const QString& name); + + // static + // Returns the length in bytes of a file in the directory. + int64_t fileLength(const QString& name) const; + + // Returns a stream reading an existing file. + IndexInput* openInput(const QString& name); + IndexInput* openInput(const QString& name, int32_t bufferSize); + + // Renames an existing file in the directory. + void renameFile(const QString& from, const QString& to); + + // Set the modified time of an existing file to now. + void touchFile(const QString& name); + + // Creates a new, empty file in the directory with the given name. + // Returns a stream writing this file. + IndexOutput* createOutput(const QString& name); + + // Construct a {@link Lock}. + // @param name the name of the lock file + LuceneLock* makeLock(const QString& name); + + // Decrease the ref-count to the directory by one. If the object is no + // longer needed, then the object is removed from the directory pool. + void close(); + + // If MMap is available, this can disable use of mmap reading. + void setUseMMap(bool value) { useMMap = value; } + + // Gets whether the directory is using MMap for inputstreams. + bool getUseMMap() const { return useMMap; } + + QString toString() const; + + static QString DirectoryType() { return QLatin1String("FS"); } + QString getDirectoryType() const { return QLatin1String("FS"); } + + // Set whether Lucene's use of lock files is disabled. By default, + // lock files are enabled. They should only be disabled if the index + // is on a read-only medium like a CD-ROM. + static void setDisableLocks(bool doDisableLocks) + { disableLocks = doDisableLocks; } + + // Returns whether Lucene's use of lock files is disabled. + // @return true if locks are disabled, false if locks are enabled. + static bool getDisableLocks() { return disableLocks; } + +protected: + FSDirectory(const QString& path, const bool createDir); + // Removes an existing file in the directory. + bool doDeleteFile(const QString& name); + +private: + class FSLock : public LuceneLock { + public: + FSLock (const QString& lockDir, const QString& name); + ~FSLock(); + + bool obtain(); + void release(); + bool isLocked(); + QString toString() const; + + QString lockDir; + QString lockFile; + }; + friend class FSDirectory::FSLock; + + class FSIndexInput : public BufferedIndexInput { + public: + FSIndexInput(const QString& path, int32_t bufferSize = + CL_NS(store)::BufferedIndexOutput::BUFFER_SIZE); + ~FSIndexInput(); + + void close(); + IndexInput* clone() const; + + int64_t length() + { return handle->_length; } + + QString getDirectoryType() const + { return FSDirectory::DirectoryType(); } + + protected: + FSIndexInput(const FSIndexInput& clone); + // Random-access methods + void seekInternal(const int64_t position); + // IndexInput methods + void readInternal(uint8_t* b, const int32_t len); + + private: + // We used a shared handle between all the fsindexinput clones. + // This reduces number of file handles we need, and it means + // we dont have to use file tell (which is slow) before doing a read. + class SharedHandle : LUCENE_REFBASE { + public: + SharedHandle(); + ~SharedHandle(); + + int64_t _fpos; + int64_t _length; + + QFile fhandle; + DEFINE_MUTEX(*THIS_LOCK) + }; + SharedHandle* handle; + int64_t _pos; + }; + friend class FSDirectory::FSIndexInput; + + class FSIndexOutput : public BufferedIndexOutput { + public: + FSIndexOutput(const QString& path); + ~FSIndexOutput(); + + void close(); + int64_t length(); + void seek(const int64_t pos); + + protected: + void flushBuffer(const uint8_t* b, const int32_t size); + + private: + QFile fhandle; + }; + friend class FSDirectory::FSIndexOutput; + +private: + QString directory; + int refCount; + void create(); + + QString lockDir; + QString getLockPrefix() const; + static bool disableLocks; + + void priv_getFN(QString& buffer, const QString& name) const; + bool useMMap; +}; + +CL_NS_END + +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/store/IndexInput.cpp b/src/assistant/3rdparty/clucene/src/CLucene/store/IndexInput.cpp new file mode 100644 index 000000000..cf7bd16b7 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/store/IndexInput.cpp @@ -0,0 +1,233 @@ + /*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#include "CLucene/StdHeader.h" +#include "IndexInput.h" + +CL_NS_USE(util) +CL_NS_DEF(store) + + IndexInput::IndexInput() + { + } + IndexInput::IndexInput(const IndexInput& other) + { + } + + int32_t IndexInput::readInt() { + int32_t b = (readByte() << 24); + b |= (readByte() << 16); + b |= (readByte() << 8); + return (b | readByte()); + } + + int32_t IndexInput::readVInt() { + uint8_t b = readByte(); + int32_t i = b & 0x7F; + for (int32_t shift = 7; (b & 0x80) != 0; shift += 7) { + b = readByte(); + i |= (b & 0x7F) << shift; + } + return i; + } + + int64_t IndexInput::readLong() { + int64_t i = ((int64_t)readInt() << 32); + return (i | ((int64_t)readInt() & 0xFFFFFFFFL)); + } + + int64_t IndexInput::readVLong() { + uint8_t b = readByte(); + int64_t i = b & 0x7F; + for (int32_t shift = 7; (b & 0x80) != 0; shift += 7) { + b = readByte(); + i |= (((int64_t)b) & 0x7FL) << shift; + } + return i; + } + + void IndexInput::skipChars( const int32_t count) { + for (int32_t i = 0; i < count; i++) { + TCHAR b = readByte(); + if ((b & 0x80) == 0) { + // Do Nothing. + } else if ((b & 0xE0) != 0xE0) { + readByte(); + } else { + readByte(); + readByte(); + } + } + } + + int32_t IndexInput::readString(TCHAR* buffer, const int32_t maxLength){ + int32_t len = readVInt(); + int32_t ml=maxLength-1; + if ( len >= ml ){ + readChars(buffer, 0, ml); + buffer[ml] = 0; + //we have to finish reading all the data for this string! + if ( len-ml > 0 ){ + //seek(getFilePointer()+(len-ml)); <- that was the wrong way to "finish reading" + skipChars(len-ml); + } + return ml; + }else{ + readChars(buffer, 0, len); + buffer[len] = 0; + return len; + } + } + + TCHAR* IndexInput::readString(const bool _unique){ + int32_t len = readVInt(); + + if ( len == 0){ + if ( _unique ) //todo: does non unique ever occur? + return stringDuplicate(LUCENE_BLANK_STRING); + else + return LUCENE_BLANK_STRING; + } + + TCHAR* ret = _CL_NEWARRAY(TCHAR,len+1); + readChars(ret, 0, len); + ret[len] = 0; + + return ret; + } + + void IndexInput::readChars( TCHAR* buffer, const int32_t start, const int32_t len) { + const int32_t end = start + len; + TCHAR b; + for (int32_t i = start; i < end; ++i) { + b = readByte(); + if ((b & 0x80) == 0) { + b = (b & 0x7F); + } else if ((b & 0xE0) != 0xE0) { + b = (((b & 0x1F) << 6) + | (readByte() & 0x3F)); + } else { + b = ((b & 0x0F) << 12) | ((readByte() & 0x3F) << 6); + b |= (readByte() & 0x3F); + } + buffer[i] = b; + } + } + + + + + + +BufferedIndexInput::BufferedIndexInput(int32_t _bufferSize): + buffer(NULL), + bufferSize(_bufferSize), + bufferStart(0), + bufferLength(0), + bufferPosition(0) + { + } + + BufferedIndexInput::BufferedIndexInput(const BufferedIndexInput& other): + IndexInput(other), + buffer(NULL), + bufferSize(other.bufferSize), + bufferStart(other.bufferStart), + bufferLength(other.bufferLength), + bufferPosition(other.bufferPosition) + { + /* DSR: Does the fact that sometime clone.buffer is not NULL even when + ** clone.bufferLength is zero indicate memory corruption/leakage? + ** if ( clone.buffer != NULL) { */ + if (other.bufferLength != 0 && other.buffer != NULL) { + buffer = _CL_NEWARRAY(uint8_t,bufferLength); + memcpy(buffer,other.buffer,bufferLength * sizeof(uint8_t)); + } + } + + void BufferedIndexInput::readBytes(uint8_t* b, const int32_t len){ + if (len < bufferSize) { + for (int32_t i = 0; i < len; ++i) // read byte-by-byte + b[i] = readByte(); + } else { // read all-at-once + int64_t start = getFilePointer(); + seekInternal(start); + readInternal(b, len); + + bufferStart = start + len; // adjust stream variables + bufferPosition = 0; + bufferLength = 0; // trigger refill() on read + } + } + + int64_t BufferedIndexInput::getFilePointer() const{ + return bufferStart + bufferPosition; + } + + void BufferedIndexInput::seek(const int64_t pos) { + if ( pos < 0 ) + _CLTHROWA(CL_ERR_IO, "IO Argument Error. Value must be a positive value."); + if (pos >= bufferStart && pos < (bufferStart + bufferLength)) + bufferPosition = (int32_t)(pos - bufferStart); // seek within buffer + else { + bufferStart = pos; + bufferPosition = 0; + bufferLength = 0; // trigger refill() on read() + seekInternal(pos); + } + } + void BufferedIndexInput::close(){ + _CLDELETE_ARRAY(buffer); + bufferLength = 0; + bufferPosition = 0; + bufferStart = 0; + } + + + BufferedIndexInput::~BufferedIndexInput(){ + BufferedIndexInput::close(); + } + + void BufferedIndexInput::refill() { + int64_t start = bufferStart + bufferPosition; + int64_t end = start + bufferSize; + if (end > length()) // don't read past EOF + end = length(); + bufferLength = (int32_t)(end - start); + if (bufferLength == 0) + _CLTHROWA(CL_ERR_IO, "IndexInput read past EOF"); + + if (buffer == NULL){ + buffer = _CL_NEWARRAY(uint8_t,bufferSize); // allocate buffer lazily + } + readInternal(buffer, bufferLength); + + + bufferStart = start; + bufferPosition = 0; + } + + +IndexInputStream::IndexInputStream(IndexInput* input){ + this->input = input; + this->size = input->length(); + this->position = input->getFilePointer(); +} +IndexInputStream::~IndexInputStream(){ +} +int32_t IndexInputStream::fillBuffer(char* start, int32_t space){ + int64_t avail = input->length()-input->getFilePointer(); + if ( avail == 0 ) + return -1; + else if ( avail<space ) + space = (int32_t)avail; + + input->readBytes((uint8_t*)start,space); + return space; +} + +CL_NS_END + diff --git a/src/assistant/3rdparty/clucene/src/CLucene/store/IndexInput.h b/src/assistant/3rdparty/clucene/src/CLucene/store/IndexInput.h new file mode 100644 index 000000000..9453b5cf1 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/store/IndexInput.h @@ -0,0 +1,190 @@ +/* + * Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team + * + * Distributable under the terms of either the Apache License (Version 2.0) or + * the GNU Lesser General Public License, as specified in the COPYING file. + * + * Changes are Copyright(C) 2007, 2008 by Nokia Corporation and/or its subsidiary(-ies), all rights reserved. +*/ +#ifndef _lucene_store_IndexInput_ +#define _lucene_store_IndexInput_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include <QtCore/QString> + +#include "CLucene/util/bufferedstream.h" +#include "IndexOutput.h" + +CL_NS_DEF(store) + + /** Abstract base class for input from a file in a {@link Directory}. A + * random-access input stream. Used for all Lucene index input operations. + * @see Directory + * @see IndexOutput + */ + class IndexInput: LUCENE_BASE { + private: + void skipChars( const int32_t count); + protected: + IndexInput(); + IndexInput(const IndexInput& clone); + public: + virtual ~IndexInput(){} + virtual IndexInput* clone() const =0; + + DEFINE_MUTEX(THIS_LOCK) + + /** Reads and returns a single byte. + * @see IndexOutput#writeByte(byte) + */ + virtual uint8_t readByte() =0; + + /** Reads a specified number of bytes into an array at the specified offset. + * @param b the array to read bytes into + * @param offset the offset in the array to start storing bytes + * @param len the number of bytes to read + * @see IndexOutput#writeBytes(byte[],int32_t) + */ + virtual void readBytes(uint8_t* b, const int32_t len) =0; + + /** Reads four bytes and returns an int. + * @see IndexOutput#writeInt(int32_t) + */ + int32_t readInt(); + + /** Reads an int stored in variable-length format. Reads between one and + * five bytes. Smaller values take fewer bytes. Negative numbers are not + * supported. + * @see IndexOutput#writeVInt(int32_t) + */ + virtual int32_t readVInt(); + + /** Reads eight bytes and returns a long. + * @see IndexOutput#writeLong(long) + */ + int64_t readLong(); + + /** Reads a long stored in variable-length format. Reads between one and + * nine bytes. Smaller values take fewer bytes. Negative numbers are not + * supported. */ + int64_t readVLong(); + + /** Reads a string. + * @see IndexOutput#writeString(String) + * maxLength is the amount read into the buffer, the whole string is still read from the stream + * returns the amount read + */ + int32_t readString(TCHAR* buffer, const int32_t maxlength); + + /** Reads a string. + * @see IndexOutput#writeString(String) + * If unique is true (default) the string will be duplicated. + * If false and the length is zero, LUCENE_BLANK_STRING is returned + */ + TCHAR* readString(const bool unique=true); + + + /** Reads UTF-8 encoded characters into an array. + * @param buffer the array to read characters into + * @param start the offset in the array to start storing characters + * @param length the number of characters to read + * @see IndexOutput#writeChars(String,int32_t,int32_t) + */ + void readChars( TCHAR* buffer, const int32_t start, const int32_t len); + + /** Closes the stream to futher operations. */ + virtual void close() =0; + + /** Returns the current position in this file, where the next read will + * occur. + * @see #seek(long) + */ + virtual int64_t getFilePointer() const =0; + + /** Sets current position in this file, where the next read will occur. + * @see #getFilePointer() + */ + virtual void seek(const int64_t pos) =0; + + /** The number of bytes in the file. */ + virtual int64_t length() = 0; + + virtual QString getDirectoryType() const = 0; + }; + + /** Abstract base class for input from a file in a {@link Directory}. A + * random-access input stream. Used for all Lucene index input operations. + * @see Directory + * @see IndexOutput + */ + class BufferedIndexInput: public IndexInput{ + private: + uint8_t* buffer; //array of bytes + void refill(); + protected: + int32_t bufferSize; //size of the buffer + int64_t bufferStart; // position in file of buffer + int32_t bufferLength; // end of valid l_byte_ts + int32_t bufferPosition; // next uint8_t to read + + /** Returns a clone of this stream. + * + * <p>Clones of a stream access the same data, and are positioned at the same + * point as the stream they were cloned from. + * + * <p>Expert: Subclasses must ensure that clones may be positioned at + * different points in the input from each other and from the stream they + * were cloned from. + */ + BufferedIndexInput(const BufferedIndexInput& clone); + BufferedIndexInput(int32_t bufferSize = CL_NS(store)::BufferedIndexOutput::BUFFER_SIZE); + public: + + virtual ~BufferedIndexInput(); + virtual IndexInput* clone() const = 0; + void close(); + inline uint8_t readByte(){ + if (bufferPosition >= bufferLength) + refill(); + + return buffer[bufferPosition++]; + } + void readBytes(uint8_t* b, const int32_t len); + int64_t getFilePointer() const; + void seek(const int64_t pos); + + protected: + /** Expert: implements buffer refill. Reads bytes from the current position + * in the input. + * @param b the array to read bytes into + * @param offset the offset in the array to start storing bytes + * @param length the number of bytes to read + */ + virtual void readInternal(uint8_t* b, const int32_t len) = 0; + + /** Expert: implements seek. Sets current position in this file, where the + * next {@link #readInternal(byte[],int32_t,int32_t)} will occur. + * @see #readInternal(byte[],int32_t,int32_t) + */ + virtual void seekInternal(const int64_t pos) = 0; + }; + + /** + * JStream InputStream which reads from an IndexInput. This class is + * used by the FieldReader to create binary fields. You can then use + * a GZipInputStream to read compressed data or any of the other + * JStream stream types. + * + */ + class IndexInputStream: public jstreams::BufferedInputStream<char>{ + IndexInput* input; + public: + IndexInputStream(IndexInput* input); + ~IndexInputStream(); + int32_t fillBuffer(char* start, int32_t space); + }; +CL_NS_END +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/store/IndexOutput.cpp b/src/assistant/3rdparty/clucene/src/CLucene/store/IndexOutput.cpp new file mode 100644 index 000000000..04f78c348 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/store/IndexOutput.cpp @@ -0,0 +1,163 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#include "CLucene/StdHeader.h" +#include "IndexOutput.h" + +CL_NS_USE(util) +CL_NS_DEF(store) + + + IndexOutput::IndexOutput() + { + } + + IndexOutput::~IndexOutput(){ + } + + BufferedIndexOutput::BufferedIndexOutput() + { + buffer = _CL_NEWARRAY(uint8_t, BUFFER_SIZE ); + bufferStart = 0; + bufferPosition = 0; + } + + BufferedIndexOutput::~BufferedIndexOutput(){ + if ( buffer != NULL ) + close(); + } + + void BufferedIndexOutput::close(){ + flush(); + _CLDELETE_ARRAY( buffer ); + + bufferStart = 0; + bufferPosition = 0; + } + + void BufferedIndexOutput::writeByte(const uint8_t b) { + CND_PRECONDITION(buffer!=NULL,"IndexOutput is closed") + if (bufferPosition >= BUFFER_SIZE) + flush(); + buffer[bufferPosition++] = b; + } + + void BufferedIndexOutput::writeBytes(const uint8_t* b, const int32_t length) { + if ( length < 0 ) + _CLTHROWA(CL_ERR_IllegalArgument, "IO Argument Error. Value must be a positive value."); + int32_t bytesLeft = BUFFER_SIZE - bufferPosition; + // is there enough space in the buffer? + if (bytesLeft >= length) { + // we add the data to the end of the buffer + memcpy(buffer + bufferPosition, b, length); + bufferPosition += length; + // if the buffer is full, flush it + if (BUFFER_SIZE - bufferPosition == 0) + flush(); + } else { + // is data larger then buffer? + if (length > BUFFER_SIZE) { + // we flush the buffer + if (bufferPosition > 0) + flush(); + // and write data at once + flushBuffer(b, length); + bufferStart += length; + } else { + // we fill/flush the buffer (until the input is written) + int64_t pos = 0; // position in the input data + int32_t pieceLength; + while (pos < length) { + if ( length - pos < bytesLeft ) + pieceLength = length - pos; + else + pieceLength = bytesLeft; + memcpy(buffer + bufferPosition, b + pos, pieceLength); + pos += pieceLength; + bufferPosition += pieceLength; + // if the buffer is full, flush it + bytesLeft = BUFFER_SIZE - bufferPosition; + if (bytesLeft == 0) { + flush(); + bytesLeft = BUFFER_SIZE; + } + } + } + } + } + + void IndexOutput::writeInt(const int32_t i) { + writeByte((uint8_t)(i >> 24)); + writeByte((uint8_t)(i >> 16)); + writeByte((uint8_t)(i >> 8)); + writeByte((uint8_t) i); + } + + void IndexOutput::writeVInt(const int32_t vi) { + uint32_t i = vi; + while ((i & ~0x7F) != 0) { + writeByte((uint8_t)((i & 0x7f) | 0x80)); + i >>= 7; //doing unsigned shift + } + writeByte( (uint8_t)i ); + } + + void IndexOutput::writeLong(const int64_t i) { + writeInt((int32_t) (i >> 32)); + writeInt((int32_t) i); + } + + void IndexOutput::writeVLong(const int64_t vi) { + uint64_t i = vi; + while ((i & ~0x7F) != 0) { + writeByte((uint8_t)((i & 0x7f) | 0x80)); + i >>= 7; //doing unsigned shift + } + writeByte((uint8_t)i); + } + + void IndexOutput::writeString(const TCHAR* s, const int32_t length ) { + writeVInt(length); + writeChars(s, 0, length); + } + + void IndexOutput::writeChars(const TCHAR* s, const int32_t start, const int32_t length){ + if ( length < 0 || start < 0 ) + _CLTHROWA(CL_ERR_IllegalArgument, "IO Argument Error. Value must be a positive value."); + + const int32_t end = start + length; + for (int32_t i = start; i < end; ++i) { + const int32_t code = (int32_t)s[i]; + if (code >= 0x01 && code <= 0x7F) + writeByte((uint8_t)code); + else if (((code >= 0x80) && (code <= 0x7FF)) || code == 0) { + writeByte((uint8_t)(0xC0 | (code >> 6))); + writeByte((uint8_t)(0x80 | (code & 0x3F))); + } else { + writeByte((uint8_t)(0xE0 | (((uint32_t)code) >> 12))); //unsigned shift + writeByte((uint8_t)(0x80 | ((code >> 6) & 0x3F))); + writeByte((uint8_t)(0x80 | (code & 0x3F))); + } + } + } + + + int64_t BufferedIndexOutput::getFilePointer() const{ + return bufferStart + bufferPosition; + } + + void BufferedIndexOutput::seek(const int64_t pos) { + flush(); + bufferStart = pos; + } + + void BufferedIndexOutput::flush() { + flushBuffer(buffer, bufferPosition); + bufferStart += bufferPosition; + bufferPosition = 0; + } + +CL_NS_END diff --git a/src/assistant/3rdparty/clucene/src/CLucene/store/IndexOutput.h b/src/assistant/3rdparty/clucene/src/CLucene/store/IndexOutput.h new file mode 100644 index 000000000..c47ee73a7 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/store/IndexOutput.h @@ -0,0 +1,152 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _lucene_store_IndexOutput_ +#define _lucene_store_IndexOutput_ +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +CL_NS_DEF(store) + + +/** Abstract class for output to a file in a Directory. A random-access output +* stream. Used for all Lucene index output operations. +* @see Directory +* @see IndexInput +*/ +class IndexOutput:LUCENE_BASE{ + bool isclosed; +public: + IndexOutput(); + virtual ~IndexOutput(); + + /** Writes a single byte. + * @see IndexInput#readByte() + */ + virtual void writeByte(const uint8_t b) = 0; + + /** Writes an array of bytes. + * @param b the bytes to write + * @param length the number of bytes to write + * @see IndexInput#readBytes(byte[],int32_t,int32_t) + */ + virtual void writeBytes(const uint8_t* b, const int32_t length) = 0; + + /** Writes an int as four bytes. + * @see IndexInput#readInt() + */ + void writeInt(const int32_t i); + + /** Writes an int in a variable-length format. Writes between one and + * five bytes. Smaller values take fewer bytes. Negative numbers are not + * supported. + * @see IndexInput#readVInt() + */ + void writeVInt(const int32_t vi); + + /** Writes a long as eight bytes. + * @see IndexInput#readLong() + */ + void writeLong(const int64_t i); + + /** Writes an long in a variable-length format. Writes between one and five + * bytes. Smaller values take fewer bytes. Negative numbers are not + * supported. + * @see IndexInput#readVLong() + */ + void writeVLong(const int64_t vi); + + /** Writes a string. + * @see IndexInput#readString() + */ + void writeString(const TCHAR* s, const int32_t length); + + /** Writes a sequence of UTF-8 encoded characters from a string. + * @param s the source of the characters + * @param start the first character in the sequence + * @param length the number of characters in the sequence + * @see IndexInput#readChars(char[],int32_t,int32_t) + */ + void writeChars(const TCHAR* s, const int32_t start, const int32_t length); + + /** Closes this stream to further operations. */ + virtual void close() = 0; + + /** Returns the current position in this file, where the next write will + * occur. + * @see #seek(long) + */ + virtual int64_t getFilePointer() const = 0; + + /** Sets current position in this file, where the next write will occur. + * @see #getFilePointer() + */ + virtual void seek(const int64_t pos) = 0; + + /** The number of bytes in the file. */ + virtual int64_t length() = 0; + + /** Forces any buffered output to be written. */ + virtual void flush() = 0; +}; + +/** Base implementation class for buffered {@link IndexOutput}. */ +class BufferedIndexOutput : public IndexOutput{ +public: + LUCENE_STATIC_CONSTANT(int32_t, BUFFER_SIZE=LUCENE_STREAM_BUFFER_SIZE); +private: + uint8_t* buffer; + int64_t bufferStart; // position in file of buffer + int32_t bufferPosition; // position in buffer + +public: + BufferedIndexOutput(); + virtual ~BufferedIndexOutput(); + + /** Writes a single byte. + * @see IndexInput#readByte() + */ + virtual void writeByte(const uint8_t b); + + /** Writes an array of bytes. + * @param b the bytes to write + * @param length the number of bytes to write + * @see IndexInput#readBytes(byte[],int32_t,int32_t) + */ + virtual void writeBytes(const uint8_t* b, const int32_t length); + + /** Closes this stream to further operations. */ + virtual void close(); + + /** Returns the current position in this file, where the next write will + * occur. + * @see #seek(long) + */ + int64_t getFilePointer() const; + + /** Sets current position in this file, where the next write will occur. + * @see #getFilePointer() + */ + virtual void seek(const int64_t pos); + + /** The number of bytes in the file. */ + virtual int64_t length() = 0; + + /** Forces any buffered output to be written. */ + void flush(); + +protected: + /** Expert: implements buffer write. Writes bytes at the current position in + * the output. + * @param b the bytes to write + * @param len the number of bytes to write + */ + virtual void flushBuffer(const uint8_t* b, const int32_t len) = 0; +}; + +CL_NS_END +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/store/InputStream.h b/src/assistant/3rdparty/clucene/src/CLucene/store/InputStream.h new file mode 100644 index 000000000..f56819eeb --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/store/InputStream.h @@ -0,0 +1,21 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _lucene_store_InputStream_ +#define _lucene_store_InputStream_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +CL_NS_DEF(store) + +deprecated... please use IndexInput.h header +and change InputStream to IndexInput + + +CL_NS_END +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/store/Lock.cpp b/src/assistant/3rdparty/clucene/src/CLucene/store/Lock.cpp new file mode 100644 index 000000000..a66e784b0 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/store/Lock.cpp @@ -0,0 +1,27 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#include "CLucene/StdHeader.h" +#include "Lock.h" + +CL_NS_DEF(store) + + bool LuceneLock::obtain(int64_t lockWaitTimeout) { + bool locked = obtain(); + int maxSleepCount = (int)(lockWaitTimeout / LOCK_POLL_INTERVAL); + int sleepCount = 0; + while (!locked) { + if (sleepCount++ == maxSleepCount) { + _CLTHROWA(CL_ERR_IO,"Lock obtain timed out"); + } + _LUCENE_SLEEP(LOCK_POLL_INTERVAL); + locked = obtain(); + } + return locked; + } + + +CL_NS_END diff --git a/src/assistant/3rdparty/clucene/src/CLucene/store/Lock.h b/src/assistant/3rdparty/clucene/src/CLucene/store/Lock.h new file mode 100644 index 000000000..b5dda3b06 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/store/Lock.h @@ -0,0 +1,106 @@ +/* + * Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team + * + * Distributable under the terms of either the Apache License (Version 2.0) or + * the GNU Lesser General Public License, as specified in the COPYING file. + * + * Changes are Copyright(C) 2007, 2008 by Nokia Corporation and/or its subsidiary(-ies), all rights reserved. +*/ +#ifndef _lucene_store_Lock_ +#define _lucene_store_Lock_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +CL_NS_DEF(store) + +class LuceneLock : LUCENE_BASE +{ +public: + LUCENE_STATIC_CONSTANT(int64_t, LOCK_POLL_INTERVAL = 1000); + + virtual ~LuceneLock() {} + + // Attempts to obtain exclusive access and immediately return upon success + // or failure. Return true if exclusive access is obtained. + virtual bool obtain() = 0; + + // Attempts to obtain an exclusive lock within amount of time given. + // Currently polls once per second until lockWaitTimeout is passed. + // @param lockWaitTimeout length of time to wait in ms + // @return true if lock was obtained + // @throws IOException if lock wait times out or obtain() throws an IOException + bool obtain(int64_t lockWaitTimeout); + + // Release exclusive access. + virtual void release() = 0; + + // Returns true if the resource is currently locked. Note that one must + // still call {@link #obtain()} before using the resource. + virtual bool isLocked() = 0; + + virtual QString toString() const = 0; +}; + + +// Utility class for executing code with exclusive access. +template<typename T> +class LuceneLockWith +{ +public: + // Constructs an executor that will grab the named lock. Defaults + // lockWaitTimeout to LUCENE_COMMIT_LOCK_TIMEOUT. + // @deprecated Kept only to avoid breaking existing code. + LuceneLockWith(LuceneLock* lock, int64_t lockWaitTimeout) + { + this->lock = lock; + this->lockWaitTimeout = lockWaitTimeout; + } + + virtual ~LuceneLockWith() {} + + // Calls {@link #doBody} while <i>lock</i> is obtained. Blocks if lock + // cannot be obtained immediately. Retries to obtain lock once per second + // until it is obtained, or until it has tried ten times. Lock is released + // when {@link #doBody} exits. + T runAndReturn() + { + bool locked = false; + T ret = NULL; + try { + locked = lock->obtain(lockWaitTimeout); + ret = doBody(); + } _CLFINALLY ( + if (locked) + lock->release(); + ); + return ret; + } + + // @see runAndReturn + // Same as runAndReturn, except doesn't return any value. The only + // difference is that no void values are used + void run() + { + bool locked = false; + try { + locked = lock->obtain(lockWaitTimeout); + doBody(); + } _CLFINALLY ( + if (locked) + lock->release(); + ); + } + +protected: + virtual T doBody() = 0; + +private: + LuceneLock* lock; + int64_t lockWaitTimeout; +}; + +CL_NS_END + +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/store/MMapInput.cpp b/src/assistant/3rdparty/clucene/src/CLucene/store/MMapInput.cpp new file mode 100644 index 000000000..d660032c6 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/store/MMapInput.cpp @@ -0,0 +1,203 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#include "CLucene/StdHeader.h" +#if defined(LUCENE_FS_MMAP) + +#include "FSDirectory.h" +#include "CLucene/util/Misc.h" +#include "CLucene/debug/condition.h" + +#ifndef _CLCOMPILER_MSVC + #include <sys/mman.h> +#endif + +CL_NS_DEF(store) +CL_NS_USE(util) + + FSDirectory::MMapIndexInput::MMapIndexInput(const char* path): + pos(0), + data(NULL), + _length(0), + isClone(false) + { + //Func - Constructor. + // Opens the file named path + //Pre - path != NULL + //Post - if the file could not be opened an exception is thrown. + + CND_PRECONDITION(path != NULL, "path is NULL"); + +#ifdef _CLCOMPILER_MSVC + mmaphandle = NULL; + fhandle = CreateFileA(path,GENERIC_READ,FILE_SHARE_READ, 0,OPEN_EXISTING,0,0); + + //Check if a valid fhandle was retrieved + if (fhandle < 0){ + DWORD err = GetLastError(); + if ( err == ERROR_FILE_NOT_FOUND ) + _CLTHROWA(CL_ERR_IO, "File does not exist"); + else if ( err == EACCES ) + _CLTHROWA(ERROR_ACCESS_DENIED, "File Access denied"); + else if ( err == ERROR_TOO_MANY_OPEN_FILES ) + _CLTHROWA(CL_ERR_IO, "Too many open files"); + else + _CLTHROWA(CL_ERR_IO, "File IO Error"); + } + + DWORD dummy=0; + _length = GetFileSize(fhandle,&dummy); + + if ( _length > 0 ){ + mmaphandle = CreateFileMappingA(fhandle,NULL,PAGE_READONLY,0,0,NULL); + if ( mmaphandle != NULL ){ + void* address = MapViewOfFile(mmaphandle,FILE_MAP_READ,0,0,0); + if ( address != NULL ){ + data = (uint8_t*)address; + return; //SUCCESS! + } + } + CloseHandle(mmaphandle); + + char* lpMsgBuf=0; + DWORD dw = GetLastError(); + + FormatMessageA( + FORMAT_MESSAGE_ALLOCATE_BUFFER | + FORMAT_MESSAGE_FROM_SYSTEM, + NULL, + dw, + MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), + lpMsgBuf, + 0, NULL ); + + char* errstr = _CL_NEWARRAY(char, strlen(lpMsgBuf)+40); + sprintf(errstr, "MMapIndexInput::MMapIndexInput failed with error %d: %s", dw, lpMsgBuf); + LocalFree(lpMsgBuf); + + _CLTHROWA_DEL(CL_ERR_IO,errstr); + } + +#else //_CLCOMPILER_MSVC + fhandle = ::open (path, O_RDONLY); + if (fhandle < 0){ + _CLTHROWA(CL_ERR_IO,strerror(errno)); + }else{ + // stat it + struct stat sb; + if (::fstat (fhandle, &sb)){ + _CLTHROWA(CL_ERR_IO,strerror(errno)); + }else{ + // get length from stat + _length = sb.st_size; + + // mmap the file + void* address = ::mmap(0, _length, PROT_READ, MAP_SHARED, fhandle, 0); + if (address == MAP_FAILED){ + _CLTHROWA(CL_ERR_IO,strerror(errno)); + }else{ + data = (uint8_t*)address; + } + } + } +#endif + } + + FSDirectory::MMapIndexInput::MMapIndexInput(const MMapIndexInput& clone): IndexInput(clone){ + //Func - Constructor + // Uses clone for its initialization + //Pre - clone is a valide instance of FSIndexInput + //Post - The instance has been created and initialized by clone + +#ifdef _CLCOMPILER_MSVC + mmaphandle = NULL; + fhandle = NULL; +#endif + + data = clone.data; + pos = clone.pos; + + //clone the file length + _length = clone._length; + //Keep in mind that this instance is a clone + isClone = true; + } + + uint8_t FSDirectory::MMapIndexInput::readByte(){ + return *(data+(pos++)); + } + + void FSDirectory::MMapIndexInput::readBytes(uint8_t* b, const int32_t len){ + memcpy(b, data+pos, len); + pos+=len; + } + int32_t FSDirectory::MMapIndexInput::readVInt(){ + uint8_t b = *(data+(pos++)); + int32_t i = b & 0x7F; + for (int shift = 7; (b & 0x80) != 0; shift += 7) { + b = *(data+(pos++)); + i |= (b & 0x7F) << shift; + } + return i; + } + int64_t FSDirectory::MMapIndexInput::getFilePointer() const{ + return pos; + } + void FSDirectory::MMapIndexInput::seek(const int64_t pos){ + this->pos=pos; + } + + FSDirectory::MMapIndexInput::~MMapIndexInput(){ + //Func - Destructor + //Pre - True + //Post - The file for which this instance is responsible has been closed. + // The instance has been destroyed + + close(); + } + + IndexInput* FSDirectory::MMapIndexInput::clone() const + { + return _CLNEW FSDirectory::MMapIndexInput(*this); + } + void FSDirectory::MMapIndexInput::close() { + //IndexInput::close(); + + if ( !isClone ){ +#ifdef _CLCOMPILER_MSVC + if ( data != NULL ){ + if ( ! UnmapViewOfFile(data) ){ + CND_PRECONDITION( false, "UnmapViewOfFile(data) failed"); //todo: change to rich error + } + } + + if ( mmaphandle != NULL ){ + if ( ! CloseHandle(mmaphandle) ){ + CND_PRECONDITION( false, "CloseHandle(mmaphandle) failed"); + } + } + if ( fhandle != NULL ){ + if ( !CloseHandle(fhandle) ){ + CND_PRECONDITION( false, "CloseHandle(fhandle) failed"); + } + } + mmaphandle = NULL; + fhandle = NULL; +#else + if ( data != NULL ) + ::munmap(data, _length); + if ( fhandle > 0 ) + ::close(fhandle); + fhandle = 0; +#endif + } + data = NULL; + pos = 0; + } + + +CL_NS_END +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/store/OutputStream.h b/src/assistant/3rdparty/clucene/src/CLucene/store/OutputStream.h new file mode 100644 index 000000000..a82d6718a --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/store/OutputStream.h @@ -0,0 +1,23 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _lucene_store_IndexOutput_ +#define _lucene_store_IndexOutput_ +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +CL_NS_DEF(store) + + +deprecated... please use IndexOutput.h header +and change OutputStream to OutdexInput + + + +CL_NS_END + +#endif // _lucene_store_IndexOutput_ diff --git a/src/assistant/3rdparty/clucene/src/CLucene/store/RAMDirectory.cpp b/src/assistant/3rdparty/clucene/src/CLucene/store/RAMDirectory.cpp new file mode 100644 index 000000000..b0a7c4d64 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/store/RAMDirectory.cpp @@ -0,0 +1,446 @@ +/* + * Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team + * + * Distributable under the terms of either the Apache License (Version 2.0) or + * the GNU Lesser General Public License, as specified in the COPYING file. + * + * Changes are Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies). +*/ +#include "CLucene/StdHeader.h" +#include "RAMDirectory.h" + +#include "Lock.h" +#include "Directory.h" +#include "FSDirectory.h" +#include "CLucene/index/IndexReader.h" +#include "CLucene/util/VoidMap.h" +#include "CLucene/util/Misc.h" +#include "CLucene/debug/condition.h" + +CL_NS_USE(util) +CL_NS_DEF(store) + +RAMFile::RAMFile() +{ + length = 0; + lastModified = Misc::currentTimeMillis(); +} + +RAMFile::~RAMFile() +{ +} + + +RAMDirectory::RAMLock::RAMLock(const QString& name, RAMDirectory* dir) + : directory(dir) +{ + fname = name; +} + +RAMDirectory::RAMLock::~RAMLock() +{ + directory = NULL; +} + +QString RAMDirectory::RAMLock::toString() const +{ + return QLatin1String("LockFile@RAM"); +} + +bool RAMDirectory::RAMLock::isLocked() +{ + return directory->fileExists(fname); +} + +bool RAMDirectory::RAMLock::obtain() +{ + SCOPED_LOCK_MUTEX(directory->files_mutex); + if (!directory->fileExists(fname)) { + IndexOutput* tmp = directory->createOutput(fname); + tmp->close(); + _CLDELETE(tmp); + + return true; + } + return false; +} + +void RAMDirectory::RAMLock::release() +{ + directory->deleteFile(fname); +} + +RAMIndexOutput::~RAMIndexOutput() +{ + if (deleteFile) + _CLDELETE(file); + file = NULL; +} + +RAMIndexOutput::RAMIndexOutput(RAMFile* f) + : file(f) +{ + pointer = 0; + deleteFile = false; +} + +RAMIndexOutput::RAMIndexOutput() + : file(_CLNEW RAMFile) +{ + pointer = 0; + deleteFile = true; +} + +void RAMIndexOutput::writeTo(IndexOutput* out) +{ + flush(); + int64_t end = file->length; + int64_t pos = 0; + int32_t p = 0; + while (pos < end) { + int32_t length = CL_NS(store)::BufferedIndexOutput::BUFFER_SIZE; + int64_t nextPos = pos + length; + if (nextPos > end) { // at the last buffer + length = (int32_t)(end - pos); + } + out->writeBytes((uint8_t*)file->buffers[p++], length); + pos = nextPos; + } +} + +void RAMIndexOutput::reset() +{ + seek(_ILONGLONG(0)); + file->length = _ILONGLONG(0); +} + +void RAMIndexOutput::flushBuffer(const uint8_t* src, const int32_t len) +{ + uint8_t* b = NULL; + int32_t bufferPos = 0; + while (bufferPos != len) { + uint32_t bufferNumber = pointer/CL_NS(store)::BufferedIndexOutput::BUFFER_SIZE; + int32_t bufferOffset = pointer%CL_NS(store)::BufferedIndexOutput::BUFFER_SIZE; + int32_t bytesInBuffer = CL_NS(store)::BufferedIndexOutput::BUFFER_SIZE - bufferOffset; + int32_t remainInSrcBuffer = len - bufferPos; + int32_t bytesToCopy = bytesInBuffer >= remainInSrcBuffer ? remainInSrcBuffer : bytesInBuffer; + + if (bufferNumber == file->buffers.size()){ + b = _CL_NEWARRAY(uint8_t, CL_NS(store)::BufferedIndexOutput::BUFFER_SIZE); + file->buffers.push_back( b ); + }else{ + b = file->buffers[bufferNumber]; + } + memcpy(b+bufferOffset, src+bufferPos, bytesToCopy * sizeof(uint8_t)); + bufferPos += bytesToCopy; + pointer += bytesToCopy; + } + if (pointer > file->length) + file->length = pointer; + + file->lastModified = Misc::currentTimeMillis(); +} + +void RAMIndexOutput::close() +{ + BufferedIndexOutput::close(); +} + +/** Random-at methods */ +void RAMIndexOutput::seek(const int64_t pos) +{ + BufferedIndexOutput::seek(pos); + pointer = (int32_t)pos; +} + +int64_t RAMIndexOutput::length() +{ + return file->length; +} + + +RAMIndexInput::RAMIndexInput(RAMFile* f) + : file(f) +{ + pointer = 0; + _length = f->length; +} + +RAMIndexInput::RAMIndexInput(const RAMIndexInput& other) + : BufferedIndexInput(other) +{ + file = other.file; + pointer = other.pointer; + _length = other._length; +} + +RAMIndexInput::~RAMIndexInput() +{ + RAMIndexInput::close(); +} + +IndexInput* RAMIndexInput::clone() const +{ + return _CLNEW RAMIndexInput(*this); +} + +int64_t RAMIndexInput::length() +{ + return _length; +} + +QString RAMIndexInput::getDirectoryType() const +{ + return RAMDirectory::DirectoryType(); +} + +void RAMIndexInput::readInternal(uint8_t* dest, const int32_t len) +{ + const int64_t bytesAvailable = file->length - pointer; + int64_t remainder = len <= bytesAvailable ? len : bytesAvailable; + int32_t start = pointer; + int32_t destOffset = 0; + while (remainder != 0) { + int32_t bufferNumber = start / CL_NS(store)::BufferedIndexOutput::BUFFER_SIZE; + int32_t bufferOffset = start % CL_NS(store)::BufferedIndexOutput::BUFFER_SIZE; + int32_t bytesInBuffer = CL_NS(store)::BufferedIndexOutput::BUFFER_SIZE - bufferOffset; + + /* The buffer's entire length (bufferLength) is defined by IndexInput.h + ** as int32_t, so obviously the number of bytes in a given segment of the + ** buffer won't exceed the the capacity of int32_t. Therefore, the + ** int64_t->int32_t cast on the next line is safe. */ + int32_t bytesToCopy = bytesInBuffer >= remainder ? static_cast<int32_t>(remainder) : bytesInBuffer; + uint8_t* b = file->buffers[bufferNumber]; + memcpy(dest + destOffset, b + bufferOffset, bytesToCopy * sizeof(uint8_t)); + + destOffset += bytesToCopy; + start += bytesToCopy; + remainder -= bytesToCopy; + pointer += bytesToCopy; + } +} + +void RAMIndexInput::close() +{ + BufferedIndexInput::close(); +} + +void RAMIndexInput::seekInternal(const int64_t pos) +{ + CND_PRECONDITION(pos >= 0 && pos < this->_length, "Seeking out of range") + pointer = (int32_t)pos; +} + +// #pragma mark -- RAMDirectory + +QStringList RAMDirectory::list() const +{ + SCOPED_LOCK_MUTEX(files_mutex); + + QStringList names; + + FileMap::const_iterator itr; + for (itr = files.begin(); itr != files.end(); ++itr) + names.push_back(itr->first); + + return names; +} + +RAMDirectory::RAMDirectory() + : Directory() + , files(false, true) +{ +} + +RAMDirectory::~RAMDirectory() +{ + //todo: should call close directory? +} + +void RAMDirectory::_copyFromDir(Directory* dir, bool closeDir) +{ + QStringList names = dir->list(); + uint8_t buf[CL_NS(store)::BufferedIndexOutput::BUFFER_SIZE]; + + foreach (const QString& name, names) { + if (!CL_NS(index)::IndexReader::isLuceneFile(name)) + continue; + + // make place on ram disk + IndexOutput* os = createOutput(name); + // read current file + IndexInput* is = dir->openInput(name); + + // and copy to ram disk + //todo: this could be a problem when copying from big indexes... + int64_t readCount = 0; + int64_t len = is->length(); + while (readCount < len) { + int32_t toRead = CL_NS(store)::BufferedIndexOutput::BUFFER_SIZE; + if ((readCount + toRead) > len) + toRead = int32_t(len - readCount); + is->readBytes(buf, toRead); + os->writeBytes(buf, toRead); + readCount += toRead; + } + + // graceful cleanup + is->close(); + _CLDELETE(is); + os->close(); + _CLDELETE(os); + } + if (closeDir) + dir->close(); +} + +RAMDirectory::RAMDirectory(Directory* dir) + : Directory() + , files(false, true) +{ + _copyFromDir(dir, false); +} + +RAMDirectory::RAMDirectory(const QString& dir) + : Directory() + , files(false, true) +{ + Directory* fsdir = FSDirectory::getDirectory(dir, false); + try { + _copyFromDir(fsdir, false); + } _CLFINALLY ( + fsdir->close(); + _CLDECDELETE(fsdir); + ); +} + +bool RAMDirectory::fileExists(const QString& name) const +{ + SCOPED_LOCK_MUTEX(files_mutex); + return files.exists(name); +} + +int64_t RAMDirectory::fileModified(const QString& name) const +{ + SCOPED_LOCK_MUTEX(files_mutex); + const RAMFile* f = files.get(name); + return f->lastModified; +} + +int64_t RAMDirectory::fileLength(const QString& name) const +{ + SCOPED_LOCK_MUTEX(files_mutex); + RAMFile* f = files.get(name); + return f->length; +} + + +IndexInput* RAMDirectory::openInput(const QString& name) +{ + SCOPED_LOCK_MUTEX(files_mutex); + RAMFile* file = files.get(name); + if (file == NULL) { + _CLTHROWA(CL_ERR_IO, // DSR:PROPOSED: Better error checking. + "[RAMDirectory::open] The requested file does not exist."); + } + return _CLNEW RAMIndexInput(file); +} + +void RAMDirectory::close() +{ + SCOPED_LOCK_MUTEX(files_mutex); + files.clear(); +} + +bool RAMDirectory::doDeleteFile(const QString& name) +{ + SCOPED_LOCK_MUTEX(files_mutex); + files.remove(name); + return true; +} + +void RAMDirectory::renameFile(const QString& from, const QString& to) +{ + SCOPED_LOCK_MUTEX(files_mutex); + FileMap::iterator itr = files.find(from); + + /* DSR:CL_BUG_LEAK: + ** If a file named $to already existed, its old value was leaked. + ** My inclination would be to prevent this implicit deletion with an + ** exception, but it happens routinely in CLucene's internals (e.g., during + ** IndexWriter.addIndexes with the file named 'segments'). */ + if (files.exists(to)) + files.remove(to); + + if (itr == files.end()) { + char tmp[1024]; + _snprintf(tmp, 1024, "cannot rename %s, file does not exist", + from.toLocal8Bit().constData()); + _CLTHROWT(CL_ERR_IO, tmp); + } + + CND_PRECONDITION(itr != files.end(), "itr == files.end()") + + RAMFile* file = itr->second; + files.removeitr(itr, true, true); + files.put(to, file); +} + + +void RAMDirectory::touchFile(const QString& name) +{ + RAMFile* file = NULL; + { + SCOPED_LOCK_MUTEX(files_mutex); + file = files.get(name); + } + uint64_t ts1 = file->lastModified; + uint64_t ts2 = Misc::currentTimeMillis(); + + //make sure that the time has actually changed + while (ts1 == ts2) { + _LUCENE_SLEEP(1); + ts2 = Misc::currentTimeMillis(); + }; + + file->lastModified = ts2; +} + +IndexOutput* RAMDirectory::createOutput(const QString& name) +{ + /* Check the $files VoidMap to see if there was a previous file named + ** $name. If so, delete the old RAMFile object, but reuse the existing + ** char buffer ($n) that holds the filename. If not, duplicate the + ** supplied filename buffer ($name) and pass ownership of that memory ($n) + ** to $files. */ + + SCOPED_LOCK_MUTEX(files_mutex); + + QString n = files.getKey(name); + if (!n.isEmpty()) { + RAMFile* rf = files.get(name); + _CLDELETE(rf); + } else { + n = name; + } + + RAMFile* file = _CLNEW RAMFile(); +#ifdef _DEBUG + file->filename = n; +#endif + files[n] = file; + + return _CLNEW RAMIndexOutput(file); +} + +LuceneLock* RAMDirectory::makeLock(const QString& name) +{ + return _CLNEW RAMLock(name, this); +} + +QString RAMDirectory::toString() const +{ + return QLatin1String("RAMDirectory"); +} + +CL_NS_END diff --git a/src/assistant/3rdparty/clucene/src/CLucene/store/RAMDirectory.h b/src/assistant/3rdparty/clucene/src/CLucene/store/RAMDirectory.h new file mode 100644 index 000000000..af92e30b2 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/store/RAMDirectory.h @@ -0,0 +1,195 @@ +/* + * Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team + * + * Distributable under the terms of either the Apache License (Version 2.0) or + * the GNU Lesser General Public License, as specified in the COPYING file. + * + * Changes are Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies). +*/ +#ifndef _lucene_store_RAMDirectory_ +#define _lucene_store_RAMDirectory_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include <QtCore/QString> +#include <QtCore/QStringList> + +#include "Lock.h" +#include "Directory.h" +#include "CLucene/util/VoidMap.h" +#include "CLucene/util/Arrays.h" + +CL_NS_DEF(store) + +class RAMFile : LUCENE_BASE +{ +public: + CL_NS(util)::CLVector<uint8_t*, CL_NS(util)::Deletor::Array<uint8_t> > buffers; + int64_t length; + uint64_t lastModified; + +#ifdef _DEBUG + QString filename; +#endif + + RAMFile(); + ~RAMFile(); +}; + +class RAMIndexOutput : public BufferedIndexOutput +{ +protected: + RAMFile* file; + int32_t pointer; + bool deleteFile; + + // output methods: + void flushBuffer(const uint8_t* src, const int32_t len); + +public: + RAMIndexOutput(RAMFile* f); + RAMIndexOutput(); + /** Construct an empty output buffer. */ + virtual ~RAMIndexOutput(); + + virtual void close(); + + // Random-at methods + virtual void seek(const int64_t pos); + int64_t length(); + /** Resets this to an empty buffer. */ + void reset(); + /** Copy the current contents of this buffer to the named output. */ + void writeTo(IndexOutput* output); +}; + +class RAMIndexInput : public BufferedIndexInput +{ +private: + RAMFile* file; + int32_t pointer; + int64_t _length; + +protected: + /** IndexInput methods */ + RAMIndexInput(const RAMIndexInput& clone); + void readInternal(uint8_t *dest, const int32_t len); + + /** Random-at methods */ + void seekInternal(const int64_t pos); + +public: + RAMIndexInput(RAMFile* f); + ~RAMIndexInput(); + IndexInput* clone() const; + + void close(); + int64_t length(); + QString getDirectoryType() const; +}; + + +/** +* A memory-resident {@link Directory} implementation. +*/ +class RAMDirectory : public Directory +{ + class RAMLock : public LuceneLock + { + private: + RAMDirectory* directory; + QString fname; + public: + RAMLock(const QString& name, RAMDirectory* dir); + virtual ~RAMLock(); + bool obtain(); + void release(); + bool isLocked(); + virtual QString toString() const; + }; + + typedef CL_NS(util)::CLHashMap<QString, RAMFile*, + CL_NS(util)::Compare::Qstring, CL_NS(util)::Equals::Qstring, + CL_NS(util)::Deletor::DummyQString, + CL_NS(util)::Deletor::Object<RAMFile> > FileMap; + +protected: + /// Removes an existing file in the directory. + virtual bool doDeleteFile(const QString& name); + + /** + * Creates a new <code>RAMDirectory</code> instance from a different + * <code>Directory</code> implementation. This can be used to load + * a disk-based index into memory. + * <P> + * This should be used only with indices that can fit into memory. + * + * @param dir a <code>Directory</code> value + * @exception IOException if an error occurs + */ + void _copyFromDir(Directory* dir, bool closeDir); + FileMap files; // unlike the java Hashtable, FileMap is not synchronized, and all access must be protected by a lock + +public: +#ifndef _CL_DISABLE_MULTITHREADING //do this so that the mutable keyword still works without mt enabled + mutable DEFINE_MUTEX(files_mutex) // mutable: const methods must also be able to synchronize properly +#endif + + // Returns a null terminated array of strings, one for each file in the directory. + QStringList list() const; + + /** Constructs an empty {@link Directory}. */ + RAMDirectory(); + + // Destructor - only call this if you are sure the directory + // is not being used anymore. Otherwise use the ref-counting + // facilities of dir->close + virtual ~RAMDirectory(); + RAMDirectory(Directory* dir); + + /** + * Creates a new <code>RAMDirectory</code> instance from the {@link FSDirectory}. + * + * @param dir a <code>String</code> specifying the full index directory path + */ + RAMDirectory(const QString& dir); + + /// Returns true iff the named file exists in this directory. + bool fileExists(const QString& name) const; + + /// Returns the time the named file was last modified. + int64_t fileModified(const QString& name) const; + + /// Returns the length in bytes of a file in the directory. + int64_t fileLength(const QString& name) const; + + /// Removes an existing file in the directory. + virtual void renameFile(const QString& from, const QString& to); + + /** Set the modified time of an existing file to now. */ + void touchFile(const QString& name); + + /// Creates a new, empty file in the directory with the given name. + /// Returns a stream writing this file. + virtual IndexOutput* createOutput(const QString& name); + + /// Construct a {@link Lock}. + /// @param name the name of the lock file + LuceneLock* makeLock(const QString& name); + + /// Returns a stream reading an existing file. + IndexInput* openInput(const QString& name); + + virtual void close(); + + QString toString() const; + + static QString DirectoryType() { return QLatin1String("RAM"); } + QString getDirectoryType() const { return DirectoryType(); } +}; + +CL_NS_END + +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/store/TransactionalRAMDirectory.cpp b/src/assistant/3rdparty/clucene/src/CLucene/store/TransactionalRAMDirectory.cpp new file mode 100644 index 000000000..056fa9bc3 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/store/TransactionalRAMDirectory.cpp @@ -0,0 +1,212 @@ +/* + * Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team + * + * Distributable under the terms of either the Apache License (Version 2.0) or + * the GNU Lesser General Public License, as specified in the COPYING file. + * + * Changes are Copyright(C) 2007, 2008 by Nokia Corporation and/or its subsidiary(-ies), all rights reserved. +*/ +#include "CLucene/StdHeader.h" +#include "TransactionalRAMDirectory.h" + +CL_NS_DEF(store) +CL_NS_USE(util) + +TransactionalRAMDirectory::TransactionalRAMDirectory() + : RAMDirectory() + , filesToRestoreOnAbort(false, true) +{ + transOpen = false; +} + +TransactionalRAMDirectory::~TransactionalRAMDirectory() +{ +} + +bool TransactionalRAMDirectory::archiveOrigFileIfNecessary(const QString& name) +{ + // If a file named $name was present when the transaction started and the + // original RAMFile object has not been archived for restoration upon + // transaction abort, then do so, and return true. + // In any other case, return false. + if (fileExists(name) && filesToRemoveOnAbort.find(name) == filesToRemoveOnAbort.end()) { + // The file exists, but isn't recorded as having been created after the + // start of the transaction, so it must've been present at the start of + // the transaction. + + // Transfer memory ownership of both the key and the value from files to + // filesToRestoreOnAbort. + QString origName = files.getKey(name); + RAMFile* origFile = files.get(name); + files.remove(name, true, true); + filesToRestoreOnAbort.put(origName, origFile); + + CND_CONDITION(!fileExists(name), + "File should not exist immediately after archival."); + return true; + } + + return false; +} + +void TransactionalRAMDirectory::unarchiveOrigFile(const QString& name) +{ + QString origName = filesToRestoreOnAbort.getKey(name); + if (origName.isEmpty()) { + _CLTHROWA(CL_ERR_RAMTransaction, + "File submitted for unarchival was not archived."); + } + RAMFile* origFile = filesToRestoreOnAbort.get(name); + // Transfer memory ownership back to files from filesToRestoreOnAbort. + filesToRestoreOnAbort.remove(name, true, true); + files.put(origName, origFile); +} + +bool TransactionalRAMDirectory::transIsOpen() const +{ + return transOpen; +} + +void TransactionalRAMDirectory::transStart() +{ + if (transOpen) { + _CLTHROWA(CL_ERR_RAMTransaction, + "Must resolve previous transaction before starting another."); + } + + CND_CONDITION(filesToRemoveOnAbort.size() == 0, + "filesToRemoveOnAbort should have been cleared by either its" + " constructor or transResolved."); + + CND_CONDITION(filesToRestoreOnAbort.size() == 0, + "filesToRestoreOnAbort should have been cleared by either its" + " constructor or transResolved."); + + transOpen = true; +} + +void TransactionalRAMDirectory::transResolved() +{ + // This method implements actions common to both forms of transaction + // resolution. + filesToRemoveOnAbort.clear(); + filesToRestoreOnAbort.clear(); + transOpen = false; +} + +void TransactionalRAMDirectory::transCommit() +{ + if (!transOpen) + _CLTHROWA(CL_ERR_RAMTransaction, "There is no open transaction."); + + // All storage is in memory, so commit is ultra-simple. + transResolved(); +} + +void TransactionalRAMDirectory::transAbort() +{ + if (!transOpen) + _CLTHROWA(CL_ERR_RAMTransaction, "There is no open transaction."); + + // Delete each file in filesToRemoveOnAbort. + FilenameSet::const_iterator itrDel = filesToRemoveOnAbort.begin(); + for ( ; itrDel != filesToRemoveOnAbort.end(); ++itrDel) { + size_t nameLength = itrDel->first.length(); + + // Special exception: Refrain from deleting a lock's flag file, as that + // would interfere with the operation of the lock. + if (!(nameLength >= 5 + && itrDel->first.rightRef(5) == QLatin1String(".lock"))) { + RAMDirectory::deleteFile(itrDel->first); + } + } + // Ownership of the memory of both the key and the value never left files, + // so there's no need for a special directive to filesToRemoveOnAbort. + filesToRemoveOnAbort.clear(); + + // Now that any new-since-trans-start files with the same names as + // already-present-at-trans-start files are out of the way, restore each + // file in filesToRestoreOnAbort. + TransFileMap::const_iterator itr = filesToRestoreOnAbort.begin(); + for ( ; itr != filesToRestoreOnAbort.end(); ++itr) { + files.put(itr->first, itr->second); + filesToRestoreOnAbort.remove(itr->first); + } + + CND_CONDITION(filesToRestoreOnAbort.size() == 0, + "filesToRestoreOnAbort should be empty."); + + transResolved(); +} + +bool TransactionalRAMDirectory::doDeleteFile(const QString& name) +{ + if (!transOpen) + return RAMDirectory::doDeleteFile(name); + + bool wasOriginalAndWasArchived = archiveOrigFileIfNecessary(name); + if (!wasOriginalAndWasArchived) { + // The file to be deleted wasn't present at transaction start, so instead + // of archiving it, we delete it the conventional way, making sure to + // erase its record in filesToRemoveOnAbort if it was listed there. + filesToRemoveOnAbort.remove(name); + return RAMDirectory::doDeleteFile(name); + } + return true; +} + +void TransactionalRAMDirectory::renameFile(const QString& from, const QString& to) +{ + // During the review on 2005.03.18, decided not to implement transactional + // renameFile for two reasons: + // a) It's not needed in the limited scenario for which + // TransactionalRAMDirectory was designed (IndexWriter::addDocument and + // subcode). + // b) Supporting renaming during a transaction would add considerable + // bookkeeping overhead, reducing the performance of the overwhelmingly + // typical case (commit) in order to support the rare case (abort). + // + // This was not a thinly disguised punt due to the complication of + // implementing renameFile transactionally; rather, several implementations + // were considered, but it seemed wrongheaded to degrade the performance of + // the typical case based on the mere potential need to support renameFile + // at some future point for the benefit of the atypical case. + if (transOpen) { + _CLTHROWA(CL_ERR_RAMTransaction, + "TransactionalRAMDirectory disallows renameFile during a transaction."); + } + RAMDirectory::renameFile(from, to); +} + +IndexOutput* TransactionalRAMDirectory::createOutput(const QString& name) +{ + if (!transOpen) + return RAMDirectory::createOutput(name); + + bool wasOriginalAndWasArchived = archiveOrigFileIfNecessary(name); + try { + IndexOutput* ret = RAMDirectory::createOutput(name); + // Importantly, we store a pointer to the filename memory managed by + // files, rather than that passed in by the client (name). We don't make + // an additional copy of the filename's memory because the transactional + // metadata container filesToRemoveOnAbort is not at risk of outliving + // files. + filesToRemoveOnAbort.put(files.getKey(name), NULL); + return ret; + } catch (...) { + if (wasOriginalAndWasArchived) { + unarchiveOrigFile(name); + } + throw; + } +} + +void TransactionalRAMDirectory::close() +{ + if (transOpen) + transAbort(); + + RAMDirectory::close(); +} + +CL_NS_END diff --git a/src/assistant/3rdparty/clucene/src/CLucene/store/TransactionalRAMDirectory.h b/src/assistant/3rdparty/clucene/src/CLucene/store/TransactionalRAMDirectory.h new file mode 100644 index 000000000..44c5e8e99 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/store/TransactionalRAMDirectory.h @@ -0,0 +1,76 @@ +/* + * Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team + * + * Distributable under the terms of either the Apache License (Version 2.0) or + * the GNU Lesser General Public License, as specified in the COPYING file. + * + * Changes are Copyright(C) 2007, 2008 by Nokia Corporation and/or its subsidiary(-ies), all rights reserved. +*/ +#ifndef _lucene_store_TransactionalRAMDirectory_ +#define _lucene_store_TransactionalRAMDirectory_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include <QtCore/QString> + +#include "RAMDirectory.h" +#include "CLucene/util/VoidList.h" + +CL_NS_DEF(store) + +/*** +This transactional in-memory Directory was created to address a specific +situation, and was deliberately pared down to the simplest viable +implementation. For the sake of simplicity, this implementation imposes +restrictions on what operations can be performed in the directory while a +transaction is in progress (documented in TransactionalRAMDirectory.cpp). + +Because the Lucene Directory interface itself is rather simplistic, it +would not be difficult to expand TransactionalRAMDirectory so that it +provided fully general transactionality. However, the developer of this +original implementation was of the opinion that the last thing CLucene +needs is gratuitous features that exceed their required complexity and +haven't been rigorously tested. +*/ +class TransactionalRAMDirectory : public RAMDirectory +{ +private: + typedef CL_NS(util)::CLSet<QString, void*, CL_NS(util)::Compare::Qstring, + CL_NS(util)::Deletor::DummyQString> FilenameSet; + FilenameSet filesToRemoveOnAbort; + + typedef CL_NS(util)::CLSet<QString, RAMFile*, CL_NS(util)::Compare::Qstring, + CL_NS(util)::Deletor::DummyQString, + CL_NS(util)::Deletor::Object<RAMFile> > TransFileMap; + TransFileMap filesToRestoreOnAbort; + + bool transOpen; + + void transResolved(); + bool archiveOrigFileIfNecessary(const QString& name); + void unarchiveOrigFile(const QString& name); + +protected: + bool doDeleteFile(const QString& name); + +public: + TransactionalRAMDirectory(); + virtual ~TransactionalRAMDirectory(); + + bool transIsOpen() const; + void transStart(); + void transCommit(); + void transAbort(); + + // Constrained operations: + void renameFile(const QString& from, const QString& to); + IndexOutput* createOutput(const QString& name); + + void close(); +}; + +CL_NS_END + +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/util/Arrays.h b/src/assistant/3rdparty/clucene/src/CLucene/util/Arrays.h new file mode 100644 index 000000000..ba60c5638 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/util/Arrays.h @@ -0,0 +1,164 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _lucene_util_Arrays_ +#define _lucene_util_Arrays_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include "VoidList.h" + +CL_NS_DEF(util) + class Arrays{ + public: + template<typename _type> + class _Arrays { + protected: + //used by binarySearch to check for equality + virtual bool equals(_type a,_type b) const = 0; + virtual int32_t compare(_type a,_type b) const = 0; + public: + virtual ~_Arrays(){ + } + + void sort(_type* a, int32_t alen, int32_t fromIndex, int32_t toIndex) const{ + CND_PRECONDITION(fromIndex < toIndex,"fromIndex >= toIndex"); + CND_PRECONDITION(fromIndex >= 0,"fromIndex < 0"); + + // First presort the array in chunks of length 6 with insertion + // sort. A mergesort would give too much overhead for this length. + for (int32_t chunk = fromIndex; chunk < toIndex; chunk += 6) + { + int32_t end = min(chunk + 6, toIndex); + for (int32_t i = chunk + 1; i < end; i++) + { + if (compare(a[i - 1], a[i]) > 0) + { + // not already sorted + int32_t j = i; + _type elem = a[j]; + do + { + a[j] = a[j - 1]; + j--; + } + while (j > chunk && compare(a[j - 1], elem) > 0); + a[j] = elem; + } + } + } + + int32_t len = toIndex - fromIndex; + // If length is smaller or equal 6 we are done. + if (len <= 6) + return; + + _type* src = a; + _type* dest = _CL_NEWARRAY(_type,alen); + _type* t = NULL; // t is used for swapping src and dest + + // The difference of the fromIndex of the src and dest array. + int32_t srcDestDiff = -fromIndex; + + // The merges are done in this loop + for (int32_t size = 6; size < len; size <<= 1) + { + for (int32_t start = fromIndex; start < toIndex; start += size << 1) + { + // mid is the start of the second sublist; + // end the start of the next sublist (or end of array). + int32_t mid = start + size; + int32_t end = min(toIndex, mid + size); + + // The second list is empty or the elements are already in + // order - no need to merge + if (mid >= end || compare(src[mid - 1], src[mid]) <= 0) + { + memcpy(dest + start + srcDestDiff, src+start, (end-start)*sizeof(_type)); + }// The two halves just need swapping - no need to merge + else if (compare(src[start], src[end - 1]) > 0) + { + memcpy(dest+end-size+srcDestDiff, src+start, size * sizeof(_type)); + memcpy(dest+start+srcDestDiff, src+mid, (end-mid) * sizeof(_type)); + + }else{ + // Declare a lot of variables to save repeating + // calculations. Hopefully a decent JIT will put these + // in registers and make this fast + int32_t p1 = start; + int32_t p2 = mid; + int32_t i = start + srcDestDiff; + + // The main merge loop; terminates as soon as either + // half is ended + while (p1 < mid && p2 < end) + { + dest[i++] = src[(compare(src[p1], src[p2]) <= 0 + ? p1++ : p2++)]; + } + + // Finish up by copying the remainder of whichever half + // wasn't finished. + if (p1 < mid) + memcpy(dest+i,src+p1, (mid-p1) * sizeof(_type)); + else + memcpy(dest+i,src+p2, (end-p2) * sizeof(_type)); + } + } + // swap src and dest ready for the next merge + t = src; + src = dest; + dest = t; + fromIndex += srcDestDiff; + toIndex += srcDestDiff; + srcDestDiff = -srcDestDiff; + } + + // make sure the result ends up back in the right place. Note + // that src and dest may have been swapped above, so src + // contains the sorted array. + if (src != a) + { + // Note that fromIndex == 0. + memcpy(a+srcDestDiff,src,toIndex * sizeof(_type)); + } + } + }; + }; + + template <typename _kt, typename _comparator, + typename class1, typename class2> + class CLListEquals: + public CL_NS_STD(binary_function)<class1*,class2*,bool> + { + typedef typename class1::const_iterator _itr1; + typedef typename class2::const_iterator _itr2; + public: + CLListEquals(){ + } + bool equals( class1* val1, class2* val2 ) const{ + static _comparator comp; + if ( val1 == val2 ) + return true; + size_t size = val1->size(); + if ( size != val2->size() ) + return false; + + _itr1 itr1 = val1->begin(); + _itr2 itr2 = val2->begin(); + while ( --size >= 0 ){ + if ( !comp(*itr1,*itr2) ) + return false; + itr1++; + itr2++; + } + return true; + } + }; +CL_NS_END +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/util/BitSet.cpp b/src/assistant/3rdparty/clucene/src/CLucene/util/BitSet.cpp new file mode 100644 index 000000000..3679bd120 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/util/BitSet.cpp @@ -0,0 +1,119 @@ +/* + * Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team + * + * Distributable under the terms of either the Apache License (Version 2.0) or + * the GNU Lesser General Public License, as specified in the COPYING file. + * + * Changes are Copyright(C) 2007, 2008 by Nokia Corporation and/or its subsidiary(-ies), all rights reserved. +*/ +#include "CLucene/StdHeader.h" +#include "BitSet.h" +#include "CLucene/store/Directory.h" + +CL_NS_USE(store) +CL_NS_DEF(util) + +BitSet::BitSet(const BitSet& copy) + : _size(copy._size) + , _count(-1) +{ + int32_t len = (_size >> 3) + 1; + bits = _CL_NEWARRAY(uint8_t, len); + memcpy(bits, copy.bits, len); +} + +BitSet::BitSet(int32_t size) + : _size(size) + , _count(-1) +{ + int32_t len = (_size >> 3) + 1; + bits = _CL_NEWARRAY(uint8_t, len); + memset(bits, 0, len); +} + +BitSet::BitSet(CL_NS(store)::Directory* d, const QString& name) +{ + _count = -1; + CL_NS(store)::IndexInput* input = d->openInput(name); + try { + _size = input->readInt(); // read size + _count = input->readInt(); // read count + + bits = _CL_NEWARRAY(uint8_t,(_size >> 3) + 1); // allocate bits + input->readBytes(bits, (_size >> 3) + 1); // read bits + } _CLFINALLY ( + input->close(); + _CLDELETE(input ); + ); +} + +void BitSet::write(CL_NS(store)::Directory* d, const QString& name) +{ + CL_NS(store)::IndexOutput* output = d->createOutput(name); + try { + output->writeInt(size()); // write size + output->writeInt(count()); // write count + output->writeBytes(bits, (_size >> 3) + 1); // write bits + } _CLFINALLY ( + output->close(); + _CLDELETE(output); + ); +} + +BitSet::~BitSet() +{ + _CLDELETE_ARRAY(bits); +} + +void BitSet::set(int32_t bit, bool val) +{ + if (val) + bits[bit >> 3] |= 1 << (bit & 7); + else + bits[bit >> 3] &= ~(1 << (bit & 7)); + + _count = -1; +} + +int32_t BitSet::size() const +{ + return _size; +} + +int32_t BitSet::count() +{ + // if the BitSet has been modified + if (_count == -1) { + static const uint8_t BYTE_COUNTS[] = { + 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4, + 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, + 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, + 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, + 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, + 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, + 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, + 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, + 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, + 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, + 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, + 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, + 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, + 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, + 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, + 4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8}; + + int32_t c = 0; + int32_t end = (_size >> 3) + 1; + for (int32_t i = 0; i < end; i++) + c += BYTE_COUNTS[bits[i]]; // sum bits per uint8_t + _count = c; + } + return _count; +} + +BitSet* BitSet::clone() const +{ + return _CLNEW BitSet(*this); +} + +CL_NS_END diff --git a/src/assistant/3rdparty/clucene/src/CLucene/util/BitSet.h b/src/assistant/3rdparty/clucene/src/CLucene/util/BitSet.h new file mode 100644 index 000000000..e93847e98 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/util/BitSet.h @@ -0,0 +1,62 @@ +/* + * Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team + * + * Distributable under the terms of either the Apache License (Version 2.0) or + * the GNU Lesser General Public License, as specified in the COPYING file. + * + * Changes are Copyright(C) 2007, 2008 by Nokia Corporation and/or its subsidiary(-ies), all rights reserved. +*/ +#ifndef _lucene_util_BitSet_ +#define _lucene_util_BitSet_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include <QtCore/QString> + +#include "CLucene/store/Directory.h" + +CL_NS_DEF(util) + +class BitSet : LUCENE_BASE +{ +public: + // Create a bitset with the specified size + BitSet (int32_t size); + BitSet(CL_NS(store)::Directory* d, const QString& name); + void write(CL_NS(store)::Directory* d, const QString& name); + + // Destructor for the bit set + ~BitSet(); + + // get the value of the specified bit + inline bool get(const int32_t bit) const + { + return (bits[bit >> 3] & (1 << (bit & 7))) != 0; + } + + // set the value of the specified bit + void set(int32_t bit, bool val = true); + + ///returns the size of the bitset + int32_t size() const; + + // Returns the total number of one bits in this BitSet. This is + // efficiently computed and cached, so that, if the BitSet is not changed, + // no recomputation is done for repeated calls. + int32_t count(); + BitSet *clone() const; + +protected: + BitSet(const BitSet& copy); + +private: + int32_t _size; + int32_t _count; + uint8_t *bits; +}; + +CL_NS_END + +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/util/Equators.cpp b/src/assistant/3rdparty/clucene/src/CLucene/util/Equators.cpp new file mode 100644 index 000000000..e112bd234 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/util/Equators.cpp @@ -0,0 +1,180 @@ +/* + * Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team + * + * Distributable under the terms of either the Apache License (Version 2.0) or + * the GNU Lesser General Public License, as specified in the COPYING file. + * + * Changes are Copyright(C) 2007, 2008 by Nokia Corporation and/or its subsidiary(-ies), all rights reserved. +*/ +#include "CLucene/StdHeader.h" +#include "Equators.h" + +CL_NS_DEF(util) + +bool Equals::Int32::operator()(const int32_t val1, const int32_t val2) const +{ + return (val1)==(val2); +} + +bool Equals::Char::operator()(const char* val1, const char* val2) const +{ + if ( val1 == val2 ) + return true; + return (strcmp(val1, val2) == 0); +} + +#ifdef _UCS2 +bool Equals::WChar::operator()(const wchar_t* val1, const wchar_t* val2) const +{ + if (val1 == val2) + return true; + return (_tcscmp(val1, val2) == 0); +} +#endif + +bool Equals::Qstring::operator()(const QString& val1, const QString& val2) const +{ + return (val1 == val2); +} + +/////////////////////////////////////////////////////////////////////////////// +// Comparors +/////////////////////////////////////////////////////////////////////////////// + +int32_t Compare::Int32::getValue() const +{ + return value; +} + +Compare::Int32::Int32(int32_t val) +{ + value = val; +} + +Compare::Int32::Int32() +{ + value = 0; +} + +int32_t Compare::Int32::compareTo(void* o) +{ + try { + Int32* other = (Int32*)o; + if (value == other->value) + return 0; + // Returns just -1 or 1 on inequality; doing math might overflow. + return value > other->value ? 1 : -1; + } catch(...) { + _CLTHROWA(CL_ERR_Runtime, "Couldnt compare types"); + } +} + +bool Compare::Int32::operator()(int32_t t1, int32_t t2) const +{ + return t1 > t2 ? true : false; +} + +size_t Compare::Int32::operator()(int32_t t) const +{ + return t; +} + +qreal Compare::Float::getValue() const +{ + return value; +} + +Compare::Float::Float(qreal val) +{ + value = val; +} + +int32_t Compare::Float::compareTo(void* o) +{ + try { + Float* other = (Float*)o; + if (value == other->value) + return 0; + // Returns just -1 or 1 on inequality; doing math might overflow. + return value > other->value ? 1 : -1; + } catch(...) { + _CLTHROWA(CL_ERR_Runtime,"Couldnt compare types"); + } +} + +bool Compare::Char::operator()(const char* val1, const char* val2) const +{ + if ( val1 == val2) + return false; + return (strcmp(val1, val2) < 0); +} + +size_t Compare::Char::operator()(const char* val1) const +{ + return CL_NS(util)::Misc::ahashCode(val1); +} + +#ifdef _UCS2 +bool Compare::WChar::operator()(const wchar_t* val1, const wchar_t* val2) const +{ + if ( val1==val2) + return false; + return (_tcscmp(val1, val2) < 0); +} + +size_t Compare::WChar::operator()(const wchar_t* val1) const +{ + return CL_NS(util)::Misc::whashCode(val1); +} +#endif + +const TCHAR* Compare::TChar::getValue() const +{ + return s; +} + +Compare::TChar::TChar() +{ + s = NULL; +} + +Compare::TChar::TChar(const TCHAR* str) +{ + this->s = str; +} + +int32_t Compare::TChar::compareTo(void* o) +{ + try { + TChar* os = (TChar*)o; + return _tcscmp(s, os->s); + } catch(...) { + _CLTHROWA(CL_ERR_Runtime,"Couldnt compare types"); + } + +} + +bool Compare::TChar::operator()(const TCHAR* val1, const TCHAR* val2) const +{ + if (val1 == val2) + return false; + + return (_tcscmp(val1, val2) < 0); +} + +size_t Compare::TChar::operator()(const TCHAR* val1) const +{ + return CL_NS(util)::Misc::thashCode(val1); +} + +bool Compare::Qstring::operator()(const QString& val1, const QString& val2) const +{ + return (val1 < val2); +} + +size_t Compare::Qstring::operator ()(const QString& val1) const +{ + return CL_NS(util)::Misc::qhashCode(val1); +} + +CL_NS_END diff --git a/src/assistant/3rdparty/clucene/src/CLucene/util/Equators.h b/src/assistant/3rdparty/clucene/src/CLucene/util/Equators.h new file mode 100644 index 000000000..11fcb0eaf --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/util/Equators.h @@ -0,0 +1,274 @@ +/* + * Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team + * + * Distributable under the terms of either the Apache License (Version 2.0) or + * the GNU Lesser General Public License, as specified in the COPYING file. + * + * Changes are Copyright(C) 2007, 2008 by Nokia Corporation and/or its subsidiary(-ies), all rights reserved. +*/ +#ifndef _lucene_util_Equators_ +#define _lucene_util_Equators_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include <QtCore/QString> + +//#ifdef QT_LINUXBASE +// we are going to use qreal now, we basically maintain our own clucene anyway +//// LSB doesn't define float_t - see http://bugs.linuxbase.org/show_bug.cgi?id=2374 +//typedef float float_t; +//#endif + +CL_NS_DEF(util) + +/////////////////////////////////////////////////////////////////////////////// +// Equators +/////////////////////////////////////////////////////////////////////////////// + +class Equals{ +public: + class Int32:public CL_NS_STD(binary_function)<const int32_t*,const int32_t*,bool> + { + public: + bool operator()( const int32_t val1, const int32_t val2 ) const; + }; + + class Char:public CL_NS_STD(binary_function)<const char*,const char*,bool> + { + public: + bool operator()( const char* val1, const char* val2 ) const; + }; +#ifdef _UCS2 + class WChar: public CL_NS_STD(binary_function)<const wchar_t*,const wchar_t*,bool> + { + public: + bool operator()( const wchar_t* val1, const wchar_t* val2 ) const; + }; + class TChar: public WChar{ + }; +#else + class TChar: public Char{ + }; +#endif + + template<typename _cl> + class Void:public CL_NS_STD(binary_function)<const void*,const void*,bool> + { + public: + bool operator()( _cl* val1, _cl* val2 ) const{ + return val1==val2; + } + }; + + class Qstring : public CL_NS_STD(binary_function)<const QString&, const QString&, bool> + { + public: + bool operator() (const QString& val1, const QString& val2) const; + }; +}; + + +/////////////////////////////////////////////////////////////////////////////// +// Comparors +/////////////////////////////////////////////////////////////////////////////// + +class Comparable : LUCENE_BASE +{ +public: + virtual ~Comparable(){ + } + + virtual int32_t compareTo(void* o) = 0; +}; + +/** @internal */ +class Compare{ +public: + class _base + { // traits class for hash containers + public: + enum + { // parameters for hash table + bucket_size = 4, // 0 < bucket_size + min_buckets = 8 + }; // min_buckets = 2 ^^ N, 0 < N + + _base() + { + } + }; + + class Int32:public _base, public Comparable{ + int32_t value; + public: + int32_t getValue() const; + Int32(int32_t val); + Int32(); + int32_t compareTo(void* o); + bool operator()( int32_t t1, int32_t t2 ) const; + size_t operator()( int32_t t ) const; + }; + + + class Float:public Comparable{ + qreal value; + public: + qreal getValue() const; + Float(qreal val); + int32_t compareTo(void* o); + }; + + + class Char: public _base //<char*> + { + public: + bool operator()( const char* val1, const char* val2 ) const; + size_t operator()( const char* val1) const; + }; + +#ifdef _UCS2 + class WChar: public _base //<wchar_t*> + { + public: + bool operator()( const wchar_t* val1, const wchar_t* val2 ) const; + size_t operator()( const wchar_t* val1) const; + }; +#endif + + class TChar: public _base, public Comparable{ + const TCHAR* s; + public: + const TCHAR* getValue() const; + + TChar(); + TChar(const TCHAR* str); + int32_t compareTo(void* o); + bool operator()( const TCHAR* val1, const TCHAR* val2 ) const; + size_t operator()( const TCHAR* val1) const; + }; + + + template<typename _cl> + class Void:public _base //<const void*,const void*,bool> + { + public: + int32_t compareTo(_cl* o){ + if ( this == o ) + return o; + else + return this > o ? 1 : -1; + } + bool operator()( _cl* t1, _cl* t2 ) const{ + return t1 > t2 ? true : false; + } + size_t operator()( _cl* t ) const{ + return (size_t)t; + } + }; + + class Qstring : public _base + { + public: + bool operator() (const QString& val1, const QString& val2) const; + size_t operator() (const QString& val1) const; + }; +}; + +/////////////////////////////////////////////////////////////////////////////// +// allocators +/////////////////////////////////////////////////////////////////////////////// + +class Deletor +{ +public: + + template<typename _kt> + class Array{ + public: + static void doDelete(_kt* arr){ + _CLDELETE_LARRAY(arr); + } + }; + class tcArray{ + public: + static void doDelete(const TCHAR* arr){ + _CLDELETE_CARRAY(arr); + } + }; + class acArray{ + public: + static void doDelete(const char* arr){ + _CLDELETE_CaARRAY(arr); + } + }; + + class Unintern{ + public: + static void doDelete(TCHAR* arr); + }; + template<typename _kt> + class Object{ + public: + static void doDelete(_kt* obj){ + _CLLDELETE(obj); + } + }; + template<typename _kt> + class Void{ + public: + static void doDelete(_kt* obj){ + _CLVDELETE(obj); + } + }; + class Dummy{ + public: + static void doDelete(const void* nothing) + { + // TODO: remove all occurances where it hits this point + // CND_WARNING(false, "Deletor::Dummy::doDelete run, set deleteKey + // or deleteValue to false"); + } + }; + class DummyInt32{ + public: + static void doDelete(const int32_t nothing){ + } + }; + class DummyFloat{ + public: + static void doDelete(const qreal nothing){ + } + }; + template <typename _type> + class ConstNullVal{ + public: + static void doDelete(const _type nothing) + { + // TODO: remove all occurances where it hits this point + // CND_WARNING(false, "Deletor::Dummy::doDelete run, set deleteKey + // or deleteValue to false"); + } + }; + + template <typename _type> + class NullVal{ + public: + static void doDelete(_type nothing) + { + // TODO: remove all occurances where it hits this point + // CND_WARNING(false, "Deletor::Dummy::doDelete run, set deleteKey + // or deleteValue to false"); + } + }; + class DummyQString { + public: + static void doDelete(const QString& nothing) { + } + }; +}; +//////////////////////////////////////////////////////////////////////////////// + +CL_NS_END +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/util/FastCharStream.cpp b/src/assistant/3rdparty/clucene/src/CLucene/util/FastCharStream.cpp new file mode 100644 index 000000000..f9fbe9b10 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/util/FastCharStream.cpp @@ -0,0 +1,107 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#include "CLucene/StdHeader.h" +#include "FastCharStream.h" + +#include "CLucene/util/Reader.h" + +CL_NS_DEF(util) + +const int32_t FastCharStream::maxRewindSize = LUCENE_MAX_WORD_LEN*2; + + FastCharStream::FastCharStream(Reader* reader): + pos(0), + rewindPos(0), + resetPos(0), + col(1), + line(1), + input(reader) + { + input->mark(maxRewindSize); + } + FastCharStream::~FastCharStream(){ + } + void FastCharStream::readChar(TCHAR &c) { + try{ + int32_t r = input->read(); + if ( r == -1 ) + input = NULL; + c = r; + }catch(CLuceneError& err){ + if ( err.number() == CL_ERR_IO ) + input = 0; + throw err; + } + } + int FastCharStream::GetNext() + { + // printf("getnext\n"); + if (input == 0 ) // end of file + { + _CLTHROWA(CL_ERR_IO,"warning : FileReader.GetNext : Read TCHAR over EOS."); + } + // this is rather inefficient + // implementing the functions from the java version of + // charstream will be much more efficient. + ++pos; + if ( pos > resetPos + maxRewindSize && rewindPos == 0) { + // move the marker one position (~expensive) + resetPos = pos-(maxRewindSize/2); + if ( resetPos != input->reset(resetPos) ) + _CLTHROWA(CL_ERR_IO,"Unexpected reset() result"); + input->mark(maxRewindSize); + input->skip((maxRewindSize/2) - 1); + } + TCHAR ch; + readChar(ch); + + if (input == NULL) { // eof + return -1; + } + if (rewindPos == 0) { + col += 1; + if(ch == '\n') { + line++; + col = 1; + } + } else { + rewindPos--; + } + return ch; + } + + void FastCharStream::UnGet(){ +// printf("UnGet \n"); + if (input == 0) + return; + if ( pos == 0 ) { + _CLTHROWA(CL_ERR_IO,"error : No character can be UnGet"); + } + rewindPos++; + + input->reset(pos-1); + pos--; + } + + int FastCharStream::Peek() { + int c = GetNext(); + UnGet(); + return c; + } + + bool FastCharStream::Eos() const { + return input==NULL; + } + + int32_t FastCharStream::Column() const { + return col; + } + + int32_t FastCharStream::Line() const { + return line; + } +CL_NS_END diff --git a/src/assistant/3rdparty/clucene/src/CLucene/util/FastCharStream.h b/src/assistant/3rdparty/clucene/src/CLucene/util/FastCharStream.h new file mode 100644 index 000000000..24e5b5612 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/util/FastCharStream.h @@ -0,0 +1,55 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _lucene_util_FastCharStream_ +#define _lucene_util_FastCharStream_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include "CLucene/util/Reader.h" + +CL_NS_DEF(util) + + /** Ported implementation of the FastCharStream class. */ + class FastCharStream:LUCENE_BASE + { + static const int32_t maxRewindSize; + int32_t pos; + int32_t rewindPos; + int64_t resetPos; + int32_t col; + int32_t line; + // read character from stream return false on error + void readChar(TCHAR &); + public: + Reader* input; + + /// Initializes a new instance of the FastCharStream class LUCENE_EXPORT. + FastCharStream(Reader* reader); + ~FastCharStream(); + + /// Returns the next TCHAR from the stream. + int GetNext(); + + void UnGet(); + + /// Returns the current top TCHAR from the input stream without removing it. + int Peek(); + + + /// Returns <b>True</b> if the end of stream was reached. + bool Eos() const; + + /// Gets the current column. + int32_t Column() const; + + /// Gets the current line. + int32_t Line() const; + }; +CL_NS_END +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/util/Misc.cpp b/src/assistant/3rdparty/clucene/src/CLucene/util/Misc.cpp new file mode 100644 index 000000000..42e3fd0a8 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/util/Misc.cpp @@ -0,0 +1,295 @@ +/* + * Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team + * + * Distributable under the terms of either the Apache License (Version 2.0) or + * the GNU Lesser General Public License, as specified in the COPYING file. + * + * Changes are Copyright(C) 2007, 2008 by Nokia Corporation and/or its subsidiary(-ies), all rights reserved. +*/ +#include "CLucene/StdHeader.h" +#include "Misc.h" + +#ifdef _CL_TIME_WITH_SYS_TIME +# include <sys/time.h> +# include <time.h> +#else +# if defined(_CL_HAVE_SYS_TIME_H) +# include <sys/time.h> +# else +# include <time.h> +# endif +#endif + +#ifdef _CL_HAVE_SYS_TIMEB_H +# include <sys/timeb.h> +#endif + +#ifdef UNDER_CE +#include <QTime> +#endif + +CL_NS_DEF(util) + +uint64_t Misc::currentTimeMillis() +{ +#ifndef UNDER_CE +#if defined(_CLCOMPILER_MSVC) || defined(__MINGW32__) || defined(__BORLANDC__) + struct _timeb tstruct; + _ftime(&tstruct); + + return (((uint64_t) tstruct.time) * 1000) + tstruct.millitm; +#else + struct timeval tstruct; + if (gettimeofday(&tstruct, NULL) < 0) { + _CLTHROWA(CL_ERR_Runtime,"Error in gettimeofday call."); + } + + return (((uint64_t) tstruct.tv_sec) * 1000) + tstruct.tv_usec / 1000; +#endif +#else //UNDER_CE + QT_USE_NAMESPACE + QTime t = QTime::currentTime(); + return t.second() * 1000 + t.msec(); +#endif //UNDER_CE +} + +// #pragma mark -- char related utils + +size_t Misc::ahashCode(const char* str) +{ + // Compute the hash code using a local variable to be reentrant. + size_t hashCode = 0; + while (*str != 0) + hashCode = hashCode * 31 + *str++; + return hashCode; +} + +size_t Misc::ahashCode(const char* str, size_t len) +{ + // Compute the hash code using a local variable to be reentrant. + size_t count = len; + size_t hashCode = 0; + for (size_t i = 0; i < count; i++) + hashCode = hashCode * 31 + *str++; + return hashCode; +} + +char* Misc::ajoin(const char* a, const char* b, const char* c, const char* d, + const char* e, const char* f) +{ +#define aLEN(x) (x == NULL ? 0 : strlen(x)) + const size_t totalLen = aLEN(a) + aLEN(b) + aLEN(c) + aLEN(d) + aLEN(e) + + aLEN(f) + sizeof(char); /* Space for terminator. */ + + char* buf = _CL_NEWARRAY(char, totalLen); + buf[0] = 0; + if (a != NULL) + strcat(buf, a); + + if (b != NULL) + strcat(buf, b); + + if (c != NULL) + strcat(buf, c); + + if (d != NULL) + strcat(buf, d); + + if (e != NULL) + strcat(buf, e); + + if (f != NULL) + strcat(buf, f); + + return buf; +} + +char* Misc::segmentname(const char* segment, const char* ext, int32_t x) +{ + CND_PRECONDITION(ext != NULL, "ext is NULL"); + + char* buf = _CL_NEWARRAY(char, CL_MAX_PATH); + if (x == -1) + _snprintf(buf, CL_MAX_PATH, "%s%s", segment, ext); + else + _snprintf(buf, CL_MAX_PATH, "%s%s%d", segment, ext, x); + return buf; +} + +void Misc::segmentname(char* buffer, int32_t bufferLen, const char* segment, + const char* ext, int32_t x) +{ + CND_PRECONDITION(buffer != NULL, "buffer is NULL"); + CND_PRECONDITION(segment != NULL, "segment is NULL"); + CND_PRECONDITION(ext != NULL, "extention is NULL"); + + if (x == -1) + _snprintf(buffer, bufferLen, "%s%s", segment, ext); + else + _snprintf(buffer, bufferLen, "%s%s%d", segment, ext, x); +} + +// #pragma mark -- qt related utils + +size_t Misc::qhashCode(const QString& str) +{ + size_t hashCode = 0; + for (int i = 0; i < str.count(); ++i) + hashCode = hashCode * 31 + str.at(i).unicode(); + return hashCode; +} + +size_t Misc::qhashCode(const QString& str, size_t len) +{ + size_t count = len; + size_t hashCode = 0; + for (size_t i = 0; i < count; ++i) + hashCode = hashCode * 31 + str.at(i).unicode(); + return hashCode; +} + +QString Misc::qjoin(const QString &a, const QString &b, const QString &c, + const QString &d, const QString &e, const QString &f) +{ + QString buffer; + + if (!a.isNull() && !a.isEmpty()) + buffer.append(a); + + if (!b.isNull() && !b.isEmpty()) + buffer.append(b); + + if (!c.isNull() && !c.isEmpty()) + buffer.append(c); + + if (!d.isNull() && !d.isEmpty()) + buffer.append(d); + + if (!e.isNull() && !e.isEmpty()) + buffer.append(e); + + if (!f.isNull() && !f.isEmpty()) + buffer.append(f); + + return buffer; +} + +QString Misc::segmentname(const QString& segment, const QString& ext, int32_t x) +{ + CND_PRECONDITION(!ext.isEmpty(), "ext is NULL"); + + if (x == -1) + return QString(segment + ext); + + QString buf(QLatin1String("%1%2%3")); + return buf.arg(segment).arg(ext).arg(x); +} + +void Misc::segmentname(QString& buffer, int32_t bufferLen, + const QString& segment, const QString& ext, int32_t x) +{ + CND_PRECONDITION(!segment.isEmpty(), "segment is NULL"); + CND_PRECONDITION(!ext.isEmpty(), "extention is NULL"); + + buffer = segment + ext; + if (x != -1) + buffer += QString::number(x); +} + +// #pragma mark -- TCHAR related utils + +int32_t Misc::stringDifference(const TCHAR* s1, int32_t len1, const TCHAR* s2, + int32_t len2) +{ + int32_t len = len1 < len2 ? len1 : len2; + for (int32_t i = 0; i < len; i++) + if (s1[i] != s2[i]) + return i; + return len; +} + +/* DSR:CL_BUG: (See comment for join method in Misc.h): */ +TCHAR* Misc::join (const TCHAR* a, const TCHAR* b, const TCHAR* c, + const TCHAR* d, const TCHAR* e, const TCHAR* f) +{ +#define LEN(x) (x == NULL ? 0 : _tcslen(x)) + const size_t totalLen = LEN(a) + LEN(b) + LEN(c) + LEN(d) + LEN(e) + LEN(f) + + sizeof(TCHAR); /* Space for terminator. */ + + TCHAR* buf = _CL_NEWARRAY(TCHAR, totalLen); + buf[0] = 0; + if (a != NULL) + _tcscat(buf, a); + + if (b != NULL) + _tcscat(buf, b); + + if (c != NULL) + _tcscat(buf, c); + + if (d != NULL) + _tcscat(buf, d); + + if (e != NULL) + _tcscat(buf, e); + + if (f != NULL) + _tcscat(buf, f); + + return buf; +} + +#ifdef _UCS2 + +size_t Misc::whashCode(const wchar_t* str) +{ + // Compute the hash code using a local variable to be reentrant. + size_t hashCode = 0; + while (*str != 0) + hashCode = hashCode * 31 + *str++; + return hashCode; +} + +size_t Misc::whashCode(const wchar_t* str, size_t len) +{ + // Compute the hash code using a local variable to be reentrant. + size_t count = len; + size_t hashCode = 0; + for (size_t i = 0; i < count; i++) + hashCode = hashCode * 31 + *str++; + return hashCode; +} + +char* Misc::_wideToChar(const wchar_t* s CL_FILELINEPARAM) +{ + size_t len = _tcslen(s); + char* msg = _CL_NEWARRAY(char, len + 1); + _cpywideToChar(s, msg, len + 1); + return msg; +} + +void Misc::_cpywideToChar(const wchar_t* s, char* d, size_t len) +{ + size_t sLen = wcslen(s); + for (uint32_t i = 0; i < len && i < sLen + 1; i++) + d[i] = LUCENE_OOR_CHAR(s[i]); +} + +wchar_t* Misc::_charToWide(const char* s CL_FILELINEPARAM) +{ + size_t len = strlen(s); + wchar_t* msg = _CL_NEWARRAY(wchar_t, len + 1); + _cpycharToWide(s, msg, len + 1); + return msg; +} + +void Misc::_cpycharToWide(const char* s, wchar_t* d, size_t len) +{ + size_t sLen = strlen(s); + for (uint32_t i = 0; i < len && i < sLen + 1; i++) + d[i] = s[i]; +} + +#endif + +CL_NS_END diff --git a/src/assistant/3rdparty/clucene/src/CLucene/util/Misc.h b/src/assistant/3rdparty/clucene/src/CLucene/util/Misc.h new file mode 100644 index 000000000..561c6e4d9 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/util/Misc.h @@ -0,0 +1,75 @@ +/* + * Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team + * + * Distributable under the terms of either the Apache License (Version 2.0) or + * the GNU Lesser General Public License, as specified in the COPYING file. + * + * Changes are Copyright(C) 2007, 2008 by Nokia Corporation and/or its subsidiary(-ies), all rights reserved. +*/ +#ifndef _lucene_util_Misc_H +#define _lucene_util_Misc_H + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include <QtCore/QString> + +CL_NS_DEF(util) + +class Misc +{ +public: + static uint64_t currentTimeMillis(); + + static size_t ahashCode(const char* str); + static size_t ahashCode(const char* str, size_t len); + static char* ajoin(const char* a, const char* b, const char* c = NULL, + const char* d = NULL, const char* e = NULL, const char* f = NULL); + static char* segmentname(const char* segment, const char* ext, int32_t x = -1); + static void segmentname(char* buffer, int32_t bufferLen, const char* segment, + const char* ext, int32_t x = -1); + + static size_t qhashCode(const QString& str); + static size_t qhashCode(const QString& str, size_t len); + static QString qjoin(const QString& a, const QString& b, + const QString& c = QString(), const QString& d = QString(), + const QString& e = QString(), const QString& f = QString()); + static QString segmentname(const QString& segment, const QString& ext, + int32_t x = -1 ); + static void segmentname(QString& buffer, int32_t bufferLen, + const QString& Segment, const QString& ext, int32_t x = -1); + + // Compares two strings, character by character, and returns the + // first position where the two strings differ from one another. + // + // @param s1 The first string to compare + // @param s1Len The length of the first string to compare + // @param s2 The second string to compare + // @param s2Len The length of the second string to compare + // @return The first position where the two strings differ. + static int32_t stringDifference(const TCHAR* s1, int32_t s1Len, + const TCHAR* s2, int32_t s2Len); + static TCHAR* join (const TCHAR* a, const TCHAR* b, const TCHAR* c = NULL, + const TCHAR* d = NULL, const TCHAR* e = NULL, const TCHAR* f = NULL ); + +#ifdef _UCS2 + static size_t whashCode(const wchar_t* str); + static size_t whashCode(const wchar_t* str, size_t len); + +# define thashCode whashCode + + static char* _wideToChar(const wchar_t* s CL_FILELINEPARAM); + static void _cpywideToChar(const wchar_t* s, char* d, size_t len); + + static wchar_t* _charToWide(const char* s CL_FILELINEPARAM); + static void _cpycharToWide(const char* s, wchar_t* d, size_t len); + +#else +# define thashCode ahashCode +#endif +}; + +CL_NS_END + +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/util/PriorityQueue.h b/src/assistant/3rdparty/clucene/src/CLucene/util/PriorityQueue.h new file mode 100644 index 000000000..45649ee7f --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/util/PriorityQueue.h @@ -0,0 +1,177 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _lucene_util_PriorityQueue_ +#define _lucene_util_PriorityQueue_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif +CL_NS_DEF(util) + +// A PriorityQueue maintains a partial ordering of its elements such that the +// least element can always be found in constant time. Put()'s and pop()'s +// require log(size) time. +template <class _type,typename _valueDeletor> class PriorityQueue:LUCENE_BASE { + private: + _type* heap; //(was object[]) + size_t _size; + bool dk; + size_t maxSize; + + void upHeap(){ + size_t i = _size; + _type node = heap[i]; // save bottom node (WAS object) + int32_t j = ((uint32_t)i) >> 1; + while (j > 0 && lessThan(node,heap[j])) { + heap[i] = heap[j]; // shift parents down + i = j; + j = ((uint32_t)j) >> 1; + } + heap[i] = node; // install saved node + } + void downHeap(){ + size_t i = 1; + _type node = heap[i]; // save top node + size_t j = i << 1; // find smaller child + size_t k = j + 1; + if (k <= _size && lessThan(heap[k], heap[j])) { + j = k; + } + while (j <= _size && lessThan(heap[j],node)) { + heap[i] = heap[j]; // shift up child + i = j; + j = i << 1; + k = j + 1; + if (k <= _size && lessThan(heap[k], heap[j])) { + j = k; + } + } + heap[i] = node; // install saved node + } + + protected: + PriorityQueue(){ + this->_size = 0; + this->dk = false; + this->heap = NULL; + this->maxSize = 0; + } + + // Determines the ordering of objects in this priority queue. Subclasses + // must define this one method. + virtual bool lessThan(_type a, _type b)=0; + + // Subclass constructors must call this. + void initialize(const int32_t maxSize, bool deleteOnClear){ + _size = 0; + dk = deleteOnClear; + int32_t heapSize = maxSize + 1; + heap = _CL_NEWARRAY(_type,heapSize); + this->maxSize = maxSize; + } + + public: + virtual ~PriorityQueue(){ + clear(); + _CLDELETE_ARRAY(heap); + } + + /** + * Adds an Object to a PriorityQueue in log(size) time. + * If one tries to add more objects than maxSize from initialize + * a RuntimeException (ArrayIndexOutOfBound) is thrown. + */ + void put(_type element){ + if ( _size>=maxSize ) + _CLTHROWA(CL_ERR_IndexOutOfBounds,"add is out of bounds"); + + ++_size; + heap[_size] = element; + upHeap(); + } + + /** + * Adds element to the PriorityQueue in log(size) time if either + * the PriorityQueue is not full, or not lessThan(element, top()). + * @param element + * @return true if element is added, false otherwise. + */ + bool insert(_type element){ + if(_size < maxSize){ + put(element); + return true; + }else if(_size > 0 && !lessThan(element, top())){ + if ( dk ){ + _valueDeletor::doDelete(heap[1]); + } + heap[1] = element; + adjustTop(); + return true; + }else + return false; + } + + /** + * Returns the least element of the PriorityQueue in constant time. + */ + _type top(){ + if (_size > 0) + return heap[1]; + else + return NULL; + } + + /** Removes and returns the least element of the PriorityQueue in log(size) + * time. + */ + _type pop(){ + if (_size > 0) { + _type result = heap[1]; // save first value + heap[1] = heap[_size]; // move last to first + + heap[_size] = (_type)0; // permit GC of objects + --_size; + downHeap(); // adjust heap + return result; + } else + return (_type)NULL; + } + + /**Should be called when the object at top changes values. Still log(n) + worst case, but it's at least twice as fast to <pre> + { pq.top().change(); pq.adjustTop(); } + </pre> instead of <pre> + { o = pq.pop(); o.change(); pq.push(o); } + </pre> + */ + void adjustTop(){ + downHeap(); + } + + + /** + * Returns the number of elements currently stored in the PriorityQueue. + */ + size_t size(){ + return _size; + } + + /** + * Removes all entries from the PriorityQueue. + */ + void clear(){ + for (size_t i = 1; i <= _size; ++i){ + if ( dk ){ + _valueDeletor::doDelete(heap[i]); + } + } + _size = 0; + } + }; + +CL_NS_END +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/util/Reader.cpp b/src/assistant/3rdparty/clucene/src/CLucene/util/Reader.cpp new file mode 100644 index 000000000..1ce97106d --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/util/Reader.cpp @@ -0,0 +1,186 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#include "CLucene/StdHeader.h" +#include "Reader.h" + +CL_NS_DEF(util) + +StringReader::StringReader ( const TCHAR* value ): + Reader(NULL,true){ + reader = new jstreams::StringReader<TCHAR>(value); +} +StringReader::StringReader ( const TCHAR* value, const int32_t length ): + Reader(NULL,true){ + reader = new jstreams::StringReader<TCHAR>(value,length); +} +StringReader::StringReader ( const TCHAR* value, const int32_t length, bool copyData ): + Reader(NULL,true){ + reader = new jstreams::StringReader<TCHAR>(value,length, copyData); +} +StringReader::~StringReader(){ +} + + +FileReader::FileReader ( const char* path, const char* enc, + const int32_t cachelen, const int32_t /*cachebuff*/ ): + Reader(NULL, true) +{ + this->input = new jstreams::FileInputStream(path, cachelen); + this->reader = new SimpleInputStreamReader(this->input,enc); //(this is a jstream object) +} + +FileReader::~FileReader (){ + if (input) + delete input; +} +int32_t FileReader::read(const TCHAR*& start, int32_t _min, int32_t _max) { + return reader->read(start, _min, _max); +} +int64_t FileReader::mark(int32_t readlimit) { + return reader->mark(readlimit); +} +int64_t FileReader::reset(int64_t newpos) { + return reader->reset(newpos); +} + + + +SimpleInputStreamReader::SimpleInputStreamReader(jstreams::StreamBase<char> *i, const char* enc) +{ + finishedDecoding = false; + input = i; + charbuf.setSize(262); + + if ( strcmp(enc,"ASCII")==0 ) + encoding = ASCII; +#ifdef _UCS2 + else if ( strcmp(enc,"UTF-8")==0 ) + encoding = UTF8; + else if ( strcmp(enc,"UCS-2LE")==0 ) + encoding = UCS2_LE; +#endif + else + _CLTHROWA(CL_ERR_IllegalArgument,"Unsupported encoding, use jstreams iconv based instead"); + + mark(262); + charsLeft = 0; +} +SimpleInputStreamReader::~SimpleInputStreamReader(){ + input = NULL; +} +int32_t SimpleInputStreamReader::decode(TCHAR* start, int32_t space){ + // decode from charbuf + const char *inbuf = charbuf.readPos; + const char *inbufend = charbuf.readPos + charbuf.avail; + TCHAR *outbuf = start; + const TCHAR *outbufend = outbuf + space; + + if ( encoding == ASCII ){ + while ( outbuf<outbufend && inbuf<inbufend ){ + *outbuf = *inbuf; + outbuf++; + inbuf++; + } + +#ifdef _UCS2 + } + else if ( encoding == UCS2_LE ){ + while ( outbuf<outbufend && (inbuf+1)<inbufend ){ + uint8_t c1 = *inbuf; + uint8_t c2 = *(inbuf+1); + unsigned short c = c1 | (c2<<8); + + #ifdef _UCS2 + *outbuf = c; + #else + *outbuf = LUCENE_OOR_CHAR(c); + #endif + outbuf++; + inbuf+=2; + } + + }else if ( encoding == UTF8 ){ + while ( outbuf<outbufend && inbuf<inbufend ){ + size_t utflen = lucene_utf8charlen(inbuf); + if ( utflen==0 ){ + error = "Invalid multibyte sequence."; + status = jstreams::Error; + return -1; + }else if ( inbuf+utflen > inbufend ){ + break; //character incomplete + }else{ + size_t rd = lucene_utf8towc(outbuf,inbuf,inbufend-inbuf); + if ( rd == 0 ){ + error = "Invalid multibyte sequence."; + status = jstreams::Error; + return -1; + }else{ + inbuf+=rd; + outbuf++; + } + } + } +#endif //_UCS2 + }else + _CLTHROWA(CL_ERR_Runtime,"Unexpected encoding"); + + if ( outbuf < outbufend ) { + //we had enough room to convert the entire input + if ( inbuf < inbufend ) { + // last character is incomplete + // move from inbuf to the end to the start of + // the buffer + memmove(charbuf.start, inbuf, inbufend-inbuf); + charbuf.readPos = charbuf.start; + charbuf.avail = inbufend-inbuf; + } else if ( outbuf < outbufend ) { //input sequence was completely converted + charbuf.readPos = charbuf.start; + charbuf.avail = 0; + if (input == NULL) { + finishedDecoding = true; + } + } + } else { + charbuf.readPos += charbuf.avail - (inbufend-inbuf); + charbuf.avail = inbufend-inbuf; + } + return outbuf-start; +} + +int32_t SimpleInputStreamReader::fillBuffer(TCHAR* start, int32_t space) { + // fill up charbuf + if (input && charbuf.readPos == charbuf.start) { + const char *begin; + int32_t numRead; + numRead = input->read(begin, 1, charbuf.size - charbuf.avail); + //printf("filled up charbuf\n"); + if (numRead < -1) { + error = input->getError(); + status = jstreams::Error; + input = 0; + return numRead; + } + if (numRead < 1) { + // signal end of input buffer + input = 0; + if (charbuf.avail) { + error = "stream ends on incomplete character"; + status = jstreams::Error; + } + return -1; + } + // copy data into other buffer + memmove( charbuf.start + charbuf.avail, begin, numRead * sizeof(char)); + charbuf.avail = numRead + charbuf.avail; + } + // decode + int32_t n = decode(start, space); + //printf("decoded %i\n", n); + return n; +} + +CL_NS_END diff --git a/src/assistant/3rdparty/clucene/src/CLucene/util/Reader.h b/src/assistant/3rdparty/clucene/src/CLucene/util/Reader.h new file mode 100644 index 000000000..6b018b3aa --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/util/Reader.h @@ -0,0 +1,138 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _lucene_util_Reader_ +#define _lucene_util_Reader_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include "streambase.h" +#include "stringreader.h" +#include "fileinputstream.h" +#include "bufferedstream.h" + +CL_NS_DEF(util) +/** +* An inline wrapper that reads from Jos van den Oever's jstreams +*/ +class Reader:LUCENE_BASE { +typedef jstreams::StreamBase<TCHAR> jsReader; +public: + bool deleteReader; + jsReader* reader; + + Reader(jsReader* reader, bool deleteReader){ + this->reader = reader; + this->deleteReader = deleteReader; + } + virtual ~Reader(){ + if ( deleteReader ) + delete reader; + reader = NULL; + } + inline int read(){ + const TCHAR*b; + int32_t nread = reader->read(b, 1,1); + if ( nread < -1 ) //if not eof + _CLTHROWA(CL_ERR_IO,reader->getError() ); + else if ( nread == -1 ) + return -1; + else + return b[0]; + } + /** + * Read at least 1 character, and as much as is conveniently available + */ + inline int32_t read(const TCHAR*& start){ + int32_t nread = reader->read(start,1,0); + if ( nread < -1 ) //if not eof + _CLTHROWA(CL_ERR_IO,reader->getError()); + else + return nread; + } + inline int32_t read(const TCHAR*& start, int32_t len){ + int32_t nread = reader->read(start, len, len); + if ( nread < -1 ) //if not eof + _CLTHROWA(CL_ERR_IO,reader->getError()); + else + return nread; + } + inline int64_t skip(int64_t ntoskip){ + int64_t skipped = reader->skip(ntoskip); + if ( skipped < 0 ) + _CLTHROWA(CL_ERR_IO,reader->getError()); + else + return skipped; + } + inline int64_t mark(int32_t readAheadlimit){ + int64_t pos = reader->mark(readAheadlimit); + if ( pos < 0 ) + _CLTHROWA(CL_ERR_IO,reader->getError()); + else + return pos; + } + int64_t reset(int64_t pos){ + int64_t r = reader->reset(pos); + if ( r < 0 ) + _CLTHROWA(CL_ERR_IO,reader->getError()); + else + return r; + } +}; + +///A helper class which constructs a the jstreams StringReader. +class StringReader: public Reader{ +public: + StringReader ( const TCHAR* value ); + StringReader ( const TCHAR* value, const int32_t length ); + StringReader ( const TCHAR* value, const int32_t length, bool copyData ); + ~StringReader(); +}; + +/** A very simple inputstreamreader implementation. For a +* more complete InputStreamReader, use the jstreams version +* located in the contrib package +*/ +class SimpleInputStreamReader: public jstreams::BufferedInputStream<TCHAR>{ + int32_t decode(TCHAR* start, int32_t space); + int encoding; + enum{ + ASCII=1, + UTF8=2, + UCS2_LE=3 + }; + bool finishedDecoding; + jstreams::StreamBase<char>* input; + int32_t charsLeft; + + jstreams::InputStreamBuffer<char> charbuf; + int32_t fillBuffer(TCHAR* start, int32_t space); +public: + SimpleInputStreamReader(jstreams::StreamBase<char> *i, const char* encoding); + ~SimpleInputStreamReader(); +}; + +/** +* A helper class which constructs a FileReader with a specified +* simple encodings, or a given inputstreamreader +*/ +class FileReader: public Reader{ + jstreams::FileInputStream* input; +public: + FileReader ( const char* path, const char* enc, + const int32_t cachelen = 13, + const int32_t cachebuff = 14 ); //todo: optimise these cache values + ~FileReader (); + + int32_t read(const TCHAR*& start, int32_t _min, int32_t _max); + int64_t mark(int32_t readlimit); + int64_t reset(int64_t); +}; + +CL_NS_END +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/util/StringBuffer.cpp b/src/assistant/3rdparty/clucene/src/CLucene/util/StringBuffer.cpp new file mode 100644 index 000000000..b5f1ca238 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/util/StringBuffer.cpp @@ -0,0 +1,335 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#include "CLucene/StdHeader.h" +#include "StringBuffer.h" +#include "Misc.h" + +CL_NS_DEF(util) + + StringBuffer::StringBuffer(TCHAR* buf,int32_t maxlen, const bool consumeBuffer){ + buffer = buf; + bufferLength = maxlen; + bufferOwner = !consumeBuffer; + len = 0; + } + StringBuffer::StringBuffer(){ + //Func - Constructor. Allocates a buffer with the default length. + //Pre - true + //Post - buffer of length bufferLength has been allocated + + //Initialize + bufferLength = LUCENE_DEFAULT_TOKEN_BUFFER_SIZE; + len = 0; + //Allocate a buffer of length bufferLength + buffer = _CL_NEWARRAY(TCHAR,bufferLength); + bufferOwner = true; + } + + StringBuffer::StringBuffer(const int32_t initSize){ + //Func - Constructor. Allocates a buffer of length initSize + 1 + //Pre - initSize > 0 + //Post - A buffer has been allocated of length initSize + 1 + + //Initialize the bufferLength to initSize + 1 The +1 is for the terminator '\0' + bufferLength = initSize + 1; + len = 0; + //Allocate a buffer of length bufferLength + buffer = _CL_NEWARRAY(TCHAR,bufferLength); + bufferOwner = true; + } + + StringBuffer::StringBuffer(const TCHAR* value){ + //Func - Constructor. + // Creates an instance of Stringbuffer containing a copy of the string value + //Pre - value != NULL + //Post - An instance of StringBuffer has been created containing the copy of the string value + + //Initialize the length of the string to be stored in buffer + len = (int32_t) _tcslen(value); + + //Calculate the space occupied in buffer by a copy of value + const int32_t occupiedLength = len + 1; + + // Minimum allocated buffer length is LUCENE_DEFAULT_TOKEN_BUFFER_SIZE. + bufferLength = (occupiedLength >= LUCENE_DEFAULT_TOKEN_BUFFER_SIZE + ? occupiedLength : LUCENE_DEFAULT_TOKEN_BUFFER_SIZE); + + //Allocate a buffer of length bufferLength + buffer = _CL_NEWARRAY(TCHAR,bufferLength); + bufferOwner = true; + //Copy the string value into buffer + _tcsncpy(buffer, value, occupiedLength); + //Assert that the buffer has been terminated at the end of the string + CND_PRECONDITION (buffer[len] == '\0', "Buffer was not correctly terminated"); + } + + StringBuffer::~StringBuffer() { + // Func - Destructor + // Pre - true + // Post - Instanc has been destroyed + + if( bufferOwner ){ + _CLDELETE_CARRAY(buffer); + }else + buffer = NULL; + } + void StringBuffer::clear(){ + //Func - Clears the Stringbuffer and resets it to it default empty state + //Pre - true + //Post - pre(buffer) has been destroyed and a new one has been allocated + + //Destroy the current buffer if present + _CLDELETE_CARRAY(buffer); + + //Initialize + len = 0; + bufferLength = LUCENE_DEFAULT_TOKEN_BUFFER_SIZE; + //Allocate a buffer of length bufferLength + buffer = _CL_NEWARRAY(TCHAR,bufferLength); + } + + void StringBuffer::appendChar(const TCHAR character) { + //Func - Appends a single character + //Pre - true + //Post - The character has been appended to the string in the buffer + + //Check if the current buffer length is sufficient to have the string value appended + if (len + 1 > bufferLength){ + //Have the size of the current string buffer increased because it is too small + growBuffer(len + 1); + } + //Put character at position len which is the end of the string in the buffer + //Note that this action might overwrite the terminator of the string '\0', which + //is kind of tricky + buffer[len] = character; + //Increase the len by to represent the correct length of the string in the buffer + len++; + } + + void StringBuffer::append(const TCHAR* value) { + //Func - Appends a copy of the string value + //Pre - value != NULL + //Post - value has been copied and appended to the string in buffer + + append(value, _tcslen(value)); + } + void StringBuffer::append(const TCHAR* value, size_t appendedLength) { + //Func - Appends a copy of the string value + //Pre - value != NULL + // appendedLength contains the length of the string value which is to be appended + //Post - value has been copied and appended to the string in buffer + + //Check if the current buffer length is sufficient to have the string value appended + if (len + appendedLength + 1 > bufferLength){ + //Have the size of the current string buffer increased because it is too small + growBuffer(len + appendedLength + 1); + } + + //Copy the string value into the buffer at postion len + _tcsncpy(buffer + len, value, appendedLength); + + //Add the length of the copied string to len to reflect the new length of the string in + //the buffer (Note: len is not the bufferlength!) + len += appendedLength; + } + + void StringBuffer::appendInt(const int32_t value) { + //Func - Appends an integer (after conversion to a character string) + //Pre - true + //Post - The converted integer value has been appended to the string in buffer + + //instantiate a buffer of 30 charactes for the conversion of the integer + TCHAR buf[30]; + //Convert the integer value to a string buf using the radix 10 (duh) + _i64tot(value, buf, 10); + //Have the converted integer now stored in buf appended to the string in buffer + append(buf); + } + + void StringBuffer::appendFloat(const qreal value, const int32_t digits){ + //Func - Appends a qreal (after conversion to a character string) + //Pre - digits > 0. Indicates the minimum number of characters printed + //Post - The converted qreal value has been appended to the string in buffer + + //using sprintf("%f" was not reliable on other plaforms... we use a custom float convertor + //bvk: also, using sprintf and %f seems excessivelly slow + if(digits>8) + _CLTHROWA(CL_ERR_IllegalArgument,"Too many digits..."); + + //the maximum number of characters that int64 will hold is 23. so we need 23*2+2 + TCHAR buf[48]; //the buffer to hold + int64_t v = (int64_t)value; //the integer value of the float + _i64tot(v,buf,10); //add the whole number + + size_t len = 99-_tcslen(buf); //how many digits we have to work with? + size_t dig = len< (size_t)digits ? len : digits; + if ( dig > 0 ){ + _tcscat(buf,_T(".")); //add a decimal point + + int64_t remi=(int64_t)((value-v)*pow((qreal)10,(qreal)(dig+1))); //take the remainder and make a whole number + if ( remi<0 ) remi*=-1; + int64_t remadj=remi/10; + if ( remi-(remadj*10) >=5 ) + remadj++; //adjust remainder + + // add as many zeros as necessary between the decimal point and the + // significant part of the number. Fixes a bug when trying to print + // numbers that have zeros right after the decimal point + if (remadj) { + int32_t numZeros = dig - (int32_t)log10((qreal)remadj) - 1; + while(numZeros-- > 0) + _tcscat(buf,_T("0")); //add a zero before the decimal point + } + + _i64tot(remadj,buf+_tcslen(buf),10); //add the remainder + } + + append(buf); + } + + void StringBuffer::prepend(const TCHAR* value){ + //Func - Puts a copy of the string value infront of the current string in the StringBuffer + //Pre - value != NULL + //Post - The string in pre(buffer) has been shifted n positions where n equals the length of value. + // The string value was then copied to the beginning of stringbuffer + + prepend(value, _tcslen(value)); + } + + void StringBuffer::prepend(const TCHAR* value, const size_t prependedLength) { + //Func - Puts a copy of the string value in front of the string in the StringBuffer + //Pre - value != NULL + // prependedLength contains the length of the string value which is to be prepended + //Post - A copy of the string value is has been in front of the string in buffer + //todo: something is wrong with this code, i'm sure... it only grows (and therefore moves if the buffer is to small) + //Check if the current buffer length is sufficient to have the string value prepended + if (prependedLength + len + 1 > bufferLength){ + //Have the size of the current string buffer increased because it is too small + //Because prependedLength is passed as the second argument to growBuffer, + //growBuffer will have left the first prependedLength characters empty + //when it recopied buffer during reallocation. + growBuffer(prependedLength + len + 1, prependedLength); + } + + //Copy the string value into the buffer at postion 0 + _tcsncpy(buffer, value, prependedLength); + //Add the length of the copied string to len to reflect the new length of the string in + //the buffer (Note: len is not the bufferlength!) + len += prependedLength; + } + + int32_t StringBuffer::length() const{ + //Func - Returns the length of the string in the StringBuffer + //Pre - true + //Post - The length len of the string in the buffer has been returned + + return len; + } + TCHAR* StringBuffer::toString(){ + //Func - Returns a copy of the current string in the StringBuffer sized equal to the length of the string + // in the StringBuffer. + //Pre - true + //Post - The copied string has been returned + + //Instantiate a buffer equal to the length len + 1 + TCHAR* ret = _CL_NEWARRAY(TCHAR,len + 1); + if (ret){ + //Copy the string in buffer + _tcsncpy(ret, buffer, len); + //terminate the string + ret[len] = '\0'; + } + //return the the copy + return ret; + } + TCHAR* StringBuffer::getBuffer() { + //Func - '\0' terminates the buffer and returns its pointer + //Pre - true + //Post - buffer has been '\0' terminated and returned + + // Check if the current buffer is '\0' terminated + if (len == bufferLength){ + //Make space for terminator, if necessary. + growBuffer(len + 1); + } + //'\0' buffer so it can be returned properly + buffer[len] = '\0'; + + return buffer; + } + + void StringBuffer::reserve(const int32_t size){ + if ( bufferLength >= size ) + return; + bufferLength = size; + + //Allocate a new buffer of length bufferLength + TCHAR* tmp = _CL_NEWARRAY(TCHAR,bufferLength); + _tcsncpy(tmp, buffer, len); + tmp[len] = '\0'; + + //destroy the old buffer + if (buffer){ + _CLDELETE_CARRAY(buffer); + } + //Assign the new buffer tmp to buffer + buffer = tmp; + } + + void StringBuffer::growBuffer(const int32_t minLength) { + //Func - Has the buffer grown to a minimum length of minLength or bigger + //Pre - minLength >= len + 1 + //Post - The buffer has been grown to a minimum length of minLength or bigger + + growBuffer(minLength, 0); + } + void StringBuffer::growBuffer(const int32_t minLength, const int32_t skippingNInitialChars) { + //Func - Has the buffer grown to a minimum length of minLength or bigger and shifts the + // current string in buffer by skippingNInitialChars forward + //Pre - After growth, must have at least enough room for contents + terminator so + // minLength >= skippingNInitialChars + len + 1 + // skippingNInitialChars >= 0 + //Post - The buffer has been grown to a minimum length of minLength or bigger and + // if skippingNInitialChars > 0, the contents of the buffer has beeen shifted + // forward by skippingNInitialChars positions as the buffer is reallocated, + // leaving the first skippingNInitialChars uninitialized (presumably to be + // filled immediately thereafter by the caller). + + CND_PRECONDITION (skippingNInitialChars >= 0, "skippingNInitialChars is less than zero"); + CND_PRECONDITION (minLength >= skippingNInitialChars + len + 1,"skippingNInitialChars is not large enough"); + + //More aggressive growth strategy to offset smaller default buffer size: + if ( !bufferOwner ){ + if ( bufferLength<minLength ) + _CLTHROWA(CL_ERR_IllegalArgument,"[StringBuffer::grow] un-owned buffer could not be grown"); + return; + } + + bufferLength *= 2; + //Check that bufferLength is bigger than minLength + if (bufferLength < minLength){ + //Have bufferLength become minLength because it still was too small + bufferLength = minLength; + } + + //Allocate a new buffer of length bufferLength + TCHAR* tmp = _CL_NEWARRAY(TCHAR,bufferLength); + //The old buffer might not have been null-terminated, so we _tcsncpy + //only len bytes, not len+1 bytes (the latter might read one char off the + //end of the old buffer), then apply the terminator to the new buffer. + _tcsncpy(tmp + skippingNInitialChars, buffer, len); + tmp[skippingNInitialChars + len] = '\0'; + + //destroy the old buffer + if (buffer){ + _CLDELETE_CARRAY(buffer); + } + //Assign the new buffer tmp to buffer + buffer = tmp; + } + +CL_NS_END diff --git a/src/assistant/3rdparty/clucene/src/CLucene/util/StringBuffer.h b/src/assistant/3rdparty/clucene/src/CLucene/util/StringBuffer.h new file mode 100644 index 000000000..505b57594 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/util/StringBuffer.h @@ -0,0 +1,77 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _lucene_util_StringBuffer_ +#define _lucene_util_StringBuffer_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + + +CL_NS_DEF(util) + class StringBuffer:LUCENE_BASE{ + public: + ///Constructor. Allocates a buffer with the default length. + StringBuffer(); + ///Constructor. Allocates a buffer of length initSize + 1 + StringBuffer(const int32_t initSize); + ///Constructor. Creates an instance of Stringbuffer containing a copy of + ///the string value + StringBuffer(const TCHAR* value); + ///Constructs a StringBuffer using another buffer. The StringBuffer can + ///the be used to easily manipulate the buffer. + StringBuffer(TCHAR* buf,int32_t maxlen, const bool consumeBuffer); + ///Destructor + ~StringBuffer(); + ///Clears the Stringbuffer and resets it to it default empty state + void clear(); + + ///Appends a single character + void appendChar(const TCHAR chr); + ///Appends a copy of the string value + void append(const TCHAR* value); + ///Appends a copy of the string value + void append(const TCHAR* value, size_t appendedLength); + ///Appends an integer (after conversion to a character string) + void appendInt(const int32_t value); + ///Appends a qreal (after conversion to a character string) + void appendFloat(const qreal value, const int32_t digits); + ///Puts a copy of the string value in front of the current string in the StringBuffer + void prepend(const TCHAR* value); + ///Puts a copy of the string value in front of the current string in the StringBuffer + void prepend(const TCHAR* value, size_t prependedLength); + + ///Contains the length of string in the StringBuffer + ///Public so that analyzers can edit the length directly + int32_t len; + ///Returns the length of the string in the StringBuffer + int32_t length() const; + ///Returns a copy of the current string in the StringBuffer + TCHAR* toString(); + ///Returns a null terminated reference to the StringBuffer's text + TCHAR* getBuffer(); + + + ///reserve a minimum amount of data for the buffer. + ///no change made if the buffer is already longer than length + void reserve(const int32_t length); + private: + ///A buffer that contains strings + TCHAR* buffer; + ///The length of the buffer + int32_t bufferLength; + bool bufferOwner; + + ///Has the buffer grown to a minimum length of minLength or bigger + void growBuffer(const int32_t minLength); + ///Has the buffer grown to a minimum length of minLength or bigger and shifts the + ///current string in buffer by skippingNInitialChars forward + void growBuffer(const int32_t minLength, const int32_t skippingNInitialChars); + + }; +CL_NS_END +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/util/StringIntern.cpp b/src/assistant/3rdparty/clucene/src/CLucene/util/StringIntern.cpp new file mode 100644 index 000000000..cb7a889d1 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/util/StringIntern.cpp @@ -0,0 +1,158 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#include "CLucene/StdHeader.h" +#include "StringIntern.h" +CL_NS_DEF(util) + + __wcsintrntype::iterator wblank; + bool blanksinitd=false; + __wcsintrntype CLStringIntern::stringPool(true); + __strintrntype CLStringIntern::stringaPool(true); + DEFINE_MUTEX(CLStringIntern::THIS_LOCK) + + void CLStringIntern::shutdown(){ + #ifdef _DEBUG + SCOPED_LOCK_MUTEX(THIS_LOCK) + if ( stringaPool.size() > 0 ){ + printf("ERROR: stringaPool still contains intern'd strings (refcounts):\n"); + __strintrntype::iterator itr = stringaPool.begin(); + while ( itr != stringaPool.end() ){ + printf(" %s (%d)\n",(itr->first), (itr->second)); + ++itr; + } + } + + if ( stringPool.size() > 0 ){ + printf("ERROR: stringPool still contains intern'd strings (refcounts):\n"); + __wcsintrntype::iterator itr = stringPool.begin(); + while ( itr != stringPool.end() ){ + _tprintf(_T(" %s (%d)\n"),(itr->first), (itr->second)); + ++itr; + } + } + #endif + } + + const TCHAR* CLStringIntern::intern(const TCHAR* str CL_FILELINEPARAM){ + if ( str == NULL ) + return NULL; + if ( str[0] == 0 ) + return LUCENE_BLANK_STRING; + + SCOPED_LOCK_MUTEX(THIS_LOCK) + + __wcsintrntype::iterator itr = stringPool.find(str); + if ( itr==stringPool.end() ){ +#ifdef _UCS2 + TCHAR* ret = lucenewcsdup(str CL_FILELINEREF); +#else + TCHAR* ret = lucenestrdup(str CL_FILELINEREF); +#endif + stringPool[ret]= 1; + return ret; + }else{ + (itr->second)++; + return itr->first; + } + } + + bool CLStringIntern::unintern(const TCHAR* str){ + if ( str == NULL ) + return false; + if ( str[0] == 0 ) + return false; + + SCOPED_LOCK_MUTEX(THIS_LOCK) + + __wcsintrntype::iterator itr = stringPool.find(str); + if ( itr != stringPool.end() ){ + if ( (itr->second) == 1 ){ + stringPool.removeitr(itr); + return true; + }else + (itr->second)--; + } + return false; + } + + const char* CLStringIntern::internA(const char* str CL_FILELINEPARAM){ + if ( str == NULL ) + return NULL; + if ( str[0] == 0 ) + return _LUCENE_BLANK_ASTRING; + + SCOPED_LOCK_MUTEX(THIS_LOCK) + + __strintrntype::iterator itr = stringaPool.find(str); + if ( itr==stringaPool.end() ){ + char* ret = lucenestrdup(str CL_FILELINE); + stringaPool[ret] = 1; + return ret; + }else{ + (itr->second)++; + return itr->first; + } + } + + bool CLStringIntern::uninternA(const char* str){ + if ( str == NULL ) + return false; + if ( str[0] == 0 ) + return false; + + SCOPED_LOCK_MUTEX(THIS_LOCK) + + __strintrntype::iterator itr = stringaPool.find(str); + if ( itr!=stringaPool.end() ){ + if ( (itr->second) == 1 ){ + stringaPool.removeitr(itr); + return true; + }else + (itr->second)--; + } + return false; + } + + /* removed because of multi-threading problems... + __wcsintrntype::iterator CLStringIntern::internitr(const TCHAR* str CL_FILELINEPARAM){ + if ( str[0] == 0 ){ + if ( !blanksinitd ){ + CLStringIntern::stringPool.put(LUCENE_BLANK_STRING,1); + wblank=stringPool.find(str); + blanksinitd=true; + } + return wblank; + } + __wcsintrntype::iterator itr = stringPool.find(str); + if (itr==stringPool.end()){ +#ifdef _UCS2 + TCHAR* ret = lucenewcsdup(str CL_FILELINEREF); +#else + TCHAR* ret = lucenestrdup(str CL_FILELINEREF); +#endif + stringPool.put(ret,1); + return stringPool.find(str); + }else{ + (itr->second)++; + return itr; + } + } + bool CLStringIntern::uninternitr(__wcsintrntype::iterator itr){ + if ( itr!=stringPool.end() ){ + if ( itr==wblank ) + return false; + if ( (itr->second) == 1 ){ + stringPool.removeitr(itr); + return true; + }else + (itr->second)--; + } + return false; + } +*/ + +CL_NS_END diff --git a/src/assistant/3rdparty/clucene/src/CLucene/util/StringIntern.h b/src/assistant/3rdparty/clucene/src/CLucene/util/StringIntern.h new file mode 100644 index 000000000..ded060c64 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/util/StringIntern.h @@ -0,0 +1,61 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _lucene_util_StringIntern_H +#define _lucene_util_StringIntern_H + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include "VoidMap.h" +CL_NS_DEF(util) +typedef CL_NS(util)::CLHashMap<const TCHAR*,int,CL_NS(util)::Compare::TChar,CL_NS(util)::Equals::TChar,CL_NS(util)::Deletor::tcArray, CL_NS(util)::Deletor::DummyInt32 > __wcsintrntype; +typedef CL_NS(util)::CLHashMap<const char*,int,CL_NS(util)::Compare::Char,CL_NS(util)::Equals::Char,CL_NS(util)::Deletor::acArray, CL_NS(util)::Deletor::DummyInt32 > __strintrntype; + + /** Functions for intern'ing strings. This + * is a process of pooling strings thus using less memory, + * and furthermore allows intern'd strings to be directly + * compared: + * string1==string2, rather than _tcscmp(string1,string2) + */ + class CLStringIntern{ + static __wcsintrntype stringPool; + static __strintrntype stringaPool; + STATIC_DEFINE_MUTEX(THIS_LOCK) + public: + /** + * Internalise the specified string. + * \return Returns a pointer to the internalised string + */ + static const char* internA(const char* str CL_FILELINEPARAM); + /** + * Uninternalise the specified string. Decreases + * the reference count and frees the string if + * reference count is zero + * \returns true if string was destroyed, otherwise false + */ + static bool uninternA(const char* str); + + /** + * Internalise the specified string. + * \return Returns a pointer to the internalised string + */ + static const TCHAR* intern(const TCHAR* str CL_FILELINEPARAM); + + /** + * Uninternalise the specified string. Decreases + * the reference count and frees the string if + * reference count is zero + * \returns true if string was destroyed, otherwise false + */ + static bool unintern(const TCHAR* str); + + static void shutdown(); + }; + +CL_NS_END +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/util/ThreadLocal.cpp b/src/assistant/3rdparty/clucene/src/CLucene/util/ThreadLocal.cpp new file mode 100644 index 000000000..a54c86916 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/util/ThreadLocal.cpp @@ -0,0 +1,55 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#include "CLucene/StdHeader.h" +#include "CLucene/LuceneThreads.h" +#include "ThreadLocal.h" + +CL_NS_DEF(util) + +DEFINE_MUTEX(ThreadLocalBase::ThreadLocalBase_THIS_LOCK) + +ThreadLocalBase::ShutdownHooksType ThreadLocalBase::shutdownHooks(false); +ThreadLocalBase::ThreadLocalsType ThreadLocalBase::threadLocals(false,false); + +ThreadLocalBase::ThreadLocalBase(){ +} +ThreadLocalBase::~ThreadLocalBase(){ +} + +void ThreadLocalBase::UnregisterCurrentThread(){ + _LUCENE_THREADID_TYPE id = _LUCENE_CURRTHREADID; + SCOPED_LOCK_MUTEX(ThreadLocalBase_THIS_LOCK) + + ThreadLocalsType::iterator itr = threadLocals.lower_bound(id); + ThreadLocalsType::iterator end = threadLocals.upper_bound(id); + while ( itr != end ){ + itr->second->setNull(); + ++itr; + } +} +void ThreadLocalBase::shutdown(){ + SCOPED_LOCK_MUTEX(ThreadLocalBase_THIS_LOCK) + + ThreadLocalsType::iterator itr = threadLocals.begin(); + while ( itr != threadLocals.end() ){ + itr->second->setNull(); + ++itr; + } + + ShutdownHooksType::iterator itr2 = shutdownHooks.begin(); + while ( itr2 != shutdownHooks.end() ){ + ShutdownHook* hook = *itr2; + hook(false); + } +} +void ThreadLocalBase::registerShutdownHook(ShutdownHook* hook){ + SCOPED_LOCK_MUTEX(ThreadLocalBase_THIS_LOCK) + shutdownHooks.insert(hook); +} + + +CL_NS_END diff --git a/src/assistant/3rdparty/clucene/src/CLucene/util/ThreadLocal.h b/src/assistant/3rdparty/clucene/src/CLucene/util/ThreadLocal.h new file mode 100644 index 000000000..f67c76ca9 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/util/ThreadLocal.h @@ -0,0 +1,143 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +* +* Changes are Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies). +------------------------------------------------------------------------------*/ +#ifndef _lucene_util_ThreadLocal_H +#define _lucene_util_ThreadLocal_H + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include "CLucene/util/VoidMap.h" + +CL_NS_DEF(util) + +class ThreadLocalBase: LUCENE_BASE{ +public: + /** + * A hook called when CLucene is starting or shutting down, + * this can be used for setting up and tearing down static + * variables + */ + typedef void ShutdownHook(bool startup); + +protected: + STATIC_DEFINE_MUTEX(ThreadLocalBase_THIS_LOCK) + typedef CL_NS(util)::CLMultiMap<_LUCENE_THREADID_TYPE, ThreadLocalBase*, + CL_NS(util)::CLuceneThreadIdCompare, + CL_NS(util)::Deletor::ConstNullVal<_LUCENE_THREADID_TYPE>, + CL_NS(util)::Deletor::ConstNullVal<ThreadLocalBase*> > ThreadLocalsType; + static ThreadLocalsType threadLocals; + //todo: using http://en.wikipedia.org/wiki/Thread-local_storage#Pthreads_implementation + //would work better... but lots of testing would be needed first... + typedef CL_NS(util)::CLSetList<ShutdownHook*, + CL_NS(util)::Compare::Void<ShutdownHook>, + CL_NS(util)::Deletor::ConstNullVal<ShutdownHook*> > ShutdownHooksType; + static ShutdownHooksType shutdownHooks; + + ThreadLocalBase(); +public: + virtual ~ThreadLocalBase(); + + /** + * Call this function to clear the local thread data for this + * ThreadLocal. Calling set(NULL) does the same thing, except + * this function is virtual and can be called without knowing + * the template. + */ + virtual void setNull() = 0; + + /** + * If you want to clean up thread specific memory, then you should + * make sure this thread is called when the thread is not going to be used + * again. This will clean up threadlocal data which can contain quite a lot + * of data, so if you are creating lots of new threads, then it is a good idea + * to use this function, otherwise there will be many memory leaks. + */ + static void UnregisterCurrentThread(); + + /** + * Call this function to shutdown CLucene + */ + static void shutdown(); + + /** + * Add this function to the shutdown hook list. This function will be called + * when CLucene is shutdown. + */ + static void registerShutdownHook(ShutdownHook* hook); +}; + +template<typename T,typename _deletor> +class ThreadLocal: public ThreadLocalBase{ + typedef CL_NS(util)::CLSet<_LUCENE_THREADID_TYPE, T, + CL_NS(util)::CLuceneThreadIdCompare, + CL_NS(util)::Deletor::ConstNullVal<_LUCENE_THREADID_TYPE>, + _deletor > LocalsType; + LocalsType locals; + DEFINE_MUTEX(locals_LOCK) +public: + ThreadLocal(); + ~ThreadLocal(); + T get(); + void setNull(); + void set(T t); +}; + +template<typename T,typename _deletor> +ThreadLocal<T,_deletor>::ThreadLocal(): + locals(false,true) +{ + //add this object to the base's list of threadlocals to be + //notified in case of UnregisterThread() + _LUCENE_THREADID_TYPE id = _LUCENE_CURRTHREADID; + SCOPED_LOCK_MUTEX(ThreadLocalBase_THIS_LOCK) + threadLocals.insert( CL_NS_STD(pair)<const _LUCENE_THREADID_TYPE, ThreadLocalBase*>(id, this) ); +} + +template<typename T,typename _deletor> +ThreadLocal<T,_deletor>::~ThreadLocal(){ + //remove this object to the base's list of threadlocals + _LUCENE_THREADID_TYPE id = _LUCENE_CURRTHREADID; + SCOPED_LOCK_MUTEX(ThreadLocalBase_THIS_LOCK) + + //remove all the thread local data for this object + locals.clear(); + + //remove this object from the ThreadLocalBase threadLocal list + ThreadLocalsType::iterator itr = threadLocals.lower_bound(id); + ThreadLocalsType::iterator end = threadLocals.upper_bound(id); + while ( itr != end ){ + if ( itr->second == this){ + threadLocals.erase(itr); + break; + } + ++itr; + } +} + +template<typename T,typename _deletor> +T ThreadLocal<T,_deletor>::get(){ + return locals.get(_LUCENE_CURRTHREADID); +} + +template<typename T,typename _deletor> +void ThreadLocal<T,_deletor>::setNull(){ + set(NULL); +} + +template<typename T,typename _deletor> +void ThreadLocal<T,_deletor>::set(T t){ + _LUCENE_THREADID_TYPE id = _LUCENE_CURRTHREADID; + locals.remove(id); + if ( t != NULL ) + locals.insert( CL_NS_STD(pair)<const _LUCENE_THREADID_TYPE,T>(id, t) ); +} + +CL_NS_END +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/util/VoidList.h b/src/assistant/3rdparty/clucene/src/CLucene/util/VoidList.h new file mode 100644 index 000000000..cd6908876 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/util/VoidList.h @@ -0,0 +1,175 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _lucene_util_VoidList_ +#define _lucene_util_VoidList_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#include "Equators.h" + +CL_NS_DEF(util) + +/** +* A template to encapsulate various list type classes +* @internal +*/ +template<typename _kt,typename _base,typename _valueDeletor> +class __CLList:public _base,LUCENE_BASE { +private: + bool dv; + typedef _base base; +public: + DEFINE_MUTEX(THIS_LOCK) + + typedef typename _base::const_iterator const_iterator; + typedef typename _base::iterator iterator; + + virtual ~__CLList(){ + clear(); + } + + __CLList ( const bool deleteValue ): + dv(deleteValue) + { + } + + void setDoDelete(bool val){ dv=val; } + + //sets array to the contents of this array. + //array must be size+1, otherwise memory may be overwritten + void toArray(_kt* into) const{ + int i=0; + for ( const_iterator itr=base::begin();itr!=base::end();itr++ ){ + into[i] = *itr; + i++; + } + into[i] = NULL; + } + + void set(int32_t i, _kt val) { + if ( dv ) + _valueDeletor::doDelete((*this)[i]); + (*this)[i] = val; + } + + //todo: check this + void delete_back(){ + if ( base::size() > 0 ){ + iterator itr = base::end(); + if ( itr != base::begin()) + itr --; + _kt key = *itr; + base::erase(itr); + if ( dv ) + _valueDeletor::doDelete(key); + } + } + + void delete_front(){ + if ( base::size() > 0 ){ + iterator itr = base::begin(); + _kt key = *itr; + base::erase(itr); + if ( dv ) + _valueDeletor::doDelete(key); + } + } + + void clear(){ + if ( dv ){ + iterator itr = base::begin(); + while ( itr != base::end() ){ + _valueDeletor::doDelete(*itr); + ++itr; + } + } + base::clear(); + } + + void remove(int32_t i, bool dontDelete=false){ + iterator itr=base::begin(); + itr+=i; + _kt key = *itr; + base::erase( itr ); + if ( dv && !dontDelete ) + _valueDeletor::doDelete(key); + } + void remove(iterator itr, bool dontDelete=false){ + _kt key = *itr; + base::erase( itr ); + if ( dv && !dontDelete ) + _valueDeletor::doDelete(key); + } + +}; + +//growable arrays of Objects (like a collection or list) +//a list, so can contain duplicates +//it grows in chunks... todo: check jlucene for initial size of array, and growfactors +template<typename _kt, typename _valueDeletor=CL_NS(util)::Deletor::Dummy> +class CLVector:public __CLList<_kt, CL_NS_STD(vector)<_kt> , _valueDeletor> +{ +public: + CLVector ( const bool deleteValue=true ): + __CLList<_kt, CL_NS_STD(vector)<_kt> , _valueDeletor>(deleteValue) + { + } +}; + +//An array-backed implementation of the List interface +//a list, so can contain duplicates +//*** a very simple list - use <valarray> +//(This class is roughly equivalent to Vector, except that it is unsynchronized.) +#define CLArrayList CLVector +#define CLHashSet CLHashList + +//implementation of the List interface, provides access to the first and last list elements in O(1) +//no comparator is required... and so can contain duplicates +//a simple list with no comparator +//*** a very simple list - use <list> +#ifdef LUCENE_DISABLE_HASHING + #define CLHashList CLSetList +#else + +template<typename _kt, + typename _Comparator=CL_NS(util)::Compare::TChar, + typename _valueDeletor=CL_NS(util)::Deletor::Dummy> +class CLHashList:public __CLList<_kt, CL_NS_HASHING(hash_set)<_kt,_Comparator> , _valueDeletor> +{ +public: + CLHashList ( const bool deleteValue=true ): + __CLList<_kt, CL_NS_HASHING(hash_set)<_kt,_Comparator> , _valueDeletor>(deleteValue) + { + } +}; +#endif + +template<typename _kt, typename _valueDeletor=CL_NS(util)::Deletor::Dummy> +class CLLinkedList:public __CLList<_kt, CL_NS_STD(list)<_kt> , _valueDeletor> +{ +public: + CLLinkedList ( const bool deleteValue=true ): + __CLList<_kt, CL_NS_STD(list)<_kt> , _valueDeletor>(deleteValue) + { + } +}; +template<typename _kt, + typename _Comparator=CL_NS(util)::Compare::TChar, + typename _valueDeletor=CL_NS(util)::Deletor::Dummy> +class CLSetList:public __CLList<_kt, CL_NS_STD(set)<_kt,_Comparator> , _valueDeletor> +{ +public: + CLSetList ( const bool deleteValue=true ): + __CLList<_kt, CL_NS_STD(set)<_kt,_Comparator> , _valueDeletor>(deleteValue) + { + } +}; + +CL_NS_END +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/util/VoidMap.h b/src/assistant/3rdparty/clucene/src/CLucene/util/VoidMap.h new file mode 100644 index 000000000..b22b507e9 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/util/VoidMap.h @@ -0,0 +1,270 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _lucene_util_VoidMap_ +#define _lucene_util_VoidMap_ + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + + +CL_NS_DEF(util) + +/** +* A template to encapsulate various map type classes +* @internal +*/ +template<typename _kt, typename _vt, + typename _base, + typename _KeyDeletor=CL_NS(util)::Deletor::Dummy, + typename _ValueDeletor=CL_NS(util)::Deletor::Dummy> +class __CLMap:public _base,LUCENE_BASE { +private: + bool dk; + bool dv; + typedef _base base; +public: + DEFINE_MUTEX(THIS_LOCK) + + typedef typename _base::iterator iterator; + typedef typename _base::const_iterator const_iterator; + typedef CL_NS_STD(pair)<_kt, _vt> _pair; + + ///Default constructor for the __CLMap + __CLMap (): + dk(true), + dv(true) + { + } + + ///Deconstructor for the __CLMap + ~__CLMap (){ + clear(); + } + + void setDeleteKey(bool val){ dk = val; } + void setDeleteValue(bool val){ dv = val; } + + ///Construct the VoidMap and set the deleteTypes to the specified values + ///\param deleteKey if true then the key variable is deleted when an object is deleted + ///\param keyDelType delete the key variable using the specified type + ///\param deleteValue if true then the value variable is deleted when an object is deleted + ///\param valueDelType delete the value variable using the specified type + /*__CLMap ( const bool deleteKey, const bool deleteValue ): + dk(deleteKey), + dv(deleteValue) + { + }*/ + + ///checks to see if the specified key exists + ///\param k the key to check for + ///\returns true if the key exists + bool exists(_kt k)const{ + const_iterator itr = base::find(k); + bool ret = itr!=base::end(); + return ret; + } + + ///put the specified pair into the map. remove any old items first + ///\param k the key + ///\param v the value + void put(_kt k,_vt v){ + //todo: check if this is always right! + //must should look through code, for + //cases where map is not unique!!! + if ( dk || dv ) + remove(k); + + //todo: replacing the old item might be quicker... + + base::insert(_pair(k,v)); + } + + + ///using a non-const key, get a non-const value + _vt get( _kt k) const { + const_iterator itr = base::find(k); + if ( itr==base::end() ) + return _vt(); + else + return itr->second; + } + ///using a non-const key, get the actual key + _kt getKey( _kt k) const { + const_iterator itr = base::find(k); + if ( itr==base::end() ) + return _kt(); + else + return itr->first; + } + + void removeitr (iterator itr, const bool dontDeleteKey = false, const bool dontDeleteValue = false){ + //delete key&val first. This prevents potential loops (deleting object removes itself) + _kt key = itr->first; + _vt val = itr->second; + base::erase(itr); + + //keys & vals need to be deleted after erase, because the hashvalue is still needed + if ( dk && !dontDeleteKey ) + _KeyDeletor::doDelete(key); + if ( dv && !dontDeleteValue ) + _ValueDeletor::doDelete(val); + } + ///delete and optionally delete the specified key and associated value + void remove(_kt key, const bool dontDeleteKey = false, const bool dontDeleteValue = false){ + iterator itr = base::find(key); + if ( itr!=base::end() ) + removeitr(itr,dontDeleteKey,dontDeleteValue); + } + + ///clear all keys and values in the map + void clear(){ + if ( dk || dv ){ + iterator itr = base::begin(); + while ( itr!=base::end() ){ + #ifdef _CL_HAVE_EXT_HASH_MAP + removeitr(itr); + itr = base::begin(); + + #else + if ( dk ) + _KeyDeletor::doDelete(itr->first); + if ( dv ) + _ValueDeletor::doDelete(itr->second); + ++itr; + + #endif + } + } + base::clear(); + } +}; + +// makes no guarantees as to the order of the map +// cannot contain duplicate keys; each key can map to at most one value +#define CLHashtable CLHashMap + +#if defined(_CL_HAVE_GOOGLE_DENSE_HASH_MAP) +//do nothing +#elif defined(LUCENE_DISABLE_HASHING) + + //a CLSet with CLHashMap traits +template<typename _kt, typename _vt, + typename _Compare, + typename _EqualDummy, + typename _KeyDeletor=CL_NS(util)::Deletor::Dummy, + typename _ValueDeletor=CL_NS(util)::Deletor::Dummy> +class CLHashMap:public __CLMap<_kt,_vt, + CL_NS_STD(map)<_kt,_vt, _Compare>, + _KeyDeletor,_ValueDeletor> +{ + typedef typename CL_NS_STD(map)<_kt,_vt,_Compare> _base; + typedef __CLMap<_kt, _vt, CL_NS_STD(map)<_kt,_vt, _Compare>, + _KeyDeletor,_ValueDeletor> _this; +public: + CLHashMap ( const bool deleteKey=false, const bool deleteValue=false ) + { + _this::setDeleteKey(deleteKey); + _this::setDeleteValue(deleteValue); + } +}; +#elif defined(_CL_HAVE_EXT_HASH_MAP) + //ext/hash_map syntax +//HashMap class is roughly equivalent to Hashtable, except that it is unsynchronized +template<typename _kt, typename _vt, + typename _Hasher, + typename _Equals, + typename _KeyDeletor=CL_NS(util)::Deletor::Dummy, + typename _ValueDeletor=CL_NS(util)::Deletor::Dummy> +class CLHashMap:public __CLMap<_kt,_vt, + CL_NS_HASHING(hash_map)<_kt,_vt, _Hasher,_Equals>, + _KeyDeletor,_ValueDeletor> +{ + typedef __CLMap<_kt,_vt, CL_NS_HASHING(hash_map)<_kt,_vt, _Hasher,_Equals>, + _KeyDeletor,_ValueDeletor> _this; +public: + CLHashMap ( const bool deleteKey=false, const bool deleteValue=false ) + { + _this::setDeleteKey(deleteKey); + _this::setDeleteValue(deleteValue); + } +}; + +#else +//HashMap class is roughly equivalent to Hashtable, except that it is unsynchronized +template<typename _kt, typename _vt, + typename _Hasher, + typename _Equals, + typename _KeyDeletor=CL_NS(util)::Deletor::Dummy, + typename _ValueDeletor=CL_NS(util)::Deletor::Dummy> +class CLHashMap:public __CLMap<_kt,_vt, + CL_NS_HASHING(hash_map)<_kt,_vt, _Hasher>, + _KeyDeletor,_ValueDeletor> +{ + typedef __CLMap<_kt,_vt, CL_NS_HASHING(hash_map)<_kt,_vt, _Hasher>, + _KeyDeletor,_ValueDeletor> _this; +public: + CLHashMap ( const bool deleteKey=false, const bool deleteValue=false ) + { + _this::setDeleteKey(deleteKey); + _this::setDeleteValue(deleteValue); + } +}; +#endif + +//A collection that contains no duplicates +//does not guarantee that the order will remain constant over time +template<typename _kt, typename _vt, + typename _Compare, + typename _KeyDeletor=CL_NS(util)::Deletor::Dummy, + typename _ValueDeletor=CL_NS(util)::Deletor::Dummy> +class CLSet:public __CLMap<_kt,_vt, + CL_NS_STD(map)<_kt,_vt, _Compare>, + _KeyDeletor,_ValueDeletor> +{ + typedef typename CL_NS_STD(map)<_kt,_vt,_Compare> _base; + typedef __CLMap<_kt, _vt, CL_NS_STD(map)<_kt,_vt, _Compare>, + _KeyDeletor,_ValueDeletor> _this; +public: + CLSet ( const bool deleteKey=false, const bool deleteValue=false ) + { + _this::setDeleteKey(deleteKey); + _this::setDeleteValue(deleteValue); + } +}; + + +//A collection that can contains duplicates +template<typename _kt, typename _vt, + typename _Compare, + typename _KeyDeletor=CL_NS(util)::Deletor::Dummy, + typename _ValueDeletor=CL_NS(util)::Deletor::Dummy> +class CLMultiMap:public __CLMap<_kt,_vt, + CL_NS_STD(multimap)<_kt,_vt>, + _KeyDeletor,_ValueDeletor> +{ + typedef typename CL_NS_STD(multimap)<_kt,_vt> _base; + typedef __CLMap<_kt, _vt, CL_NS_STD(multimap)<_kt,_vt>, + _KeyDeletor,_ValueDeletor> _this; +public: + CLMultiMap ( const bool deleteKey=false, const bool deleteValue=false ) + { + _this::setDeleteKey(deleteKey); + _this::setDeleteValue(deleteValue); + } +}; + + +//*** need to create a class that allows duplicates - use <set> +//#define CLSet __CLMap +CL_NS_END + +#ifdef _CL_HAVE_GOOGLE_DENSE_HASH_MAP +#include "GoogleSparseMap.h" +#endif + +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/util/bufferedstream.h b/src/assistant/3rdparty/clucene/src/CLucene/util/bufferedstream.h new file mode 100644 index 000000000..d905955b1 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/util/bufferedstream.h @@ -0,0 +1,157 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Jos van den Oever +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +/* This file is part of Strigi Desktop Search + * + * Copyright (C) 2006 Jos van den Oever <jos@vandenoever.info> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public License + * along with this library; see the file COPYING.LIB. If not, write to + * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ +#ifndef BUFFEREDSTREAM_H +#define BUFFEREDSTREAM_H + +#include "streambase.h" +#include "inputstreambuffer.h" + +#include <cassert> +#include <stdio.h> + +namespace jstreams { + +template <class T> +class BufferedInputStream : public StreamBase<T> { +private: + bool finishedWritingToBuffer; + InputStreamBuffer<T> buffer; + + void writeToBuffer(int32_t minsize); + int32_t read_(const T*& start, int32_t min, int32_t max); +protected: + /** + * This function must be implemented by the subclasses. + * It should write a maximum of @p space characters at the buffer + * position pointed to by @p start. If no more data is available due to + * end of file, -1 should be returned. If an error occurs, the status + * should be set to Error, an error message should be set and the function + * must return -1. + **/ + virtual int32_t fillBuffer(T* start, int32_t space) = 0; + // this function might be useful if you want to reuse a bufferedstream + void resetBuffer() {printf("implement 'resetBuffer'\n");} + BufferedInputStream<T>(); +public: + int32_t read(const T*& start, int32_t min, int32_t max); + int64_t reset(int64_t); + virtual int64_t skip(int64_t ntoskip); +}; + +template <class T> +BufferedInputStream<T>::BufferedInputStream() { + finishedWritingToBuffer = false; +} + +template <class T> +void +BufferedInputStream<T>::writeToBuffer(int32_t ntoread) { + int32_t missing = ntoread - buffer.avail; + int32_t nwritten = 0; + while (missing > 0 && nwritten >= 0) { + int32_t space; + space = buffer.makeSpace(missing); + T* start = buffer.readPos + buffer.avail; + nwritten = fillBuffer(start, space); + assert(StreamBase<T>::status != Eof); + if (nwritten > 0) { + buffer.avail += nwritten; + missing = ntoread - buffer.avail; + } + } + if (nwritten < 0) { + finishedWritingToBuffer = true; + } +} +template <class T> +int32_t +BufferedInputStream<T>::read(const T*& start, int32_t min, int32_t max) { + if (StreamBase<T>::status == Error) return -2; + if (StreamBase<T>::status == Eof) return -1; + + // do we need to read data into the buffer? + if (!finishedWritingToBuffer && min > buffer.avail) { + // do we have enough space in the buffer? + writeToBuffer(min); + if (StreamBase<T>::status == Error) return -2; + } + + int32_t nread = buffer.read(start, max); + + BufferedInputStream<T>::position += nread; + if (BufferedInputStream<T>::position > BufferedInputStream<T>::size + && BufferedInputStream<T>::size > 0) { + // error: we read more than was specified in size + // this is an error because all dependent code might have been labouring + // under a misapprehension + BufferedInputStream<T>::status = Error; + BufferedInputStream<T>::error = "Stream is longer than specified."; + nread = -2; + } else if (BufferedInputStream<T>::status == Ok && buffer.avail == 0 + && finishedWritingToBuffer) { + BufferedInputStream<T>::status = Eof; + if (BufferedInputStream<T>::size == -1) { + BufferedInputStream<T>::size = BufferedInputStream<T>::position; + } + // save one call to read() by already returning -1 if no data is there + if (nread == 0) nread = -1; + } + return nread; +} +template <class T> +int64_t +BufferedInputStream<T>::reset(int64_t newpos) { + if (StreamBase<T>::status == Error) return -2; + // check to see if we have this position + int64_t d = BufferedInputStream<T>::position - newpos; + if (buffer.readPos - d >= buffer.start && -d < buffer.avail) { + BufferedInputStream<T>::position -= d; + buffer.avail += (int32_t)d; + buffer.readPos -= d; + StreamBase<T>::status = Ok; + } + return StreamBase<T>::position; +} +template <class T> +int64_t +BufferedInputStream<T>::skip(int64_t ntoskip) { + const T *begin; + int32_t nread; + int64_t skipped = 0; + while (ntoskip) { + int32_t step = (int32_t)((ntoskip > buffer.size) ?buffer.size :ntoskip); + nread = read(begin, 1, step); + if (nread <= 0) { + return skipped; + } + ntoskip -= nread; + skipped += nread; + } + return skipped; +} +} + +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/util/dirent.cpp b/src/assistant/3rdparty/clucene/src/CLucene/util/dirent.cpp new file mode 100644 index 000000000..3c5c54200 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/util/dirent.cpp @@ -0,0 +1,221 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Matt J. Weinstein +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#include "CLucene/StdHeader.h" + +#if !defined(_CL_HAVE_DIRENT_H) && !defined(_CL_HAVE_SYS_NDIR_H) && !defined(_CL_HAVE_SYS_DIR_H) && !defined(_CL_HAVE_NDIR_H) +#include "dirent.h" + +DIR * +opendir (const char *szPath) +{ + DIR *nd; + char szFullPath[CL_MAX_PATH]; + + errno = 0; + + if (!szPath) + { + errno = EFAULT; + return NULL; + } + + if (szPath[0] == '\0') + { + errno = ENOTDIR; + return NULL; + } + + /* Attempt to determine if the given path really is a directory. */ + struct _stat rcs; + if ( _stat(szPath,&rcs) == -1) + { + /* call GetLastError for more error info */ + errno = ENOENT; + return NULL; + } + if (!(rcs.st_mode & _S_IFDIR)) + { + /* Error, entry exists but not a directory. */ + errno = ENOTDIR; + return NULL; + } + + /* Make an absolute pathname. */ + _realpath(szPath,szFullPath); + + /* Allocate enough space to store DIR structure and the complete + * directory path given. */ + //nd = (DIR *) malloc (sizeof (DIR) + _tcslen (szFullPath) + _tcslen (DIRENT_SLASH) + + // _tcslen (DIRENT_SEARCH_SUFFIX)+1); + nd = new DIR; + + if (!nd) + { + /* Error, out of memory. */ + errno = ENOMEM; + return NULL; + } + + /* Create the search expression. */ + strcpy (nd->dd_name, szFullPath); + + /* Add on a slash if the path does not end with one. */ + if (nd->dd_name[0] != '\0' && + nd->dd_name[strlen (nd->dd_name) - 1] != '/' && + nd->dd_name[strlen (nd->dd_name) - 1] != '\\') + { + strcat (nd->dd_name, DIRENT_SLASH); + } + + /* Add on the search pattern */ + strcat (nd->dd_name, DIRENT_SEARCH_SUFFIX); + + /* Initialize handle to -1 so that a premature closedir doesn't try + * to call _findclose on it. */ + nd->dd_handle = -1; + + /* Initialize the status. */ + nd->dd_stat = 0; + + /* Initialize the dirent structure. ino and reclen are invalid under + * Win32, and name simply points at the appropriate part of the + * findfirst_t structure. */ + //nd->dd_dir.d_ino = 0; + //nd->dd_dir.d_reclen = 0; + nd->dd_dir.d_namlen = 0; + nd->dd_dir.d_name = nd->dd_dta.name; + + return nd; +} + + +struct dirent * readdir (DIR * dirp) +{ + errno = 0; + + /* Check for valid DIR struct. */ + if (!dirp) + { + errno = EFAULT; + return NULL; + } + + if (dirp->dd_dir.d_name != dirp->dd_dta.name) + { + /* The structure does not seem to be set up correctly. */ + errno = EINVAL; + return NULL; + } + + bool bCallFindNext = true; + + if (dirp->dd_stat < 0) + { + /* We have already returned all files in the directory + * (or the structure has an invalid dd_stat). */ + return NULL; + } + else if (dirp->dd_stat == 0) + { + /* We haven't started the search yet. */ + /* Start the search */ + dirp->dd_handle = _findfirst (dirp->dd_name, &(dirp->dd_dta)); + + if (dirp->dd_handle == -1) + { + /* Whoops! Seems there are no files in that + * directory. */ + dirp->dd_stat = -1; + } + else + { + dirp->dd_stat = 1; + } + + /* Dont call _findnext first time. */ + bCallFindNext = false; + } + + while (dirp->dd_stat > 0) + { + if (bCallFindNext) + { + /* Get the next search entry. */ + if (_findnext (dirp->dd_handle, &(dirp->dd_dta))) + { + /* We are off the end or otherwise error. */ + _findclose (dirp->dd_handle); + dirp->dd_handle = -1; + dirp->dd_stat = -1; + return NULL; + } + else + { + /* Update the status to indicate the correct + * number. */ + dirp->dd_stat++; + } + } + + /* Successfully got an entry. Everything about the file is + * already appropriately filled in except the length of the + * file name. */ + dirp->dd_dir.d_namlen = strlen (dirp->dd_dir.d_name); + + bool bThisFolderOrUpFolder = dirp->dd_dir.d_name[0] == '.' && + (dirp->dd_dir.d_name[1] == 0 || (dirp->dd_dir.d_name[1] == '.' && dirp->dd_dir.d_name[2] == 0)); + + if (!bThisFolderOrUpFolder) + { + struct _stat buf; + char buffer[CL_MAX_DIR]; + size_t bl = strlen(dirp->dd_name)-strlen(DIRENT_SEARCH_SUFFIX); + strncpy(buffer,dirp->dd_name,bl); + buffer[bl]=0; + strcat(buffer, dirp->dd_dir.d_name); + if ( _stat(buffer,&buf) == 0 ) + { + /* Finally we have a valid entry. */ + return &dirp->dd_dir; + } + } + + /* Allow to find next file. */ + bCallFindNext = true; + } + + return NULL; +} + + + +int32_t +closedir (DIR * dirp) +{ + int32_t rc; + + errno = 0; + rc = 0; + + if (!dirp) + { + errno = EFAULT; + return -1; + } + + if (dirp->dd_handle != -1) + { + rc = _findclose (dirp->dd_handle); + } + + /* Delete the dir structure. */ + _CLVDELETE(dirp); + + return rc; +} +#endif //HAVE_DIRENT_H + diff --git a/src/assistant/3rdparty/clucene/src/CLucene/util/dirent.h b/src/assistant/3rdparty/clucene/src/CLucene/util/dirent.h new file mode 100644 index 000000000..71cd34c0a --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/util/dirent.h @@ -0,0 +1,105 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Matt J. Weinstein +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef lucene_util_dirent_H +#define lucene_util_dirent_H + +#if defined(_LUCENE_PRAGMA_ONCE) +# pragma once +#endif + +#if !defined(_CL_HAVE_DIRENT_H) && !defined(_CL_HAVE_SYS_NDIR_H) && !defined(_CL_HAVE_SYS_DIR_H) && !defined(_CL_HAVE_NDIR_H) + +/** +\unit + * dirent.c + * + * Derived from DIRLIB.C by Matt J. Weinstein + * This note appears in the DIRLIB.H + * DIRLIB.H by M. J. Weinstein Released to public domain 1-Jan-89 + * + * Updated by Jeremy Bettis <jeremy@hksys.com> + * Significantly revised and rewinddir, seekdir and telldir added by Colin + * Cut down again & changed by Ben van Klinken + * Peters <colin@fu.is.saga-u.ac.jp> + * + */ + +/** dirent structure - used by the dirent.h directory iteration functions */ +struct dirent +{ + unsigned short d_namlen; /* Length of name in d_name. */ + char *d_name; /* File name. */ +}; + +/** DIR structure - used by the dirent.h directory iteration functions*/ +struct DIR +{ + /** disk transfer area for this dir */ + struct _finddata_t dd_dta; + + /* dirent struct to return from dir (NOTE: this makes this thread + * safe as long as only one thread uses a particular DIR struct at + * a time) */ + struct dirent dd_dir; + + /** _findnext handle */ + intptr_t dd_handle; + + /** + * Status of search: + * 0 = not started yet (next entry to read is first entry) + * -1 = off the end + * positive = 0 based index of next entry + */ + int32_t dd_stat; + + /** given path for dir with search pattern (struct is extended) */ + char dd_name[CL_MAX_DIR]; + +}; + +#define DIRENT_SEARCH_SUFFIX "*" +#define DIRENT_SLASH PATH_DELIMITERA + + +/** +* Returns a pointer to a DIR structure appropriately filled in to begin +* searching a directory. +*/ +DIR* opendir (const char* filespec); + +/** +* Return a pointer to a dirent structure filled with the information on the +* next entry in the directory. +*/ +struct dirent* readdir (DIR* dir); + +/** +* Frees up resources allocated by opendir. +*/ +int32_t closedir (DIR* dir); + + +#elif defined (_CL_HAVE_DIRENT_H) +# include <dirent.h> +# define NAMLEN(dirent) strlen((dirent)->d_name) + +#else +# define dirent direct +# define NAMLEN(dirent) (dirent)->d_namlen +# if defined(_CL_HAVE_SYS_NDIR_H) +# include <sys/ndir.h> +# endif +# if defined(_CL_HHAVE_SYS_DIR_H) +# include <sys/dir.h> +# endif +# if defined(_CL_HHAVE_NDIR_H) +# include <ndir.h> +# endif + +#endif //HAVE_DIRENT_H +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/util/fileinputstream.cpp b/src/assistant/3rdparty/clucene/src/CLucene/util/fileinputstream.cpp new file mode 100644 index 000000000..9125d8478 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/util/fileinputstream.cpp @@ -0,0 +1,103 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Jos van den Oever +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +/* This file is part of Strigi Desktop Search + * + * Copyright (C) 2006 Jos van den Oever <jos@vandenoever.info> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public License + * along with this library; see the file COPYING.LIB. If not, write to + * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ +#include "jstreamsconfig.h" +#include "fileinputstream.h" + +#ifndef UNDER_CE +#include <cerrno> +#endif +#include <cstring> +namespace jstreams { + +const int32_t FileInputStream::defaultBufferSize = 1048576; +FileInputStream::FileInputStream(const char *filepath, int32_t buffersize) { + // try to open the file for reading + file = fopen(filepath, "rb"); + this->filepath = filepath; + if (file == 0) { + // handle error + error = "Could not read file '"; + error += filepath; + error += "': "; +#ifndef UNDER_CE + error += strerror(errno); +#endif + status = Error; + return; + } + // determine file size. if the stream is not seekable, the size will be -1 + fseek(file, 0, SEEK_END); + size = ftell(file); + fseek(file, 0, SEEK_SET); + + // if the file has size 0, make sure that it's really empty + // this is useful for filesystems like /proc that report files as size 0 + // for files that do contain content + if (size == 0) { + char dummy[1]; + size_t n = fread(dummy, 1, 1, file); + if (n == 1) { + size = -1; + fseek(file, 0, SEEK_SET); + } else { + fclose(file); + file = 0; + return; + } + } + + // allocate memory in the buffer + int32_t bufsize = (size <= buffersize) ?size+1 :buffersize; + mark(bufsize); +} +FileInputStream::~FileInputStream() { + if (file) { + if (fclose(file)) { + // handle error + error = "Could not close file '" + filepath + "'."; + } + } +} +int32_t +FileInputStream::fillBuffer(char* start, int32_t space) { + if (file == 0) return -1; + // read into the buffer + int32_t nwritten = fread(start, 1, space, file); + // check the file stream status + if (ferror(file)) { + error = "Could not read from file '" + filepath + "'."; + fclose(file); + file = 0; + status = Error; + return -1; + } + if (feof(file)) { + fclose(file); + file = 0; + } + return nwritten; +} +} diff --git a/src/assistant/3rdparty/clucene/src/CLucene/util/fileinputstream.h b/src/assistant/3rdparty/clucene/src/CLucene/util/fileinputstream.h new file mode 100644 index 000000000..144423da8 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/util/fileinputstream.h @@ -0,0 +1,38 @@ +/** + * Copyright 2003-2006 The Apache Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef FILEINPUTSTREAM_H +#define FILEINPUTSTREAM_H + +#include "bufferedstream.h" + +namespace jstreams { + +class FileInputStream : public BufferedInputStream<char> { +private: + FILE *file; + std::string filepath; + +public: + static const int32_t defaultBufferSize; + FileInputStream(const char *filepath, int32_t buffersize=defaultBufferSize); + ~FileInputStream(); + int32_t fillBuffer(char* start, int32_t space); +}; + +} // end namespace jstreams + +#endif + diff --git a/src/assistant/3rdparty/clucene/src/CLucene/util/inputstreambuffer.h b/src/assistant/3rdparty/clucene/src/CLucene/util/inputstreambuffer.h new file mode 100644 index 000000000..873e811cd --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/util/inputstreambuffer.h @@ -0,0 +1,126 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Jos van den Oever +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +/* This file is part of Strigi Desktop Search + * + * Copyright (C) 2006 Jos van den Oever <jos@vandenoever.info> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public License + * along with this library; see the file COPYING.LIB. If not, write to + * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ +#ifndef INPUTSTREAMBUFFER_H +#define INPUTSTREAMBUFFER_H + +#include <cstdlib> + +namespace jstreams { + +template <class T> +class InputStreamBuffer { +private: +public: + T* start; + int32_t size; + T* readPos; + int32_t avail; + + InputStreamBuffer(); + ~InputStreamBuffer(); + void setSize(int32_t size); + int32_t read(const T*& start, int32_t max=0); + + /** + * This function prepares the buffer for a new write. + * returns the number of available places. + **/ + int32_t makeSpace(int32_t needed); +}; + +template <class T> +InputStreamBuffer<T>::InputStreamBuffer() { + readPos = start = 0; + size = avail = 0; +} +template <class T> +InputStreamBuffer<T>::~InputStreamBuffer() { + free(start); +} +template <class T> +void +InputStreamBuffer<T>::setSize(int32_t size) { + // store pointer information + int32_t offset = (int32_t)(readPos - start); + + // allocate memory in the buffer + if ( start == 0 ) + start = (T*)malloc(size*sizeof(T)); + else + start = (T*)realloc(start, size*sizeof(T)); + this->size = size; + + // restore pointer information + readPos = start + offset; +} +template <class T> +int32_t +InputStreamBuffer<T>::makeSpace(int32_t needed) { + // determine how much space is available for writing + int32_t space = size - ((int32_t)(readPos - start)) - avail; + if (space >= needed) { + // there's enough space + return space; + } + + if (avail) { + if (readPos != start) { +// printf("moving\n"); + // move data to the start of the buffer + memmove(start, readPos, avail*sizeof(T)); + space += (int32_t)(readPos - start); + readPos = start; + } + } else { + // we may start writing at the start of the buffer + readPos = start; + space = size; + } + if (space >= needed) { + // there's enough space now + return space; + } + + // still not enough space, we have to allocate more +// printf("resize %i %i %i %i %i\n", avail, needed, space, size + needed - space, size); + setSize(size + needed - space); + return needed; +} +template <class T> +int32_t +InputStreamBuffer<T>::read(const T*& start, int32_t max) { + start = readPos; + if (max <= 0 || max > avail) { + max = avail; + } + readPos += max; + avail -= max; + return max; +} + +} // end namespace jstreams + +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/util/jstreamsconfig.h b/src/assistant/3rdparty/clucene/src/CLucene/util/jstreamsconfig.h new file mode 100644 index 000000000..2a6ce9f8d --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/util/jstreamsconfig.h @@ -0,0 +1,9 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ + +//this is just a compatibility header for jstreams +#include "CLucene/StdHeader.h" diff --git a/src/assistant/3rdparty/clucene/src/CLucene/util/streambase.h b/src/assistant/3rdparty/clucene/src/CLucene/util/streambase.h new file mode 100644 index 000000000..b0d9dc167 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/util/streambase.h @@ -0,0 +1,148 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Jos van den Oever +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +* +* Changes are Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies). +------------------------------------------------------------------------------*/ +/* This file is part of Strigi Desktop Search + * + * Copyright (C) 2006 Jos van den Oever <jos@vandenoever.info> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public License + * along with this library; see the file COPYING.LIB. If not, write to + * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ +#ifndef STREAMBASE_H +#define STREAMBASE_H + +#include <string> + +#if defined(_BUILD_FOR_QT_) + #include "StdHeader.h" +#endif + +#define INT32MAX 0x7FFFFFFFL + +namespace jstreams { + +enum StreamStatus { Ok, Eof, Error }; + +/** + * @short Base class for stream read access to many different file types. + * + * This class is based on the interface java.io.InputStream. It allows + * for uniform access to streamed resources. + * The main difference with the java equivalent is a performance improvement. + * When reading data, data is not copied into a buffer provided by the caller, + * but a pointer to the read data is provided. This makes this interface + * especially useful for deriving from it and implementing filterers or + * transformers. + */ +// java mapping: long=int64, int=int32, byte=uint8_t +template <class T> +class StreamBase { +protected: + int64_t size; + int64_t position; + std::string error; + StreamStatus status; +public: + StreamBase() :size(-1), position(0), status(Ok){ } + virtual ~StreamBase(){} + /** + * @brief Return a string representation of the last error. + * If no error has occurred, an empty string is returned. + **/ + const char* getError() const { return error.c_str(); } + StreamStatus getStatus() const { return status; } + /** + * @brief Get the current position in the stream. + * The value obtained from this function can be used to reset the stream. + **/ + int64_t getPosition() const { return position; } + /** + * @brief Return the size of the stream. + * If the size of the stream is unknown, -1 + * is returned. If the end of the stream has been reached the size is + * always known. + **/ + int64_t getSize() const { return size; } + /** + * @brief Reads characters from the stream and sets \a start to + * the first character that was read. + * + * If @p ntoread is @c 0, then at least one character will be read. + * + * @param start Pointer passed by reference that will be set to point to + * the retrieved array of characters. If the end of the stream + * is encountered or an error occurs, the value of @p start + * is undefined. + * @param min The number of characters to read from the stream. + * @param max The maximum number of characters to read from the stream. + * @return the number of characters that were read. If -1 is returned, the + * end of the stream has been reached. If -2 is returned, an error + * has occurred. + **/ + virtual int32_t read(const T*& start, int32_t min, int32_t max) = 0; + /** + * Skip @param ntoskip bytes. Unless an error occurs or the end of file is + * encountered, this amount of bytes is skipped. + * This function returns new position in the stream. + **/ + virtual int64_t skip(int64_t ntoskip); + /** + * @brief Repositions this stream to given requested position. + * Reset is guaranteed to work after a successful call to read(), + * when the new position is in the range of the data returned by read(). + * This means that @p pos must lie between than the position + * corresponding to the start parameter (x) of the read function + * and the position corresponding to the last position in the returned + * buffer (x + nread). + **/ + virtual int64_t reset(int64_t pos) = 0; + int64_t mark(int32_t readlimit) { + int64_t p = getPosition(); + const T* ptr; + read(ptr, readlimit, -1); + return reset(p); + } +}; +#define SKIPSTEP 1024 +template <class T> +int64_t +StreamBase<T>::skip(int64_t ntoskip) { + const T *begin; + int32_t nread; + int64_t skipped = 0; + while (ntoskip) { + int32_t step = (int32_t)((ntoskip > SKIPSTEP) ?SKIPSTEP :ntoskip); + nread = read(begin, 1, step); + if (nread < -1 ) { + // an error occurred + return nread; + } else if (nread < 1) { + ntoskip = 0; + } else { + skipped += nread; + ntoskip -= nread; + } + } + return skipped; +} + +} // end namespace jstreams + +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/util/stringreader.h b/src/assistant/3rdparty/clucene/src/CLucene/util/stringreader.h new file mode 100644 index 000000000..698d07e37 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/util/stringreader.h @@ -0,0 +1,124 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Jos van den Oever +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +/* This file is part of Strigi Desktop Search + * + * Copyright (C) 2006 Jos van den Oever <jos@vandenoever.info> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public License + * along with this library; see the file COPYING.LIB. If not, write to + * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ +#ifndef STRINGREADER_H +#define STRINGREADER_H + +/** + * Author: Jos van den Oever + * Ben van Klinken + **/ + + +#include "streambase.h" + +namespace jstreams { + +template <class T> +class StringReader : public StreamBase<T> { +private: + int64_t markpt; + T* data; + bool dataowner; + StringReader(const StringReader<T>&); + void operator=(const StringReader<T>&); +public: + StringReader(const T* value, int32_t length = -1, bool copy = true); + ~StringReader(); + int32_t read(const T*& start, int32_t min, int32_t max); + int64_t skip(int64_t ntoskip); + int64_t reset(int64_t pos); +}; + +typedef StringReader<char> StringInputStream; + +template <class T> +StringReader<T>::StringReader(const T* value, int32_t length, bool copy) + : markpt(0), dataowner(copy) { + if (length < 0) { + length = 0; + while (value[length] != '\0') { + length++; + } + } + StreamBase<T>::size = length; + if (copy) { + data = new T[length+1]; + size_t s = (size_t)(length*sizeof(T)); + memcpy(data, value, s); + data[length] = 0; + } else { + // casting away const is ok, because we don't write anyway + data = (T*)value; + } +} +template <class T> +StringReader<T>::~StringReader() { + if (dataowner) { + delete [] data; + } +} +template <class T> +int32_t +StringReader<T>::read(const T*& start, int32_t min, int32_t max) { + int64_t left = StreamBase<T>::size - StreamBase<T>::position; + if (left == 0) { + StreamBase<T>::status = Eof; + return -1; + } + if (min < 0) min = 0; + int32_t nread = (int32_t)((max > left || max < 1) ?left :max); + start = data + StreamBase<T>::position; + StreamBase<T>::position += nread; + if (StreamBase<T>::position == StreamBase<T>::size) { + StreamBase<T>::status = Eof; + } + return nread; +} +template <class T> +int64_t +StringReader<T>::skip(int64_t ntoskip) { + const T* start; + return read(start, ntoskip, ntoskip); +} +template <class T> +int64_t +StringReader<T>::reset(int64_t newpos) { + if (newpos < 0) { + StreamBase<T>::status = Ok; + StreamBase<T>::position = 0; + } else if (newpos < StreamBase<T>::size) { + StreamBase<T>::status = Ok; + StreamBase<T>::position = newpos; + } else { + StreamBase<T>::position = StreamBase<T>::size; + StreamBase<T>::status = Eof; + } + return StreamBase<T>::position; +} + +} // end namespace jstreams + +#endif diff --git a/src/assistant/3rdparty/clucene/src/CLucene/util/subinputstream.h b/src/assistant/3rdparty/clucene/src/CLucene/util/subinputstream.h new file mode 100644 index 000000000..8ae3e33c7 --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/util/subinputstream.h @@ -0,0 +1,141 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Jos van den Oever +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +/* This file is part of Strigi Desktop Search + * + * Copyright (C) 2006 Jos van den Oever <jos@vandenoever.info> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public License + * along with this library; see the file COPYING.LIB. If not, write to + * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ +#ifndef SUBINPUTSTREAM_H +#define SUBINPUTSTREAM_H + +#include "streambase.h" + +namespace jstreams { + +template<class T> +class SubInputStream : public StreamBase<T> { +private: + const int64_t offset; + StreamBase<T> *input; +public: + SubInputStream(StreamBase<T> *input, int64_t size=-1); + int32_t read(const T*& start, int32_t min, int32_t max); + int64_t reset(int64_t newpos); + int64_t skip(int64_t ntoskip); +}; +template<class T> +SubInputStream<T>::SubInputStream(StreamBase<T> *i, int64_t length) + : offset(i->getPosition()), input(i) { + assert(length >= -1); +// printf("substream offset: %lli\n", offset); + StreamBase<T>::size = length; +} + +template<class T> +int32_t SubInputStream<T>::read(const T*& start, int32_t min, int32_t max) { + if (StreamBase<T>::size != -1) { + const int64_t left = StreamBase<T>::size - StreamBase<T>::position; + if (left == 0) { + return -1; + } + // restrict the amount of data that can be read + if (max <= 0 || max > left) { + max = (int32_t)left; + } + if (min > max) min = max; + if (left < min) min = (int32_t)left; + } + int32_t nread = input->read(start, min, max); + if (nread < -1) { + fprintf(stderr, "substream too short.\n"); + StreamBase<T>::status = Error; + StreamBase<T>::error = input->getError(); + } else if (nread < min) { + if (StreamBase<T>::size == -1) { + StreamBase<T>::status = Eof; + if (nread > 0) { + StreamBase<T>::position += nread; + StreamBase<T>::size = StreamBase<T>::position; + } + } else { +// fprintf(stderr, "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! nread %i min %i max %i size %lli\n", nread, min, max, size); +// fprintf(stderr, "pos %lli parentpos %lli\n", position, input->getPosition()); +// fprintf(stderr, "status: %i error: %s\n", input->getStatus(), input->getError()); + // we expected data but didn't get enough so that's an error + StreamBase<T>::status = Error; + StreamBase<T>::error = "Premature end of stream\n"; + nread = -2; + } + } else { + StreamBase<T>::position += nread; + if (StreamBase<T>::position == StreamBase<T>::size) { + StreamBase<T>::status = Eof; + } + } + return nread; +} + +template<class T> +int64_t SubInputStream<T>::reset(int64_t newpos) { +// fprintf(stderr, "subreset pos: %lli newpos: %lli offset: %lli\n", position, +// newpos, offset); + StreamBase<T>::position = input->reset(newpos + offset); + if (StreamBase<T>::position < offset) { + printf("###########\n"); + StreamBase<T>::status = Error; + StreamBase<T>::error = input->getError(); + } else { + StreamBase<T>::position -= offset; + StreamBase<T>::status = input->getStatus(); + } + return StreamBase<T>::position; +} + +template<class T> +int64_t SubInputStream<T>::skip(int64_t ntoskip) { +// printf("subskip pos: %lli ntoskip: %lli offset: %lli\n", position, ntoskip, offset); + if (StreamBase<T>::size == StreamBase<T>::position) { + StreamBase<T>::status = Eof; + return -1; + } + if (StreamBase<T>::size != -1) { + const int64_t left = StreamBase<T>::size - StreamBase<T>::position; + // restrict the amount of data that can be skipped + if (ntoskip > left) { + ntoskip = left; + } + } + int64_t skipped = input->skip(ntoskip); + if (input->getStatus() == Error) { + StreamBase<T>::status = Error; + StreamBase<T>::error = input->getError(); + } else { + StreamBase<T>::position += skipped; + if (StreamBase<T>::position == StreamBase<T>::size) { + StreamBase<T>::status = Eof; + } + } + return skipped; +} + +} //end namespace jstreams + +#endif diff --git a/src/assistant/assistant.pro b/src/assistant/assistant.pro index 97196b253..b16a1892b 100644 --- a/src/assistant/assistant.pro +++ b/src/assistant/assistant.pro @@ -1,6 +1,10 @@ TEMPLATE = subdirs CONFIG += ordered -SUBDIRS += lib/fulltextsearch \ - lib \ - tools +SUBDIRS += clucene \ + help \ + assistant \ + qhelpgenerator \ + qcollectiongenerator \ + qhelpconverter + diff --git a/src/assistant/tools/assistant/Info_mac.plist b/src/assistant/assistant/Info_mac.plist index 76369a1c5..76369a1c5 100644 --- a/src/assistant/tools/assistant/Info_mac.plist +++ b/src/assistant/assistant/Info_mac.plist diff --git a/src/assistant/tools/assistant/aboutdialog.cpp b/src/assistant/assistant/aboutdialog.cpp index d306812bb..d306812bb 100644 --- a/src/assistant/tools/assistant/aboutdialog.cpp +++ b/src/assistant/assistant/aboutdialog.cpp diff --git a/src/assistant/tools/assistant/aboutdialog.h b/src/assistant/assistant/aboutdialog.h index 70b315714..70b315714 100644 --- a/src/assistant/tools/assistant/aboutdialog.h +++ b/src/assistant/assistant/aboutdialog.h diff --git a/src/assistant/tools/assistant/assistant.icns b/src/assistant/assistant/assistant.icns Binary files differindex 6291dd397..6291dd397 100644 --- a/src/assistant/tools/assistant/assistant.icns +++ b/src/assistant/assistant/assistant.icns diff --git a/src/assistant/tools/assistant/assistant.ico b/src/assistant/assistant/assistant.ico Binary files differindex 9e1b83f1b..9e1b83f1b 100644 --- a/src/assistant/tools/assistant/assistant.ico +++ b/src/assistant/assistant/assistant.ico diff --git a/src/assistant/tools/assistant/assistant.pro b/src/assistant/assistant/assistant.pro index dc8928bad..37e0282ce 100644 --- a/src/assistant/tools/assistant/assistant.pro +++ b/src/assistant/assistant/assistant.pro @@ -1,4 +1,3 @@ -include(../../../shared/fontpanel/fontpanel.pri) TEMPLATE = app LANGUAGE = C++ TARGET = assistant @@ -10,13 +9,12 @@ TARGET = assistant CONFIG += qt \ warn_on \ help -QT += widgets network help +QT += widgets printsupport network help PROJECTNAME = Assistant -DESTDIR = $$QT.designer.bins -target.path = $$[QT_INSTALL_BINS] -INSTALLS += target + DEPENDPATH += ../shared -INCLUDEPATH += $$QT.help.includes + +include(../../shared/fontpanel/fontpanel.pri) # ## Work around a qmake issue when statically linking to # ## not-yet-installed plugins diff --git a/src/assistant/tools/assistant/assistant.qch b/src/assistant/assistant/assistant.qch Binary files differindex e6d52997b..e6d52997b 100644 --- a/src/assistant/tools/assistant/assistant.qch +++ b/src/assistant/assistant/assistant.qch diff --git a/src/assistant/tools/assistant/assistant.qrc b/src/assistant/assistant/assistant.qrc index dddf1be75..dddf1be75 100644 --- a/src/assistant/tools/assistant/assistant.qrc +++ b/src/assistant/assistant/assistant.qrc diff --git a/src/assistant/tools/assistant/assistant.rc b/src/assistant/assistant/assistant.rc index b1bf97b5c..b1bf97b5c 100644 --- a/src/assistant/tools/assistant/assistant.rc +++ b/src/assistant/assistant/assistant.rc diff --git a/src/assistant/tools/assistant/assistant_images.qrc b/src/assistant/assistant/assistant_images.qrc index b4f25236d..b4f25236d 100644 --- a/src/assistant/tools/assistant/assistant_images.qrc +++ b/src/assistant/assistant/assistant_images.qrc diff --git a/src/assistant/tools/assistant/bookmarkdialog.cpp b/src/assistant/assistant/bookmarkdialog.cpp index e3cd9ac60..e3cd9ac60 100644 --- a/src/assistant/tools/assistant/bookmarkdialog.cpp +++ b/src/assistant/assistant/bookmarkdialog.cpp diff --git a/src/assistant/tools/assistant/bookmarkdialog.h b/src/assistant/assistant/bookmarkdialog.h index dfa65bbf8..dfa65bbf8 100644 --- a/src/assistant/tools/assistant/bookmarkdialog.h +++ b/src/assistant/assistant/bookmarkdialog.h diff --git a/src/assistant/tools/assistant/bookmarkdialog.ui b/src/assistant/assistant/bookmarkdialog.ui index 51315317b..51315317b 100644 --- a/src/assistant/tools/assistant/bookmarkdialog.ui +++ b/src/assistant/assistant/bookmarkdialog.ui diff --git a/src/assistant/tools/assistant/bookmarkfiltermodel.cpp b/src/assistant/assistant/bookmarkfiltermodel.cpp index 412e08e5d..412e08e5d 100644 --- a/src/assistant/tools/assistant/bookmarkfiltermodel.cpp +++ b/src/assistant/assistant/bookmarkfiltermodel.cpp diff --git a/src/assistant/tools/assistant/bookmarkfiltermodel.h b/src/assistant/assistant/bookmarkfiltermodel.h index 310e27512..310e27512 100644 --- a/src/assistant/tools/assistant/bookmarkfiltermodel.h +++ b/src/assistant/assistant/bookmarkfiltermodel.h diff --git a/src/assistant/tools/assistant/bookmarkitem.cpp b/src/assistant/assistant/bookmarkitem.cpp index 8bcf451c1..8bcf451c1 100644 --- a/src/assistant/tools/assistant/bookmarkitem.cpp +++ b/src/assistant/assistant/bookmarkitem.cpp diff --git a/src/assistant/tools/assistant/bookmarkitem.h b/src/assistant/assistant/bookmarkitem.h index 96ce786be..96ce786be 100644 --- a/src/assistant/tools/assistant/bookmarkitem.h +++ b/src/assistant/assistant/bookmarkitem.h diff --git a/src/assistant/tools/assistant/bookmarkmanager.cpp b/src/assistant/assistant/bookmarkmanager.cpp index 29d8567a7..29d8567a7 100644 --- a/src/assistant/tools/assistant/bookmarkmanager.cpp +++ b/src/assistant/assistant/bookmarkmanager.cpp diff --git a/src/assistant/tools/assistant/bookmarkmanager.h b/src/assistant/assistant/bookmarkmanager.h index 0b6275af9..0b6275af9 100644 --- a/src/assistant/tools/assistant/bookmarkmanager.h +++ b/src/assistant/assistant/bookmarkmanager.h diff --git a/src/assistant/tools/assistant/bookmarkmanagerwidget.cpp b/src/assistant/assistant/bookmarkmanagerwidget.cpp index 87a1f65be..87a1f65be 100644 --- a/src/assistant/tools/assistant/bookmarkmanagerwidget.cpp +++ b/src/assistant/assistant/bookmarkmanagerwidget.cpp diff --git a/src/assistant/tools/assistant/bookmarkmanagerwidget.h b/src/assistant/assistant/bookmarkmanagerwidget.h index 5c461b937..5c461b937 100644 --- a/src/assistant/tools/assistant/bookmarkmanagerwidget.h +++ b/src/assistant/assistant/bookmarkmanagerwidget.h diff --git a/src/assistant/tools/assistant/bookmarkmanagerwidget.ui b/src/assistant/assistant/bookmarkmanagerwidget.ui index dc965d94e..dc965d94e 100644 --- a/src/assistant/tools/assistant/bookmarkmanagerwidget.ui +++ b/src/assistant/assistant/bookmarkmanagerwidget.ui diff --git a/src/assistant/tools/assistant/bookmarkmodel.cpp b/src/assistant/assistant/bookmarkmodel.cpp index d27731642..d27731642 100644 --- a/src/assistant/tools/assistant/bookmarkmodel.cpp +++ b/src/assistant/assistant/bookmarkmodel.cpp diff --git a/src/assistant/tools/assistant/bookmarkmodel.h b/src/assistant/assistant/bookmarkmodel.h index e23e9c41a..e23e9c41a 100644 --- a/src/assistant/tools/assistant/bookmarkmodel.h +++ b/src/assistant/assistant/bookmarkmodel.h diff --git a/src/assistant/tools/assistant/bookmarkwidget.ui b/src/assistant/assistant/bookmarkwidget.ui index a31a2779c..a31a2779c 100644 --- a/src/assistant/tools/assistant/bookmarkwidget.ui +++ b/src/assistant/assistant/bookmarkwidget.ui diff --git a/src/assistant/tools/assistant/centralwidget.cpp b/src/assistant/assistant/centralwidget.cpp index da27f033d..5544dcb24 100644 --- a/src/assistant/tools/assistant/centralwidget.cpp +++ b/src/assistant/assistant/centralwidget.cpp @@ -52,10 +52,10 @@ #include <QtGui/QKeyEvent> #include <QtWidgets/QMenu> -#include <QtWidgets/QPageSetupDialog> -#include <QtWidgets/QPrintDialog> -#include <QtWidgets/QPrintPreviewDialog> -#include <QtGui/QPrinter> +#include <QtPrintSupport/QPageSetupDialog> +#include <QtPrintSupport/QPrintDialog> +#include <QtPrintSupport/QPrintPreviewDialog> +#include <QtPrintSupport/QPrinter> #include <QtWidgets/QStackedWidget> #include <QtWidgets/QTextBrowser> #include <QtWidgets/QVBoxLayout> diff --git a/src/assistant/tools/assistant/centralwidget.h b/src/assistant/assistant/centralwidget.h index cdf132a1f..da8a799c8 100644 --- a/src/assistant/tools/assistant/centralwidget.h +++ b/src/assistant/assistant/centralwidget.h @@ -52,6 +52,7 @@ QT_BEGIN_NAMESPACE class FindWidget; class HelpViewer; class QStackedWidget; +class QPrinter; class TabBar : public QTabBar { diff --git a/src/assistant/tools/assistant/cmdlineparser.cpp b/src/assistant/assistant/cmdlineparser.cpp index bc498a9d3..bc498a9d3 100644 --- a/src/assistant/tools/assistant/cmdlineparser.cpp +++ b/src/assistant/assistant/cmdlineparser.cpp diff --git a/src/assistant/tools/assistant/cmdlineparser.h b/src/assistant/assistant/cmdlineparser.h index f45679f66..f45679f66 100644 --- a/src/assistant/tools/assistant/cmdlineparser.h +++ b/src/assistant/assistant/cmdlineparser.h diff --git a/src/assistant/tools/assistant/contentwindow.cpp b/src/assistant/assistant/contentwindow.cpp index 3fb1555e0..3fb1555e0 100644 --- a/src/assistant/tools/assistant/contentwindow.cpp +++ b/src/assistant/assistant/contentwindow.cpp diff --git a/src/assistant/tools/assistant/contentwindow.h b/src/assistant/assistant/contentwindow.h index 9c7f4c24f..9c7f4c24f 100644 --- a/src/assistant/tools/assistant/contentwindow.h +++ b/src/assistant/assistant/contentwindow.h diff --git a/src/assistant/tools/assistant/doc/HOWTO b/src/assistant/assistant/doc/HOWTO index a0143479e..a0143479e 100644 --- a/src/assistant/tools/assistant/doc/HOWTO +++ b/src/assistant/assistant/doc/HOWTO diff --git a/src/assistant/tools/assistant/doc/assistant.qdoc b/src/assistant/assistant/doc/assistant.qdoc index 8bd7432ec..8bd7432ec 100644 --- a/src/assistant/tools/assistant/doc/assistant.qdoc +++ b/src/assistant/assistant/doc/assistant.qdoc diff --git a/src/assistant/tools/assistant/doc/assistant.qdocconf b/src/assistant/assistant/doc/assistant.qdocconf index 4319b03f3..4319b03f3 100644 --- a/src/assistant/tools/assistant/doc/assistant.qdocconf +++ b/src/assistant/assistant/doc/assistant.qdocconf diff --git a/src/assistant/tools/assistant/doc/assistant.qhp b/src/assistant/assistant/doc/assistant.qhp index 7ea4cdd46..7ea4cdd46 100644 --- a/src/assistant/tools/assistant/doc/assistant.qhp +++ b/src/assistant/assistant/doc/assistant.qhp diff --git a/src/assistant/tools/assistant/doc/classic.css b/src/assistant/assistant/doc/classic.css index 911354035..911354035 100644 --- a/src/assistant/tools/assistant/doc/classic.css +++ b/src/assistant/assistant/doc/classic.css diff --git a/src/assistant/tools/assistant/doc/images/assistant-address-toolbar.png b/src/assistant/assistant/doc/images/assistant-address-toolbar.png Binary files differindex 847b7debb..847b7debb 100644 --- a/src/assistant/tools/assistant/doc/images/assistant-address-toolbar.png +++ b/src/assistant/assistant/doc/images/assistant-address-toolbar.png diff --git a/src/assistant/tools/assistant/doc/images/assistant-assistant.png b/src/assistant/assistant/doc/images/assistant-assistant.png Binary files differindex 1ff5cc976..1ff5cc976 100644 --- a/src/assistant/tools/assistant/doc/images/assistant-assistant.png +++ b/src/assistant/assistant/doc/images/assistant-assistant.png diff --git a/src/assistant/tools/assistant/doc/images/assistant-dockwidgets.png b/src/assistant/assistant/doc/images/assistant-dockwidgets.png Binary files differindex 17bc064c5..17bc064c5 100644 --- a/src/assistant/tools/assistant/doc/images/assistant-dockwidgets.png +++ b/src/assistant/assistant/doc/images/assistant-dockwidgets.png diff --git a/src/assistant/tools/assistant/doc/images/assistant-docwindow.png b/src/assistant/assistant/doc/images/assistant-docwindow.png Binary files differindex c5bac581f..c5bac581f 100644 --- a/src/assistant/tools/assistant/doc/images/assistant-docwindow.png +++ b/src/assistant/assistant/doc/images/assistant-docwindow.png diff --git a/src/assistant/tools/assistant/doc/images/assistant-examples.png b/src/assistant/assistant/doc/images/assistant-examples.png Binary files differindex 47c01bcda..47c01bcda 100644 --- a/src/assistant/tools/assistant/doc/images/assistant-examples.png +++ b/src/assistant/assistant/doc/images/assistant-examples.png diff --git a/src/assistant/tools/assistant/doc/images/assistant-filter-toolbar.png b/src/assistant/assistant/doc/images/assistant-filter-toolbar.png Binary files differindex 4e89a79b9..4e89a79b9 100644 --- a/src/assistant/tools/assistant/doc/images/assistant-filter-toolbar.png +++ b/src/assistant/assistant/doc/images/assistant-filter-toolbar.png diff --git a/src/assistant/tools/assistant/doc/images/assistant-preferences-documentation.png b/src/assistant/assistant/doc/images/assistant-preferences-documentation.png Binary files differindex 790fd9a1a..790fd9a1a 100644 --- a/src/assistant/tools/assistant/doc/images/assistant-preferences-documentation.png +++ b/src/assistant/assistant/doc/images/assistant-preferences-documentation.png diff --git a/src/assistant/tools/assistant/doc/images/assistant-preferences-filters.png b/src/assistant/assistant/doc/images/assistant-preferences-filters.png Binary files differindex 7453dd66d..7453dd66d 100644 --- a/src/assistant/tools/assistant/doc/images/assistant-preferences-filters.png +++ b/src/assistant/assistant/doc/images/assistant-preferences-filters.png diff --git a/src/assistant/tools/assistant/doc/images/assistant-preferences-fonts.png b/src/assistant/assistant/doc/images/assistant-preferences-fonts.png Binary files differindex d42d190d0..d42d190d0 100644 --- a/src/assistant/tools/assistant/doc/images/assistant-preferences-fonts.png +++ b/src/assistant/assistant/doc/images/assistant-preferences-fonts.png diff --git a/src/assistant/tools/assistant/doc/images/assistant-preferences-options.png b/src/assistant/assistant/doc/images/assistant-preferences-options.png Binary files differindex d6624151a..d6624151a 100644 --- a/src/assistant/tools/assistant/doc/images/assistant-preferences-options.png +++ b/src/assistant/assistant/doc/images/assistant-preferences-options.png diff --git a/src/assistant/tools/assistant/doc/images/assistant-search.png b/src/assistant/assistant/doc/images/assistant-search.png Binary files differindex ef75c3329..ef75c3329 100644 --- a/src/assistant/tools/assistant/doc/images/assistant-search.png +++ b/src/assistant/assistant/doc/images/assistant-search.png diff --git a/src/assistant/tools/assistant/doc/images/assistant-toolbar.png b/src/assistant/assistant/doc/images/assistant-toolbar.png Binary files differindex 1b41825c6..1b41825c6 100644 --- a/src/assistant/tools/assistant/doc/images/assistant-toolbar.png +++ b/src/assistant/assistant/doc/images/assistant-toolbar.png diff --git a/src/assistant/tools/assistant/filternamedialog.cpp b/src/assistant/assistant/filternamedialog.cpp index d1424ee2a..d1424ee2a 100644 --- a/src/assistant/tools/assistant/filternamedialog.cpp +++ b/src/assistant/assistant/filternamedialog.cpp diff --git a/src/assistant/tools/assistant/filternamedialog.h b/src/assistant/assistant/filternamedialog.h index bf65a7e15..bf65a7e15 100644 --- a/src/assistant/tools/assistant/filternamedialog.h +++ b/src/assistant/assistant/filternamedialog.h diff --git a/src/assistant/tools/assistant/filternamedialog.ui b/src/assistant/assistant/filternamedialog.ui index 755a93479..755a93479 100644 --- a/src/assistant/tools/assistant/filternamedialog.ui +++ b/src/assistant/assistant/filternamedialog.ui diff --git a/src/assistant/tools/assistant/findwidget.cpp b/src/assistant/assistant/findwidget.cpp index 296da0c8d..296da0c8d 100644 --- a/src/assistant/tools/assistant/findwidget.cpp +++ b/src/assistant/assistant/findwidget.cpp diff --git a/src/assistant/tools/assistant/findwidget.h b/src/assistant/assistant/findwidget.h index b24e88019..b24e88019 100644 --- a/src/assistant/tools/assistant/findwidget.h +++ b/src/assistant/assistant/findwidget.h diff --git a/src/assistant/tools/assistant/globalactions.cpp b/src/assistant/assistant/globalactions.cpp index 6b8b8ea29..6b8b8ea29 100644 --- a/src/assistant/tools/assistant/globalactions.cpp +++ b/src/assistant/assistant/globalactions.cpp diff --git a/src/assistant/tools/assistant/globalactions.h b/src/assistant/assistant/globalactions.h index a9059a9a4..a9059a9a4 100644 --- a/src/assistant/tools/assistant/globalactions.h +++ b/src/assistant/assistant/globalactions.h diff --git a/src/assistant/tools/assistant/helpenginewrapper.cpp b/src/assistant/assistant/helpenginewrapper.cpp index 4c7ff5b30..4c7ff5b30 100644 --- a/src/assistant/tools/assistant/helpenginewrapper.cpp +++ b/src/assistant/assistant/helpenginewrapper.cpp diff --git a/src/assistant/tools/assistant/helpenginewrapper.h b/src/assistant/assistant/helpenginewrapper.h index ff825b04a..ff825b04a 100644 --- a/src/assistant/tools/assistant/helpenginewrapper.h +++ b/src/assistant/assistant/helpenginewrapper.h diff --git a/src/assistant/tools/assistant/helpviewer.cpp b/src/assistant/assistant/helpviewer.cpp index 7bed4965a..7bed4965a 100644 --- a/src/assistant/tools/assistant/helpviewer.cpp +++ b/src/assistant/assistant/helpviewer.cpp diff --git a/src/assistant/tools/assistant/helpviewer.h b/src/assistant/assistant/helpviewer.h index a5f4de668..a5f4de668 100644 --- a/src/assistant/tools/assistant/helpviewer.h +++ b/src/assistant/assistant/helpviewer.h diff --git a/src/assistant/tools/assistant/helpviewer_p.h b/src/assistant/assistant/helpviewer_p.h index 8beeb94b2..8beeb94b2 100644 --- a/src/assistant/tools/assistant/helpviewer_p.h +++ b/src/assistant/assistant/helpviewer_p.h diff --git a/src/assistant/tools/assistant/helpviewer_qtb.cpp b/src/assistant/assistant/helpviewer_qtb.cpp index d3d128b74..d3d128b74 100644 --- a/src/assistant/tools/assistant/helpviewer_qtb.cpp +++ b/src/assistant/assistant/helpviewer_qtb.cpp diff --git a/src/assistant/tools/assistant/helpviewer_qwv.cpp b/src/assistant/assistant/helpviewer_qwv.cpp index 662177631..662177631 100644 --- a/src/assistant/tools/assistant/helpviewer_qwv.cpp +++ b/src/assistant/assistant/helpviewer_qwv.cpp diff --git a/src/assistant/tools/assistant/images/assistant-128.png b/src/assistant/assistant/images/assistant-128.png Binary files differindex f05949f6d..f05949f6d 100644 --- a/src/assistant/tools/assistant/images/assistant-128.png +++ b/src/assistant/assistant/images/assistant-128.png diff --git a/src/assistant/tools/assistant/images/assistant.png b/src/assistant/assistant/images/assistant.png Binary files differindex ea4d1e70c..ea4d1e70c 100644 --- a/src/assistant/tools/assistant/images/assistant.png +++ b/src/assistant/assistant/images/assistant.png diff --git a/src/assistant/tools/assistant/images/bookmark.png b/src/assistant/assistant/images/bookmark.png Binary files differindex 57e57e343..57e57e343 100644 --- a/src/assistant/tools/assistant/images/bookmark.png +++ b/src/assistant/assistant/images/bookmark.png diff --git a/src/assistant/tools/assistant/images/closebutton.png b/src/assistant/assistant/images/closebutton.png Binary files differindex c978cf51a..c978cf51a 100644 --- a/src/assistant/tools/assistant/images/closebutton.png +++ b/src/assistant/assistant/images/closebutton.png diff --git a/src/assistant/tools/assistant/images/darkclosebutton.png b/src/assistant/assistant/images/darkclosebutton.png Binary files differindex 1077663b2..1077663b2 100644 --- a/src/assistant/tools/assistant/images/darkclosebutton.png +++ b/src/assistant/assistant/images/darkclosebutton.png diff --git a/src/assistant/tools/assistant/images/mac/addtab.png b/src/assistant/assistant/images/mac/addtab.png Binary files differindex 20928fb40..20928fb40 100644 --- a/src/assistant/tools/assistant/images/mac/addtab.png +++ b/src/assistant/assistant/images/mac/addtab.png diff --git a/src/assistant/tools/assistant/images/mac/book.png b/src/assistant/assistant/images/mac/book.png Binary files differindex 7a3204c87..7a3204c87 100644 --- a/src/assistant/tools/assistant/images/mac/book.png +++ b/src/assistant/assistant/images/mac/book.png diff --git a/src/assistant/tools/assistant/images/mac/closetab.png b/src/assistant/assistant/images/mac/closetab.png Binary files differindex ab9d669ee..ab9d669ee 100644 --- a/src/assistant/tools/assistant/images/mac/closetab.png +++ b/src/assistant/assistant/images/mac/closetab.png diff --git a/src/assistant/tools/assistant/images/mac/editcopy.png b/src/assistant/assistant/images/mac/editcopy.png Binary files differindex f55136446..f55136446 100644 --- a/src/assistant/tools/assistant/images/mac/editcopy.png +++ b/src/assistant/assistant/images/mac/editcopy.png diff --git a/src/assistant/tools/assistant/images/mac/find.png b/src/assistant/assistant/images/mac/find.png Binary files differindex 3561745f0..3561745f0 100644 --- a/src/assistant/tools/assistant/images/mac/find.png +++ b/src/assistant/assistant/images/mac/find.png diff --git a/src/assistant/tools/assistant/images/mac/home.png b/src/assistant/assistant/images/mac/home.png Binary files differindex 78d94da18..78d94da18 100644 --- a/src/assistant/tools/assistant/images/mac/home.png +++ b/src/assistant/assistant/images/mac/home.png diff --git a/src/assistant/tools/assistant/images/mac/next.png b/src/assistant/assistant/images/mac/next.png Binary files differindex a585cab80..a585cab80 100644 --- a/src/assistant/tools/assistant/images/mac/next.png +++ b/src/assistant/assistant/images/mac/next.png diff --git a/src/assistant/tools/assistant/images/mac/previous.png b/src/assistant/assistant/images/mac/previous.png Binary files differindex 612fb34dc..612fb34dc 100644 --- a/src/assistant/tools/assistant/images/mac/previous.png +++ b/src/assistant/assistant/images/mac/previous.png diff --git a/src/assistant/tools/assistant/images/mac/print.png b/src/assistant/assistant/images/mac/print.png Binary files differindex 10ca56c82..10ca56c82 100644 --- a/src/assistant/tools/assistant/images/mac/print.png +++ b/src/assistant/assistant/images/mac/print.png diff --git a/src/assistant/tools/assistant/images/mac/resetzoom.png b/src/assistant/assistant/images/mac/resetzoom.png Binary files differindex 759b38296..759b38296 100644 --- a/src/assistant/tools/assistant/images/mac/resetzoom.png +++ b/src/assistant/assistant/images/mac/resetzoom.png diff --git a/src/assistant/tools/assistant/images/mac/synctoc.png b/src/assistant/assistant/images/mac/synctoc.png Binary files differindex 067fa941b..067fa941b 100644 --- a/src/assistant/tools/assistant/images/mac/synctoc.png +++ b/src/assistant/assistant/images/mac/synctoc.png diff --git a/src/assistant/tools/assistant/images/mac/zoomin.png b/src/assistant/assistant/images/mac/zoomin.png Binary files differindex d46f5aff0..d46f5aff0 100644 --- a/src/assistant/tools/assistant/images/mac/zoomin.png +++ b/src/assistant/assistant/images/mac/zoomin.png diff --git a/src/assistant/tools/assistant/images/mac/zoomout.png b/src/assistant/assistant/images/mac/zoomout.png Binary files differindex 46326566d..46326566d 100644 --- a/src/assistant/tools/assistant/images/mac/zoomout.png +++ b/src/assistant/assistant/images/mac/zoomout.png diff --git a/src/assistant/tools/assistant/images/trolltech-logo.png b/src/assistant/assistant/images/trolltech-logo.png Binary files differindex c53e744ca..c53e744ca 100644 --- a/src/assistant/tools/assistant/images/trolltech-logo.png +++ b/src/assistant/assistant/images/trolltech-logo.png diff --git a/src/assistant/tools/assistant/images/win/addtab.png b/src/assistant/assistant/images/win/addtab.png Binary files differindex 4bb0feb92..4bb0feb92 100644 --- a/src/assistant/tools/assistant/images/win/addtab.png +++ b/src/assistant/assistant/images/win/addtab.png diff --git a/src/assistant/tools/assistant/images/win/book.png b/src/assistant/assistant/images/win/book.png Binary files differindex 09ec4d33f..09ec4d33f 100644 --- a/src/assistant/tools/assistant/images/win/book.png +++ b/src/assistant/assistant/images/win/book.png diff --git a/src/assistant/tools/assistant/images/win/closetab.png b/src/assistant/assistant/images/win/closetab.png Binary files differindex ef9e02086..ef9e02086 100644 --- a/src/assistant/tools/assistant/images/win/closetab.png +++ b/src/assistant/assistant/images/win/closetab.png diff --git a/src/assistant/tools/assistant/images/win/editcopy.png b/src/assistant/assistant/images/win/editcopy.png Binary files differindex 1121b47d8..1121b47d8 100644 --- a/src/assistant/tools/assistant/images/win/editcopy.png +++ b/src/assistant/assistant/images/win/editcopy.png diff --git a/src/assistant/tools/assistant/images/win/find.png b/src/assistant/assistant/images/win/find.png Binary files differindex 6ea35e930..6ea35e930 100644 --- a/src/assistant/tools/assistant/images/win/find.png +++ b/src/assistant/assistant/images/win/find.png diff --git a/src/assistant/tools/assistant/images/win/home.png b/src/assistant/assistant/images/win/home.png Binary files differindex b1c6ae191..b1c6ae191 100644 --- a/src/assistant/tools/assistant/images/win/home.png +++ b/src/assistant/assistant/images/win/home.png diff --git a/src/assistant/tools/assistant/images/win/next.png b/src/assistant/assistant/images/win/next.png Binary files differindex 8df4127a0..8df4127a0 100644 --- a/src/assistant/tools/assistant/images/win/next.png +++ b/src/assistant/assistant/images/win/next.png diff --git a/src/assistant/tools/assistant/images/win/previous.png b/src/assistant/assistant/images/win/previous.png Binary files differindex 0780bc23d..0780bc23d 100644 --- a/src/assistant/tools/assistant/images/win/previous.png +++ b/src/assistant/assistant/images/win/previous.png diff --git a/src/assistant/tools/assistant/images/win/print.png b/src/assistant/assistant/images/win/print.png Binary files differindex ba7c02dc1..ba7c02dc1 100644 --- a/src/assistant/tools/assistant/images/win/print.png +++ b/src/assistant/assistant/images/win/print.png diff --git a/src/assistant/tools/assistant/images/win/resetzoom.png b/src/assistant/assistant/images/win/resetzoom.png Binary files differindex b69ae4e7f..b69ae4e7f 100644 --- a/src/assistant/tools/assistant/images/win/resetzoom.png +++ b/src/assistant/assistant/images/win/resetzoom.png diff --git a/src/assistant/tools/assistant/images/win/synctoc.png b/src/assistant/assistant/images/win/synctoc.png Binary files differindex da301bc59..da301bc59 100644 --- a/src/assistant/tools/assistant/images/win/synctoc.png +++ b/src/assistant/assistant/images/win/synctoc.png diff --git a/src/assistant/tools/assistant/images/win/zoomin.png b/src/assistant/assistant/images/win/zoomin.png Binary files differindex 2e586fc7b..2e586fc7b 100644 --- a/src/assistant/tools/assistant/images/win/zoomin.png +++ b/src/assistant/assistant/images/win/zoomin.png diff --git a/src/assistant/tools/assistant/images/win/zoomout.png b/src/assistant/assistant/images/win/zoomout.png Binary files differindex a736d3934..a736d3934 100644 --- a/src/assistant/tools/assistant/images/win/zoomout.png +++ b/src/assistant/assistant/images/win/zoomout.png diff --git a/src/assistant/tools/assistant/images/wrap.png b/src/assistant/assistant/images/wrap.png Binary files differindex 90f18d9f7..90f18d9f7 100644 --- a/src/assistant/tools/assistant/images/wrap.png +++ b/src/assistant/assistant/images/wrap.png diff --git a/src/assistant/tools/assistant/indexwindow.cpp b/src/assistant/assistant/indexwindow.cpp index 069e821cd..069e821cd 100644 --- a/src/assistant/tools/assistant/indexwindow.cpp +++ b/src/assistant/assistant/indexwindow.cpp diff --git a/src/assistant/tools/assistant/indexwindow.h b/src/assistant/assistant/indexwindow.h index 1873cd6e4..1873cd6e4 100644 --- a/src/assistant/tools/assistant/indexwindow.h +++ b/src/assistant/assistant/indexwindow.h diff --git a/src/assistant/tools/assistant/installdialog.cpp b/src/assistant/assistant/installdialog.cpp index 82c0ccaa9..82c0ccaa9 100644 --- a/src/assistant/tools/assistant/installdialog.cpp +++ b/src/assistant/assistant/installdialog.cpp diff --git a/src/assistant/tools/assistant/installdialog.h b/src/assistant/assistant/installdialog.h index 0f57b9ef0..0f57b9ef0 100644 --- a/src/assistant/tools/assistant/installdialog.h +++ b/src/assistant/assistant/installdialog.h diff --git a/src/assistant/tools/assistant/installdialog.ui b/src/assistant/assistant/installdialog.ui index 21a05da77..21a05da77 100644 --- a/src/assistant/tools/assistant/installdialog.ui +++ b/src/assistant/assistant/installdialog.ui diff --git a/src/assistant/tools/assistant/main.cpp b/src/assistant/assistant/main.cpp index d6e78d473..d6e78d473 100644 --- a/src/assistant/tools/assistant/main.cpp +++ b/src/assistant/assistant/main.cpp diff --git a/src/assistant/tools/assistant/mainwindow.cpp b/src/assistant/assistant/mainwindow.cpp index 18a215ecd..18a215ecd 100644 --- a/src/assistant/tools/assistant/mainwindow.cpp +++ b/src/assistant/assistant/mainwindow.cpp diff --git a/src/assistant/tools/assistant/mainwindow.h b/src/assistant/assistant/mainwindow.h index 1ea0bbaf2..1ea0bbaf2 100644 --- a/src/assistant/tools/assistant/mainwindow.h +++ b/src/assistant/assistant/mainwindow.h diff --git a/src/assistant/tools/assistant/openpagesmanager.cpp b/src/assistant/assistant/openpagesmanager.cpp index 7bafeeafc..7bafeeafc 100644 --- a/src/assistant/tools/assistant/openpagesmanager.cpp +++ b/src/assistant/assistant/openpagesmanager.cpp diff --git a/src/assistant/tools/assistant/openpagesmanager.h b/src/assistant/assistant/openpagesmanager.h index c34686c8c..c34686c8c 100644 --- a/src/assistant/tools/assistant/openpagesmanager.h +++ b/src/assistant/assistant/openpagesmanager.h diff --git a/src/assistant/tools/assistant/openpagesmodel.cpp b/src/assistant/assistant/openpagesmodel.cpp index 3517693f9..3517693f9 100644 --- a/src/assistant/tools/assistant/openpagesmodel.cpp +++ b/src/assistant/assistant/openpagesmodel.cpp diff --git a/src/assistant/tools/assistant/openpagesmodel.h b/src/assistant/assistant/openpagesmodel.h index dd28a7c5f..dd28a7c5f 100644 --- a/src/assistant/tools/assistant/openpagesmodel.h +++ b/src/assistant/assistant/openpagesmodel.h diff --git a/src/assistant/tools/assistant/openpagesswitcher.cpp b/src/assistant/assistant/openpagesswitcher.cpp index b1f70d784..b1f70d784 100644 --- a/src/assistant/tools/assistant/openpagesswitcher.cpp +++ b/src/assistant/assistant/openpagesswitcher.cpp diff --git a/src/assistant/tools/assistant/openpagesswitcher.h b/src/assistant/assistant/openpagesswitcher.h index c3e3a33ce..c3e3a33ce 100644 --- a/src/assistant/tools/assistant/openpagesswitcher.h +++ b/src/assistant/assistant/openpagesswitcher.h diff --git a/src/assistant/tools/assistant/openpageswidget.cpp b/src/assistant/assistant/openpageswidget.cpp index 6947c312c..6947c312c 100644 --- a/src/assistant/tools/assistant/openpageswidget.cpp +++ b/src/assistant/assistant/openpageswidget.cpp diff --git a/src/assistant/tools/assistant/openpageswidget.h b/src/assistant/assistant/openpageswidget.h index e629e4ffe..e629e4ffe 100644 --- a/src/assistant/tools/assistant/openpageswidget.h +++ b/src/assistant/assistant/openpageswidget.h diff --git a/src/assistant/tools/assistant/preferencesdialog.cpp b/src/assistant/assistant/preferencesdialog.cpp index 64fc3c42d..64fc3c42d 100644 --- a/src/assistant/tools/assistant/preferencesdialog.cpp +++ b/src/assistant/assistant/preferencesdialog.cpp diff --git a/src/assistant/tools/assistant/preferencesdialog.h b/src/assistant/assistant/preferencesdialog.h index 04bfcf8c3..04bfcf8c3 100644 --- a/src/assistant/tools/assistant/preferencesdialog.h +++ b/src/assistant/assistant/preferencesdialog.h diff --git a/src/assistant/tools/assistant/preferencesdialog.ui b/src/assistant/assistant/preferencesdialog.ui index 1c6833ad9..1c6833ad9 100644 --- a/src/assistant/tools/assistant/preferencesdialog.ui +++ b/src/assistant/assistant/preferencesdialog.ui diff --git a/src/assistant/tools/assistant/qtdocinstaller.cpp b/src/assistant/assistant/qtdocinstaller.cpp index ffe352c87..ffe352c87 100644 --- a/src/assistant/tools/assistant/qtdocinstaller.cpp +++ b/src/assistant/assistant/qtdocinstaller.cpp diff --git a/src/assistant/tools/assistant/qtdocinstaller.h b/src/assistant/assistant/qtdocinstaller.h index 909df10ce..909df10ce 100644 --- a/src/assistant/tools/assistant/qtdocinstaller.h +++ b/src/assistant/assistant/qtdocinstaller.h diff --git a/src/assistant/tools/assistant/remotecontrol.cpp b/src/assistant/assistant/remotecontrol.cpp index 0990a357f..0990a357f 100644 --- a/src/assistant/tools/assistant/remotecontrol.cpp +++ b/src/assistant/assistant/remotecontrol.cpp diff --git a/src/assistant/tools/assistant/remotecontrol.h b/src/assistant/assistant/remotecontrol.h index 4a185f050..4a185f050 100644 --- a/src/assistant/tools/assistant/remotecontrol.h +++ b/src/assistant/assistant/remotecontrol.h diff --git a/src/assistant/tools/assistant/remotecontrol_win.h b/src/assistant/assistant/remotecontrol_win.h index 807815df1..807815df1 100644 --- a/src/assistant/tools/assistant/remotecontrol_win.h +++ b/src/assistant/assistant/remotecontrol_win.h diff --git a/src/assistant/tools/assistant/searchwidget.cpp b/src/assistant/assistant/searchwidget.cpp index 31b8e739d..31b8e739d 100644 --- a/src/assistant/tools/assistant/searchwidget.cpp +++ b/src/assistant/assistant/searchwidget.cpp diff --git a/src/assistant/tools/assistant/searchwidget.h b/src/assistant/assistant/searchwidget.h index c53498c94..c53498c94 100644 --- a/src/assistant/tools/assistant/searchwidget.h +++ b/src/assistant/assistant/searchwidget.h diff --git a/src/assistant/tools/assistant/topicchooser.cpp b/src/assistant/assistant/topicchooser.cpp index 6f2810bf4..6f2810bf4 100644 --- a/src/assistant/tools/assistant/topicchooser.cpp +++ b/src/assistant/assistant/topicchooser.cpp diff --git a/src/assistant/tools/assistant/topicchooser.h b/src/assistant/assistant/topicchooser.h index 6ac59f008..6ac59f008 100644 --- a/src/assistant/tools/assistant/topicchooser.h +++ b/src/assistant/assistant/topicchooser.h diff --git a/src/assistant/tools/assistant/topicchooser.ui b/src/assistant/assistant/topicchooser.ui index d4c90bb4b..d4c90bb4b 100644 --- a/src/assistant/tools/assistant/topicchooser.ui +++ b/src/assistant/assistant/topicchooser.ui diff --git a/src/assistant/tools/assistant/tracer.h b/src/assistant/assistant/tracer.h index 1ed609acb..1ed609acb 100644 --- a/src/assistant/tools/assistant/tracer.h +++ b/src/assistant/assistant/tracer.h diff --git a/src/assistant/tools/assistant/xbelsupport.cpp b/src/assistant/assistant/xbelsupport.cpp index 2979d10c1..2979d10c1 100644 --- a/src/assistant/tools/assistant/xbelsupport.cpp +++ b/src/assistant/assistant/xbelsupport.cpp diff --git a/src/assistant/tools/assistant/xbelsupport.h b/src/assistant/assistant/xbelsupport.h index 4c2a211ad..f55d3ba82 100644 --- a/src/assistant/tools/assistant/xbelsupport.h +++ b/src/assistant/assistant/xbelsupport.h @@ -42,7 +42,7 @@ #ifndef XBELSUPPORT_H #define XBELSUPPORT_H -#include <QtXml/QXmlStreamReader> +#include <QtCore/QXmlStreamReader> #include <QtCore/QPersistentModelIndex> QT_FORWARD_DECLARE_CLASS(QIODevice) diff --git a/src/assistant/lib/fulltextsearch/fulltextsearch.pro b/src/assistant/clucene/clucene.pro index 8733e57d7..51ada580f 100644 --- a/src/assistant/lib/fulltextsearch/fulltextsearch.pro +++ b/src/assistant/clucene/clucene.pro @@ -1,21 +1,16 @@ load(qt_module) -QMAKE_TARGET_PRODUCT = QtCLucene -QMAKE_TARGET_DESCRIPTION = QtCLucene full text search library wrapper. -#if qt is built with frameworks in debug, we must build QtCLucene in debug and release -#that's a similar logic as in qbase.pri -mac:!static:contains(QT_CONFIG, qt_framework) { - CONFIG(debug, debug|release) { - !build_pass:CONFIG += build_all - } -} -QT_CONFIG -= qt_framework -QT -= gui -TEMPLATE = lib TARGET = QtCLucene +QPRO_PWD = $$PWD + +CONFIG += module +MODULE_PRI = ../../../modules/qt_clucene.pri + +QT = core + DEFINES += QHELP_LIB -include($$QT_SOURCE_TREE/src/qbase.pri) +load(qt_module_config) HEADERS += qtcluceneversion.h include(fulltextsearch.pri) diff --git a/src/assistant/lib/fulltextsearch/fulltextsearch.pri b/src/assistant/clucene/fulltextsearch.pri index aac2ec6eb..eebfba33a 100644 --- a/src/assistant/lib/fulltextsearch/fulltextsearch.pri +++ b/src/assistant/clucene/fulltextsearch.pri @@ -1,7 +1,7 @@ DEFINES += _BUILD_FOR_QT_ LUCENE_DISABLE_MEMTRACKING win32:DEFINES += _CRT_SECURE_NO_DEPRECATE _MT -CLUCENEDIR = ../../../../3rdparty/clucene/src/CLucene +CLUCENEDIR = ../3rdparty/clucene/src/CLucene INCLUDEPATH += . .. \ $$CLUCENEDIR \ diff --git a/src/assistant/lib/fulltextsearch/license.txt b/src/assistant/clucene/license.txt index 9ef3d701d..9ef3d701d 100644 --- a/src/assistant/lib/fulltextsearch/license.txt +++ b/src/assistant/clucene/license.txt diff --git a/src/assistant/lib/fulltextsearch/qanalyzer.cpp b/src/assistant/clucene/qanalyzer.cpp index 71dd2c972..71dd2c972 100644 --- a/src/assistant/lib/fulltextsearch/qanalyzer.cpp +++ b/src/assistant/clucene/qanalyzer.cpp diff --git a/src/assistant/lib/fulltextsearch/qanalyzer_p.h b/src/assistant/clucene/qanalyzer_p.h index 162d6799b..162d6799b 100644 --- a/src/assistant/lib/fulltextsearch/qanalyzer_p.h +++ b/src/assistant/clucene/qanalyzer_p.h diff --git a/src/assistant/lib/fulltextsearch/qclucene-config_p.h b/src/assistant/clucene/qclucene-config_p.h index 476296cf5..476296cf5 100644 --- a/src/assistant/lib/fulltextsearch/qclucene-config_p.h +++ b/src/assistant/clucene/qclucene-config_p.h diff --git a/src/assistant/lib/fulltextsearch/qclucene_global_p.h b/src/assistant/clucene/qclucene_global_p.h index 887b113e5..887b113e5 100644 --- a/src/assistant/lib/fulltextsearch/qclucene_global_p.h +++ b/src/assistant/clucene/qclucene_global_p.h diff --git a/src/assistant/lib/fulltextsearch/qdocument.cpp b/src/assistant/clucene/qdocument.cpp index 3c4cc068d..3c4cc068d 100644 --- a/src/assistant/lib/fulltextsearch/qdocument.cpp +++ b/src/assistant/clucene/qdocument.cpp diff --git a/src/assistant/lib/fulltextsearch/qdocument_p.h b/src/assistant/clucene/qdocument_p.h index 31369af2a..31369af2a 100644 --- a/src/assistant/lib/fulltextsearch/qdocument_p.h +++ b/src/assistant/clucene/qdocument_p.h diff --git a/src/assistant/lib/fulltextsearch/qfield.cpp b/src/assistant/clucene/qfield.cpp index 448acf0e6..448acf0e6 100644 --- a/src/assistant/lib/fulltextsearch/qfield.cpp +++ b/src/assistant/clucene/qfield.cpp diff --git a/src/assistant/lib/fulltextsearch/qfield_p.h b/src/assistant/clucene/qfield_p.h index 39f2ff308..39f2ff308 100644 --- a/src/assistant/lib/fulltextsearch/qfield_p.h +++ b/src/assistant/clucene/qfield_p.h diff --git a/src/assistant/lib/fulltextsearch/qfilter.cpp b/src/assistant/clucene/qfilter.cpp index 4807193ff..4807193ff 100644 --- a/src/assistant/lib/fulltextsearch/qfilter.cpp +++ b/src/assistant/clucene/qfilter.cpp diff --git a/src/assistant/lib/fulltextsearch/qfilter_p.h b/src/assistant/clucene/qfilter_p.h index 67f6615b1..67f6615b1 100644 --- a/src/assistant/lib/fulltextsearch/qfilter_p.h +++ b/src/assistant/clucene/qfilter_p.h diff --git a/src/assistant/lib/fulltextsearch/qhits.cpp b/src/assistant/clucene/qhits.cpp index 5223a743d..5223a743d 100644 --- a/src/assistant/lib/fulltextsearch/qhits.cpp +++ b/src/assistant/clucene/qhits.cpp diff --git a/src/assistant/lib/fulltextsearch/qhits_p.h b/src/assistant/clucene/qhits_p.h index 98cd7021a..98cd7021a 100644 --- a/src/assistant/lib/fulltextsearch/qhits_p.h +++ b/src/assistant/clucene/qhits_p.h diff --git a/src/assistant/lib/fulltextsearch/qindexreader.cpp b/src/assistant/clucene/qindexreader.cpp index 285a96334..285a96334 100644 --- a/src/assistant/lib/fulltextsearch/qindexreader.cpp +++ b/src/assistant/clucene/qindexreader.cpp diff --git a/src/assistant/lib/fulltextsearch/qindexreader_p.h b/src/assistant/clucene/qindexreader_p.h index 7421daa17..7421daa17 100644 --- a/src/assistant/lib/fulltextsearch/qindexreader_p.h +++ b/src/assistant/clucene/qindexreader_p.h diff --git a/src/assistant/lib/fulltextsearch/qindexwriter.cpp b/src/assistant/clucene/qindexwriter.cpp index 1f579eb65..1f579eb65 100644 --- a/src/assistant/lib/fulltextsearch/qindexwriter.cpp +++ b/src/assistant/clucene/qindexwriter.cpp diff --git a/src/assistant/lib/fulltextsearch/qindexwriter_p.h b/src/assistant/clucene/qindexwriter_p.h index 1f9c86115..1f9c86115 100644 --- a/src/assistant/lib/fulltextsearch/qindexwriter_p.h +++ b/src/assistant/clucene/qindexwriter_p.h diff --git a/src/assistant/lib/fulltextsearch/qquery.cpp b/src/assistant/clucene/qquery.cpp index 170341cdd..170341cdd 100644 --- a/src/assistant/lib/fulltextsearch/qquery.cpp +++ b/src/assistant/clucene/qquery.cpp diff --git a/src/assistant/lib/fulltextsearch/qquery_p.h b/src/assistant/clucene/qquery_p.h index 3268b7c42..3268b7c42 100644 --- a/src/assistant/lib/fulltextsearch/qquery_p.h +++ b/src/assistant/clucene/qquery_p.h diff --git a/src/assistant/lib/fulltextsearch/qqueryparser.cpp b/src/assistant/clucene/qqueryparser.cpp index f306a041b..f306a041b 100644 --- a/src/assistant/lib/fulltextsearch/qqueryparser.cpp +++ b/src/assistant/clucene/qqueryparser.cpp diff --git a/src/assistant/lib/fulltextsearch/qqueryparser_p.h b/src/assistant/clucene/qqueryparser_p.h index e1b8c74eb..e1b8c74eb 100644 --- a/src/assistant/lib/fulltextsearch/qqueryparser_p.h +++ b/src/assistant/clucene/qqueryparser_p.h diff --git a/src/assistant/lib/fulltextsearch/qreader.cpp b/src/assistant/clucene/qreader.cpp index 0a8b7accf..0a8b7accf 100644 --- a/src/assistant/lib/fulltextsearch/qreader.cpp +++ b/src/assistant/clucene/qreader.cpp diff --git a/src/assistant/lib/fulltextsearch/qreader_p.h b/src/assistant/clucene/qreader_p.h index e24e9d836..e24e9d836 100644 --- a/src/assistant/lib/fulltextsearch/qreader_p.h +++ b/src/assistant/clucene/qreader_p.h diff --git a/src/assistant/lib/fulltextsearch/qsearchable.cpp b/src/assistant/clucene/qsearchable.cpp index 508fc05f6..508fc05f6 100644 --- a/src/assistant/lib/fulltextsearch/qsearchable.cpp +++ b/src/assistant/clucene/qsearchable.cpp diff --git a/src/assistant/lib/fulltextsearch/qsearchable_p.h b/src/assistant/clucene/qsearchable_p.h index 149cfb049..149cfb049 100644 --- a/src/assistant/lib/fulltextsearch/qsearchable_p.h +++ b/src/assistant/clucene/qsearchable_p.h diff --git a/src/assistant/lib/fulltextsearch/qsort.cpp b/src/assistant/clucene/qsort.cpp index 4dface005..4dface005 100644 --- a/src/assistant/lib/fulltextsearch/qsort.cpp +++ b/src/assistant/clucene/qsort.cpp diff --git a/src/assistant/lib/fulltextsearch/qsort_p.h b/src/assistant/clucene/qsort_p.h index 5d9372b11..5d9372b11 100644 --- a/src/assistant/lib/fulltextsearch/qsort_p.h +++ b/src/assistant/clucene/qsort_p.h diff --git a/src/assistant/lib/fulltextsearch/qterm.cpp b/src/assistant/clucene/qterm.cpp index eeebf53c8..eeebf53c8 100644 --- a/src/assistant/lib/fulltextsearch/qterm.cpp +++ b/src/assistant/clucene/qterm.cpp diff --git a/src/assistant/lib/fulltextsearch/qterm_p.h b/src/assistant/clucene/qterm_p.h index 5b981dc53..5b981dc53 100644 --- a/src/assistant/lib/fulltextsearch/qterm_p.h +++ b/src/assistant/clucene/qterm_p.h diff --git a/src/assistant/lib/fulltextsearch/qtoken.cpp b/src/assistant/clucene/qtoken.cpp index 537d9e675..537d9e675 100644 --- a/src/assistant/lib/fulltextsearch/qtoken.cpp +++ b/src/assistant/clucene/qtoken.cpp diff --git a/src/assistant/lib/fulltextsearch/qtoken_p.h b/src/assistant/clucene/qtoken_p.h index f3d25c4ff..f3d25c4ff 100644 --- a/src/assistant/lib/fulltextsearch/qtoken_p.h +++ b/src/assistant/clucene/qtoken_p.h diff --git a/src/assistant/lib/fulltextsearch/qtokenizer.cpp b/src/assistant/clucene/qtokenizer.cpp index 8a79b21e6..8a79b21e6 100644 --- a/src/assistant/lib/fulltextsearch/qtokenizer.cpp +++ b/src/assistant/clucene/qtokenizer.cpp diff --git a/src/assistant/lib/fulltextsearch/qtokenizer_p.h b/src/assistant/clucene/qtokenizer_p.h index 0c6e8ea1d..0c6e8ea1d 100644 --- a/src/assistant/lib/fulltextsearch/qtokenizer_p.h +++ b/src/assistant/clucene/qtokenizer_p.h diff --git a/src/assistant/lib/fulltextsearch/qtokenstream.cpp b/src/assistant/clucene/qtokenstream.cpp index 337ebf1e3..337ebf1e3 100644 --- a/src/assistant/lib/fulltextsearch/qtokenstream.cpp +++ b/src/assistant/clucene/qtokenstream.cpp diff --git a/src/assistant/lib/fulltextsearch/qtokenstream_p.h b/src/assistant/clucene/qtokenstream_p.h index 8f4b9d06a..8f4b9d06a 100644 --- a/src/assistant/lib/fulltextsearch/qtokenstream_p.h +++ b/src/assistant/clucene/qtokenstream_p.h diff --git a/src/assistant/lib/lib.pro b/src/assistant/help/help.pro index 280c77915..2aa30992e 100644 --- a/src/assistant/lib/lib.pro +++ b/src/assistant/help/help.pro @@ -1,39 +1,28 @@ load(qt_module) -QT += widgets \ - sql \ - xml \ - network \ - core-private -TEMPLATE = lib TARGET = QtHelp +QPRO_PWD = $$PWD +QT += widgets sql network core-private clucene clucene-private CONFIG += module MODULE_PRI += ../../../modules/qt_help.pri -DEFINES += QHELP_LIB \ - QT_CLUCENE_SUPPORT -CONFIG += qt \ - warn_on -include($$QT_SOURCE_TREE/src/qbase.pri) +DEFINES += QHELP_LIB QT_CLUCENE_SUPPORT + +load(qt_module_config) HEADERS += qthelpversion.h -QMAKE_TARGET_PRODUCT = Help -QMAKE_TARGET_DESCRIPTION = Help \ - application \ - framework. DEFINES -= QT_ASCII_CAST_WARNINGS -INCLUDEPATH += $$QT.help.includes + qclucene = QtCLucene$${QT_LIBINFIX} if(!debug_and_release|build_pass):CONFIG(debug, debug|release) { mac:qclucene = $${qclucene}_debug win32:qclucene = $${qclucene}d } linux-lsb-g++:LIBS_PRIVATE += --lsb-shared-libs=$$qclucene -unix|win32-g++*:QMAKE_PKGCONFIG_REQUIRES += QtNetwork \ - QtSql \ - QtXml +unix|win32-g++*:QMAKE_PKGCONFIG_REQUIRES += QtCore QtNetwork QtSql + LIBS_PRIVATE += -l$$qclucene RESOURCES += helpsystem.qrc SOURCES += qhelpenginecore.cpp \ diff --git a/src/assistant/lib/helpsystem.qrc b/src/assistant/help/helpsystem.qrc index 10efc6df1..10efc6df1 100644 --- a/src/assistant/lib/helpsystem.qrc +++ b/src/assistant/help/helpsystem.qrc diff --git a/src/assistant/lib/images/1leftarrow.png b/src/assistant/help/images/1leftarrow.png Binary files differindex bd1a5a249..bd1a5a249 100644 --- a/src/assistant/lib/images/1leftarrow.png +++ b/src/assistant/help/images/1leftarrow.png diff --git a/src/assistant/lib/images/1rightarrow.png b/src/assistant/help/images/1rightarrow.png Binary files differindex 0c0c44ae6..0c0c44ae6 100644 --- a/src/assistant/lib/images/1rightarrow.png +++ b/src/assistant/help/images/1rightarrow.png diff --git a/src/assistant/lib/images/3leftarrow.png b/src/assistant/help/images/3leftarrow.png Binary files differindex 8d38b0f57..8d38b0f57 100644 --- a/src/assistant/lib/images/3leftarrow.png +++ b/src/assistant/help/images/3leftarrow.png diff --git a/src/assistant/lib/images/3rightarrow.png b/src/assistant/help/images/3rightarrow.png Binary files differindex c2faf501c..c2faf501c 100644 --- a/src/assistant/lib/images/3rightarrow.png +++ b/src/assistant/help/images/3rightarrow.png diff --git a/src/assistant/lib/qclucenefieldnames.cpp b/src/assistant/help/qclucenefieldnames.cpp index 31ef415af..31ef415af 100644 --- a/src/assistant/lib/qclucenefieldnames.cpp +++ b/src/assistant/help/qclucenefieldnames.cpp diff --git a/src/assistant/lib/qclucenefieldnames_p.h b/src/assistant/help/qclucenefieldnames_p.h index 733e27121..733e27121 100644 --- a/src/assistant/lib/qclucenefieldnames_p.h +++ b/src/assistant/help/qclucenefieldnames_p.h diff --git a/src/assistant/lib/qhelp_global.cpp b/src/assistant/help/qhelp_global.cpp index 2c7ca6214..2c7ca6214 100644 --- a/src/assistant/lib/qhelp_global.cpp +++ b/src/assistant/help/qhelp_global.cpp diff --git a/src/assistant/lib/qhelp_global.h b/src/assistant/help/qhelp_global.h index 182298fc2..182298fc2 100644 --- a/src/assistant/lib/qhelp_global.h +++ b/src/assistant/help/qhelp_global.h diff --git a/src/assistant/lib/qhelpcollectionhandler.cpp b/src/assistant/help/qhelpcollectionhandler.cpp index 169d37e05..169d37e05 100644 --- a/src/assistant/lib/qhelpcollectionhandler.cpp +++ b/src/assistant/help/qhelpcollectionhandler.cpp diff --git a/src/assistant/lib/qhelpcollectionhandler_p.h b/src/assistant/help/qhelpcollectionhandler_p.h index a97af8fa9..a97af8fa9 100644 --- a/src/assistant/lib/qhelpcollectionhandler_p.h +++ b/src/assistant/help/qhelpcollectionhandler_p.h diff --git a/src/assistant/lib/qhelpcontentwidget.cpp b/src/assistant/help/qhelpcontentwidget.cpp index 7e3cb93d1..7e3cb93d1 100644 --- a/src/assistant/lib/qhelpcontentwidget.cpp +++ b/src/assistant/help/qhelpcontentwidget.cpp diff --git a/src/assistant/lib/qhelpcontentwidget.h b/src/assistant/help/qhelpcontentwidget.h index ab7d80a43..ab7d80a43 100644 --- a/src/assistant/lib/qhelpcontentwidget.h +++ b/src/assistant/help/qhelpcontentwidget.h diff --git a/src/assistant/lib/qhelpdatainterface.cpp b/src/assistant/help/qhelpdatainterface.cpp index d3f07c756..d3f07c756 100644 --- a/src/assistant/lib/qhelpdatainterface.cpp +++ b/src/assistant/help/qhelpdatainterface.cpp diff --git a/src/assistant/lib/qhelpdatainterface_p.h b/src/assistant/help/qhelpdatainterface_p.h index 886d68650..886d68650 100644 --- a/src/assistant/lib/qhelpdatainterface_p.h +++ b/src/assistant/help/qhelpdatainterface_p.h diff --git a/src/assistant/lib/qhelpdbreader.cpp b/src/assistant/help/qhelpdbreader.cpp index c4735f8b8..c4735f8b8 100644 --- a/src/assistant/lib/qhelpdbreader.cpp +++ b/src/assistant/help/qhelpdbreader.cpp diff --git a/src/assistant/lib/qhelpdbreader_p.h b/src/assistant/help/qhelpdbreader_p.h index 5fdf2e90b..5fdf2e90b 100644 --- a/src/assistant/lib/qhelpdbreader_p.h +++ b/src/assistant/help/qhelpdbreader_p.h diff --git a/src/assistant/lib/qhelpengine.cpp b/src/assistant/help/qhelpengine.cpp index 007cc84a9..007cc84a9 100644 --- a/src/assistant/lib/qhelpengine.cpp +++ b/src/assistant/help/qhelpengine.cpp diff --git a/src/assistant/lib/qhelpengine.h b/src/assistant/help/qhelpengine.h index cc0bca299..cc0bca299 100644 --- a/src/assistant/lib/qhelpengine.h +++ b/src/assistant/help/qhelpengine.h diff --git a/src/assistant/lib/qhelpengine_p.h b/src/assistant/help/qhelpengine_p.h index 5a7c30184..5a7c30184 100644 --- a/src/assistant/lib/qhelpengine_p.h +++ b/src/assistant/help/qhelpengine_p.h diff --git a/src/assistant/lib/qhelpenginecore.cpp b/src/assistant/help/qhelpenginecore.cpp index 178e11ee6..178e11ee6 100644 --- a/src/assistant/lib/qhelpenginecore.cpp +++ b/src/assistant/help/qhelpenginecore.cpp diff --git a/src/assistant/lib/qhelpenginecore.h b/src/assistant/help/qhelpenginecore.h index fbfb8a0e8..fbfb8a0e8 100644 --- a/src/assistant/lib/qhelpenginecore.h +++ b/src/assistant/help/qhelpenginecore.h diff --git a/src/assistant/lib/qhelpgenerator.cpp b/src/assistant/help/qhelpgenerator.cpp index ce9c8562d..ce9c8562d 100644 --- a/src/assistant/lib/qhelpgenerator.cpp +++ b/src/assistant/help/qhelpgenerator.cpp diff --git a/src/assistant/lib/qhelpgenerator_p.h b/src/assistant/help/qhelpgenerator_p.h index ad32c05de..ad32c05de 100644 --- a/src/assistant/lib/qhelpgenerator_p.h +++ b/src/assistant/help/qhelpgenerator_p.h diff --git a/src/assistant/lib/qhelpindexwidget.cpp b/src/assistant/help/qhelpindexwidget.cpp index 1b34ca0aa..1b34ca0aa 100644 --- a/src/assistant/lib/qhelpindexwidget.cpp +++ b/src/assistant/help/qhelpindexwidget.cpp diff --git a/src/assistant/lib/qhelpindexwidget.h b/src/assistant/help/qhelpindexwidget.h index 41e66f647..41e66f647 100644 --- a/src/assistant/lib/qhelpindexwidget.h +++ b/src/assistant/help/qhelpindexwidget.h diff --git a/src/assistant/lib/qhelpprojectdata.cpp b/src/assistant/help/qhelpprojectdata.cpp index bd59e289c..ce0ddd720 100644 --- a/src/assistant/lib/qhelpprojectdata.cpp +++ b/src/assistant/help/qhelpprojectdata.cpp @@ -49,7 +49,7 @@ #include <QtCore/QRegExp> #include <QtCore/QUrl> #include <QtCore/QVariant> -#include <QtXml/QXmlStreamReader> +#include <QtCore/QXmlStreamReader> QT_BEGIN_NAMESPACE diff --git a/src/assistant/lib/qhelpprojectdata_p.h b/src/assistant/help/qhelpprojectdata_p.h index 64ae32505..64ae32505 100644 --- a/src/assistant/lib/qhelpprojectdata_p.h +++ b/src/assistant/help/qhelpprojectdata_p.h diff --git a/src/assistant/lib/qhelpsearchengine.cpp b/src/assistant/help/qhelpsearchengine.cpp index f8f8acf44..f8f8acf44 100644 --- a/src/assistant/lib/qhelpsearchengine.cpp +++ b/src/assistant/help/qhelpsearchengine.cpp diff --git a/src/assistant/lib/qhelpsearchengine.h b/src/assistant/help/qhelpsearchengine.h index baceb7057..baceb7057 100644 --- a/src/assistant/lib/qhelpsearchengine.h +++ b/src/assistant/help/qhelpsearchengine.h diff --git a/src/assistant/lib/qhelpsearchindex_default.cpp b/src/assistant/help/qhelpsearchindex_default.cpp index 9974f618d..9974f618d 100644 --- a/src/assistant/lib/qhelpsearchindex_default.cpp +++ b/src/assistant/help/qhelpsearchindex_default.cpp diff --git a/src/assistant/lib/qhelpsearchindex_default_p.h b/src/assistant/help/qhelpsearchindex_default_p.h index 868609960..868609960 100644 --- a/src/assistant/lib/qhelpsearchindex_default_p.h +++ b/src/assistant/help/qhelpsearchindex_default_p.h diff --git a/src/assistant/lib/qhelpsearchindexreader.cpp b/src/assistant/help/qhelpsearchindexreader.cpp index 1ee356d2f..1ee356d2f 100644 --- a/src/assistant/lib/qhelpsearchindexreader.cpp +++ b/src/assistant/help/qhelpsearchindexreader.cpp diff --git a/src/assistant/lib/qhelpsearchindexreader_clucene.cpp b/src/assistant/help/qhelpsearchindexreader_clucene.cpp index 537e1d765..b67898f8a 100644 --- a/src/assistant/lib/qhelpsearchindexreader_clucene.cpp +++ b/src/assistant/help/qhelpsearchindexreader_clucene.cpp @@ -39,9 +39,9 @@ ** ****************************************************************************/ -#include "fulltextsearch/qindexreader_p.h" -#include "fulltextsearch/qqueryparser_p.h" -#include "fulltextsearch/qsearchable_p.h" +#include "private/qindexreader_p.h" +#include "private/qqueryparser_p.h" +#include "private/qsearchable_p.h" #include "qclucenefieldnames_p.h" #include "qhelpenginecore.h" diff --git a/src/assistant/lib/qhelpsearchindexreader_clucene_p.h b/src/assistant/help/qhelpsearchindexreader_clucene_p.h index dec18a07b..1e3d199ed 100644 --- a/src/assistant/lib/qhelpsearchindexreader_clucene_p.h +++ b/src/assistant/help/qhelpsearchindexreader_clucene_p.h @@ -57,8 +57,8 @@ #include <QtCore/QString> #include <QtCore/QStringList> -#include "fulltextsearch/qanalyzer_p.h" -#include "fulltextsearch/qquery_p.h" +#include "private/qanalyzer_p.h" +#include "private/qquery_p.h" #include "qhelpsearchindexreader_p.h" QT_BEGIN_NAMESPACE diff --git a/src/assistant/lib/qhelpsearchindexreader_default.cpp b/src/assistant/help/qhelpsearchindexreader_default.cpp index e49fdfcbd..e49fdfcbd 100644 --- a/src/assistant/lib/qhelpsearchindexreader_default.cpp +++ b/src/assistant/help/qhelpsearchindexreader_default.cpp diff --git a/src/assistant/lib/qhelpsearchindexreader_default_p.h b/src/assistant/help/qhelpsearchindexreader_default_p.h index 30dfe1ed6..30dfe1ed6 100644 --- a/src/assistant/lib/qhelpsearchindexreader_default_p.h +++ b/src/assistant/help/qhelpsearchindexreader_default_p.h diff --git a/src/assistant/lib/qhelpsearchindexreader_p.h b/src/assistant/help/qhelpsearchindexreader_p.h index 0d7518a6e..0d7518a6e 100644 --- a/src/assistant/lib/qhelpsearchindexreader_p.h +++ b/src/assistant/help/qhelpsearchindexreader_p.h diff --git a/src/assistant/lib/qhelpsearchindexwriter_clucene.cpp b/src/assistant/help/qhelpsearchindexwriter_clucene.cpp index f3d5e4fc3..287151338 100644 --- a/src/assistant/lib/qhelpsearchindexwriter_clucene.cpp +++ b/src/assistant/help/qhelpsearchindexwriter_clucene.cpp @@ -42,13 +42,13 @@ #include "qclucenefieldnames_p.h" #include "qhelpenginecore.h" #include "qhelp_global.h" -#include "fulltextsearch/qhits_p.h" -#include "fulltextsearch/qquery_p.h" -#include "fulltextsearch/qanalyzer_p.h" -#include "fulltextsearch/qdocument_p.h" -#include "fulltextsearch/qsearchable_p.h" -#include "fulltextsearch/qindexreader_p.h" -#include "fulltextsearch/qindexwriter_p.h" +#include "private/qhits_p.h" +#include "private/qquery_p.h" +#include "private/qanalyzer_p.h" +#include "private/qdocument_p.h" +#include "private/qsearchable_p.h" +#include "private/qindexreader_p.h" +#include "private/qindexwriter_p.h" #include "qhelpsearchindexwriter_clucene_p.h" #include <QtCore/QDir> diff --git a/src/assistant/lib/qhelpsearchindexwriter_clucene_p.h b/src/assistant/help/qhelpsearchindexwriter_clucene_p.h index 89146f5a7..7c9a91468 100644 --- a/src/assistant/lib/qhelpsearchindexwriter_clucene_p.h +++ b/src/assistant/help/qhelpsearchindexwriter_clucene_p.h @@ -54,7 +54,7 @@ // #include "qhelpenginecore.h" -#include "fulltextsearch/qanalyzer_p.h" +#include "private/qanalyzer_p.h" #include <QtCore/QUrl> #include <QtCore/QThread> diff --git a/src/assistant/lib/qhelpsearchindexwriter_default.cpp b/src/assistant/help/qhelpsearchindexwriter_default.cpp index db7f6ba0d..db7f6ba0d 100644 --- a/src/assistant/lib/qhelpsearchindexwriter_default.cpp +++ b/src/assistant/help/qhelpsearchindexwriter_default.cpp diff --git a/src/assistant/lib/qhelpsearchindexwriter_default_p.h b/src/assistant/help/qhelpsearchindexwriter_default_p.h index d510fbc9d..d510fbc9d 100644 --- a/src/assistant/lib/qhelpsearchindexwriter_default_p.h +++ b/src/assistant/help/qhelpsearchindexwriter_default_p.h diff --git a/src/assistant/lib/qhelpsearchquerywidget.cpp b/src/assistant/help/qhelpsearchquerywidget.cpp index 55974ada5..55974ada5 100644 --- a/src/assistant/lib/qhelpsearchquerywidget.cpp +++ b/src/assistant/help/qhelpsearchquerywidget.cpp diff --git a/src/assistant/lib/qhelpsearchquerywidget.h b/src/assistant/help/qhelpsearchquerywidget.h index e438df0b5..e438df0b5 100644 --- a/src/assistant/lib/qhelpsearchquerywidget.h +++ b/src/assistant/help/qhelpsearchquerywidget.h diff --git a/src/assistant/lib/qhelpsearchresultwidget.cpp b/src/assistant/help/qhelpsearchresultwidget.cpp index 964185b5e..964185b5e 100644 --- a/src/assistant/lib/qhelpsearchresultwidget.cpp +++ b/src/assistant/help/qhelpsearchresultwidget.cpp diff --git a/src/assistant/lib/qhelpsearchresultwidget.h b/src/assistant/help/qhelpsearchresultwidget.h index f9300e979..f9300e979 100644 --- a/src/assistant/lib/qhelpsearchresultwidget.h +++ b/src/assistant/help/qhelpsearchresultwidget.h diff --git a/src/assistant/tools/qcollectiongenerator/main.cpp b/src/assistant/qcollectiongenerator/main.cpp index 058d4281a..87bdf7996 100644 --- a/src/assistant/tools/qcollectiongenerator/main.cpp +++ b/src/assistant/qcollectiongenerator/main.cpp @@ -55,7 +55,7 @@ #include <QtCore/QLocale> #include <QtCore/QLibraryInfo> #include <QtHelp/QHelpEngineCore> -#include <QtXml/QXmlStreamReader> +#include <QtCore/QXmlStreamReader> QT_USE_NAMESPACE diff --git a/src/assistant/tools/qcollectiongenerator/qcollectiongenerator.pro b/src/assistant/qcollectiongenerator/qcollectiongenerator.pro index 23d32c698..e8df8e4f6 100644 --- a/src/assistant/tools/qcollectiongenerator/qcollectiongenerator.pro +++ b/src/assistant/qcollectiongenerator/qcollectiongenerator.pro @@ -1,6 +1,4 @@ -QT += xml \ - network \ - help-private +QT += network help-private TEMPLATE = app DESTDIR = $$QT.designer.bins TARGET = qcollectiongenerator diff --git a/src/assistant/tools/qhelpconverter/adpreader.cpp b/src/assistant/qhelpconverter/adpreader.cpp index a0983755e..a0983755e 100644 --- a/src/assistant/tools/qhelpconverter/adpreader.cpp +++ b/src/assistant/qhelpconverter/adpreader.cpp diff --git a/src/assistant/tools/qhelpconverter/adpreader.h b/src/assistant/qhelpconverter/adpreader.h index f0b6ae214..9900866c9 100644 --- a/src/assistant/tools/qhelpconverter/adpreader.h +++ b/src/assistant/qhelpconverter/adpreader.h @@ -44,7 +44,7 @@ #include <QtCore/QMap> #include <QtCore/QSet> -#include <QtXml/QXmlStreamReader> +#include <QtCore/QXmlStreamReader> QT_BEGIN_NAMESPACE diff --git a/src/assistant/tools/qhelpconverter/assistant-128.png b/src/assistant/qhelpconverter/assistant-128.png Binary files differindex f05949f6d..f05949f6d 100644 --- a/src/assistant/tools/qhelpconverter/assistant-128.png +++ b/src/assistant/qhelpconverter/assistant-128.png diff --git a/src/assistant/tools/qhelpconverter/assistant.png b/src/assistant/qhelpconverter/assistant.png Binary files differindex ea4d1e70c..ea4d1e70c 100644 --- a/src/assistant/tools/qhelpconverter/assistant.png +++ b/src/assistant/qhelpconverter/assistant.png diff --git a/src/assistant/tools/qhelpconverter/conversionwizard.cpp b/src/assistant/qhelpconverter/conversionwizard.cpp index 0a80a0e71..0a80a0e71 100644 --- a/src/assistant/tools/qhelpconverter/conversionwizard.cpp +++ b/src/assistant/qhelpconverter/conversionwizard.cpp diff --git a/src/assistant/tools/qhelpconverter/conversionwizard.h b/src/assistant/qhelpconverter/conversionwizard.h index ff66d271e..ff66d271e 100644 --- a/src/assistant/tools/qhelpconverter/conversionwizard.h +++ b/src/assistant/qhelpconverter/conversionwizard.h diff --git a/src/assistant/tools/qhelpconverter/doc/filespage.html b/src/assistant/qhelpconverter/doc/filespage.html index a7aac18da..a7aac18da 100644 --- a/src/assistant/tools/qhelpconverter/doc/filespage.html +++ b/src/assistant/qhelpconverter/doc/filespage.html diff --git a/src/assistant/tools/qhelpconverter/doc/filterpage.html b/src/assistant/qhelpconverter/doc/filterpage.html index 7b3781bd5..7b3781bd5 100644 --- a/src/assistant/tools/qhelpconverter/doc/filterpage.html +++ b/src/assistant/qhelpconverter/doc/filterpage.html diff --git a/src/assistant/tools/qhelpconverter/doc/generalpage.html b/src/assistant/qhelpconverter/doc/generalpage.html index 8d106bec1..8d106bec1 100644 --- a/src/assistant/tools/qhelpconverter/doc/generalpage.html +++ b/src/assistant/qhelpconverter/doc/generalpage.html diff --git a/src/assistant/tools/qhelpconverter/doc/identifierpage.html b/src/assistant/qhelpconverter/doc/identifierpage.html index 952b88de8..952b88de8 100644 --- a/src/assistant/tools/qhelpconverter/doc/identifierpage.html +++ b/src/assistant/qhelpconverter/doc/identifierpage.html diff --git a/src/assistant/tools/qhelpconverter/doc/inputpage.html b/src/assistant/qhelpconverter/doc/inputpage.html index 4054c5472..4054c5472 100644 --- a/src/assistant/tools/qhelpconverter/doc/inputpage.html +++ b/src/assistant/qhelpconverter/doc/inputpage.html diff --git a/src/assistant/tools/qhelpconverter/doc/outputpage.html b/src/assistant/qhelpconverter/doc/outputpage.html index 332ef69fc..332ef69fc 100644 --- a/src/assistant/tools/qhelpconverter/doc/outputpage.html +++ b/src/assistant/qhelpconverter/doc/outputpage.html diff --git a/src/assistant/tools/qhelpconverter/doc/pathpage.html b/src/assistant/qhelpconverter/doc/pathpage.html index 95449b3c3..95449b3c3 100644 --- a/src/assistant/tools/qhelpconverter/doc/pathpage.html +++ b/src/assistant/qhelpconverter/doc/pathpage.html diff --git a/src/assistant/tools/qhelpconverter/filespage.cpp b/src/assistant/qhelpconverter/filespage.cpp index e30ec42ef..e30ec42ef 100644 --- a/src/assistant/tools/qhelpconverter/filespage.cpp +++ b/src/assistant/qhelpconverter/filespage.cpp diff --git a/src/assistant/tools/qhelpconverter/filespage.h b/src/assistant/qhelpconverter/filespage.h index 07022e898..07022e898 100644 --- a/src/assistant/tools/qhelpconverter/filespage.h +++ b/src/assistant/qhelpconverter/filespage.h diff --git a/src/assistant/tools/qhelpconverter/filespage.ui b/src/assistant/qhelpconverter/filespage.ui index d308b9664..d308b9664 100644 --- a/src/assistant/tools/qhelpconverter/filespage.ui +++ b/src/assistant/qhelpconverter/filespage.ui diff --git a/src/assistant/tools/qhelpconverter/filterpage.cpp b/src/assistant/qhelpconverter/filterpage.cpp index efe32c79e..efe32c79e 100644 --- a/src/assistant/tools/qhelpconverter/filterpage.cpp +++ b/src/assistant/qhelpconverter/filterpage.cpp diff --git a/src/assistant/tools/qhelpconverter/filterpage.h b/src/assistant/qhelpconverter/filterpage.h index 336c650de..336c650de 100644 --- a/src/assistant/tools/qhelpconverter/filterpage.h +++ b/src/assistant/qhelpconverter/filterpage.h diff --git a/src/assistant/tools/qhelpconverter/filterpage.ui b/src/assistant/qhelpconverter/filterpage.ui index 7cda3d9be..7cda3d9be 100644 --- a/src/assistant/tools/qhelpconverter/filterpage.ui +++ b/src/assistant/qhelpconverter/filterpage.ui diff --git a/src/assistant/tools/qhelpconverter/finishpage.cpp b/src/assistant/qhelpconverter/finishpage.cpp index cdbd634e6..cdbd634e6 100644 --- a/src/assistant/tools/qhelpconverter/finishpage.cpp +++ b/src/assistant/qhelpconverter/finishpage.cpp diff --git a/src/assistant/tools/qhelpconverter/finishpage.h b/src/assistant/qhelpconverter/finishpage.h index 41dec54ca..41dec54ca 100644 --- a/src/assistant/tools/qhelpconverter/finishpage.h +++ b/src/assistant/qhelpconverter/finishpage.h diff --git a/src/assistant/tools/qhelpconverter/generalpage.cpp b/src/assistant/qhelpconverter/generalpage.cpp index ad0171270..ad0171270 100644 --- a/src/assistant/tools/qhelpconverter/generalpage.cpp +++ b/src/assistant/qhelpconverter/generalpage.cpp diff --git a/src/assistant/tools/qhelpconverter/generalpage.h b/src/assistant/qhelpconverter/generalpage.h index c9a8b2452..c9a8b2452 100644 --- a/src/assistant/tools/qhelpconverter/generalpage.h +++ b/src/assistant/qhelpconverter/generalpage.h diff --git a/src/assistant/tools/qhelpconverter/generalpage.ui b/src/assistant/qhelpconverter/generalpage.ui index 9c2babb0a..9c2babb0a 100644 --- a/src/assistant/tools/qhelpconverter/generalpage.ui +++ b/src/assistant/qhelpconverter/generalpage.ui diff --git a/src/assistant/tools/qhelpconverter/helpwindow.cpp b/src/assistant/qhelpconverter/helpwindow.cpp index 5212092f7..5212092f7 100644 --- a/src/assistant/tools/qhelpconverter/helpwindow.cpp +++ b/src/assistant/qhelpconverter/helpwindow.cpp diff --git a/src/assistant/tools/qhelpconverter/helpwindow.h b/src/assistant/qhelpconverter/helpwindow.h index 9a45f06e0..9a45f06e0 100644 --- a/src/assistant/tools/qhelpconverter/helpwindow.h +++ b/src/assistant/qhelpconverter/helpwindow.h diff --git a/src/assistant/tools/qhelpconverter/identifierpage.cpp b/src/assistant/qhelpconverter/identifierpage.cpp index fb8310a61..fb8310a61 100644 --- a/src/assistant/tools/qhelpconverter/identifierpage.cpp +++ b/src/assistant/qhelpconverter/identifierpage.cpp diff --git a/src/assistant/tools/qhelpconverter/identifierpage.h b/src/assistant/qhelpconverter/identifierpage.h index 05de35ca9..05de35ca9 100644 --- a/src/assistant/tools/qhelpconverter/identifierpage.h +++ b/src/assistant/qhelpconverter/identifierpage.h diff --git a/src/assistant/tools/qhelpconverter/identifierpage.ui b/src/assistant/qhelpconverter/identifierpage.ui index cd0df7563..cd0df7563 100644 --- a/src/assistant/tools/qhelpconverter/identifierpage.ui +++ b/src/assistant/qhelpconverter/identifierpage.ui diff --git a/src/assistant/tools/qhelpconverter/inputpage.cpp b/src/assistant/qhelpconverter/inputpage.cpp index 9abbbe74b..9abbbe74b 100644 --- a/src/assistant/tools/qhelpconverter/inputpage.cpp +++ b/src/assistant/qhelpconverter/inputpage.cpp diff --git a/src/assistant/tools/qhelpconverter/inputpage.h b/src/assistant/qhelpconverter/inputpage.h index 185b7b2b2..185b7b2b2 100644 --- a/src/assistant/tools/qhelpconverter/inputpage.h +++ b/src/assistant/qhelpconverter/inputpage.h diff --git a/src/assistant/tools/qhelpconverter/inputpage.ui b/src/assistant/qhelpconverter/inputpage.ui index e7cd3a0fa..e7cd3a0fa 100644 --- a/src/assistant/tools/qhelpconverter/inputpage.ui +++ b/src/assistant/qhelpconverter/inputpage.ui diff --git a/src/assistant/tools/qhelpconverter/main.cpp b/src/assistant/qhelpconverter/main.cpp index 1c2d1bf12..1c2d1bf12 100644 --- a/src/assistant/tools/qhelpconverter/main.cpp +++ b/src/assistant/qhelpconverter/main.cpp diff --git a/src/assistant/tools/qhelpconverter/outputpage.cpp b/src/assistant/qhelpconverter/outputpage.cpp index 434357b34..434357b34 100644 --- a/src/assistant/tools/qhelpconverter/outputpage.cpp +++ b/src/assistant/qhelpconverter/outputpage.cpp diff --git a/src/assistant/tools/qhelpconverter/outputpage.h b/src/assistant/qhelpconverter/outputpage.h index e45afae10..e45afae10 100644 --- a/src/assistant/tools/qhelpconverter/outputpage.h +++ b/src/assistant/qhelpconverter/outputpage.h diff --git a/src/assistant/tools/qhelpconverter/outputpage.ui b/src/assistant/qhelpconverter/outputpage.ui index 755f81808..755f81808 100644 --- a/src/assistant/tools/qhelpconverter/outputpage.ui +++ b/src/assistant/qhelpconverter/outputpage.ui diff --git a/src/assistant/tools/qhelpconverter/pathpage.cpp b/src/assistant/qhelpconverter/pathpage.cpp index a9ddf53a5..a9ddf53a5 100644 --- a/src/assistant/tools/qhelpconverter/pathpage.cpp +++ b/src/assistant/qhelpconverter/pathpage.cpp diff --git a/src/assistant/tools/qhelpconverter/pathpage.h b/src/assistant/qhelpconverter/pathpage.h index 90b24236c..90b24236c 100644 --- a/src/assistant/tools/qhelpconverter/pathpage.h +++ b/src/assistant/qhelpconverter/pathpage.h diff --git a/src/assistant/tools/qhelpconverter/pathpage.ui b/src/assistant/qhelpconverter/pathpage.ui index 89083915d..89083915d 100644 --- a/src/assistant/tools/qhelpconverter/pathpage.ui +++ b/src/assistant/qhelpconverter/pathpage.ui diff --git a/src/assistant/tools/qhelpconverter/qhcpwriter.cpp b/src/assistant/qhelpconverter/qhcpwriter.cpp index cdf9f0b3c..cdf9f0b3c 100644 --- a/src/assistant/tools/qhelpconverter/qhcpwriter.cpp +++ b/src/assistant/qhelpconverter/qhcpwriter.cpp diff --git a/src/assistant/tools/qhelpconverter/qhcpwriter.h b/src/assistant/qhelpconverter/qhcpwriter.h index de0c6f8e1..933d15a15 100644 --- a/src/assistant/tools/qhelpconverter/qhcpwriter.h +++ b/src/assistant/qhelpconverter/qhcpwriter.h @@ -42,7 +42,7 @@ #ifndef QHCPWRITER_H #define QHCPWRITER_H -#include <QtXml/QXmlStreamWriter> +#include <QtCore/QXmlStreamWriter> #include "adpreader.h" QT_BEGIN_NAMESPACE diff --git a/src/assistant/tools/qhelpconverter/qhelpconverter.pro b/src/assistant/qhelpconverter/qhelpconverter.pro index f8754767d..712b16f61 100644 --- a/src/assistant/tools/qhelpconverter/qhelpconverter.pro +++ b/src/assistant/qhelpconverter/qhelpconverter.pro @@ -1,4 +1,4 @@ -QT += xml help +QT += help TEMPLATE = app TARGET = qhelpconverter DESTDIR = $$QT.designer.bins diff --git a/src/assistant/tools/qhelpconverter/qhelpconverter.qrc b/src/assistant/qhelpconverter/qhelpconverter.qrc index e2a68ab43..e2a68ab43 100644 --- a/src/assistant/tools/qhelpconverter/qhelpconverter.qrc +++ b/src/assistant/qhelpconverter/qhelpconverter.qrc diff --git a/src/assistant/tools/qhelpconverter/qhpwriter.cpp b/src/assistant/qhelpconverter/qhpwriter.cpp index 51d392d32..51d392d32 100644 --- a/src/assistant/tools/qhelpconverter/qhpwriter.cpp +++ b/src/assistant/qhelpconverter/qhpwriter.cpp diff --git a/src/assistant/tools/qhelpconverter/qhpwriter.h b/src/assistant/qhelpconverter/qhpwriter.h index 70fd04aca..c118d4c9f 100644 --- a/src/assistant/tools/qhelpconverter/qhpwriter.h +++ b/src/assistant/qhelpconverter/qhpwriter.h @@ -42,7 +42,7 @@ #ifndef QHPWRITER_H #define QHPWRITER_H -#include <QtXml/QXmlStreamWriter> +#include <QtCore/QXmlStreamWriter> #include "filterpage.h" QT_BEGIN_NAMESPACE diff --git a/src/assistant/tools/qhelpgenerator/main.cpp b/src/assistant/qhelpgenerator/main.cpp index 386e84c41..386e84c41 100644 --- a/src/assistant/tools/qhelpgenerator/main.cpp +++ b/src/assistant/qhelpgenerator/main.cpp diff --git a/src/assistant/tools/qhelpgenerator/qhelpgenerator.pro b/src/assistant/qhelpgenerator/qhelpgenerator.pro index f6db4fa11..f6db4fa11 100644 --- a/src/assistant/tools/qhelpgenerator/qhelpgenerator.pro +++ b/src/assistant/qhelpgenerator/qhelpgenerator.pro diff --git a/src/assistant/tools/shared/collectionconfiguration.cpp b/src/assistant/shared/collectionconfiguration.cpp index 35d5b66df..35d5b66df 100644 --- a/src/assistant/tools/shared/collectionconfiguration.cpp +++ b/src/assistant/shared/collectionconfiguration.cpp diff --git a/src/assistant/tools/shared/collectionconfiguration.h b/src/assistant/shared/collectionconfiguration.h index 226460ef3..226460ef3 100644 --- a/src/assistant/tools/shared/collectionconfiguration.h +++ b/src/assistant/shared/collectionconfiguration.h diff --git a/src/assistant/tools/shared/helpgenerator.cpp b/src/assistant/shared/helpgenerator.cpp index d551548fb..d551548fb 100644 --- a/src/assistant/tools/shared/helpgenerator.cpp +++ b/src/assistant/shared/helpgenerator.cpp diff --git a/src/assistant/tools/shared/helpgenerator.h b/src/assistant/shared/helpgenerator.h index e220a2b68..e220a2b68 100644 --- a/src/assistant/tools/shared/helpgenerator.h +++ b/src/assistant/shared/helpgenerator.h diff --git a/src/assistant/tools/tools.pro b/src/assistant/tools/tools.pro deleted file mode 100644 index 8bb8cd700..000000000 --- a/src/assistant/tools/tools.pro +++ /dev/null @@ -1,8 +0,0 @@ -TEMPLATE = subdirs -CONFIG += ordered - -SUBDIRS += assistant \ - qhelpgenerator \ - qcollectiongenerator \ - qhelpconverter - |
