collate.cpp

Go to the documentation of this file.

 
#include "def.h"          // for Def
#include "diagnostic.h"   // for issue_diag()
#include "path.h"         // for get_pathname()
#include "scanner.h"      // for scanner
 
#include <fstream>        // for ofstream
#include <iostream>       // for cout
#include <string>         // for string
#include <vector>         // for vector
 
#include <cassert>        // for assert()
#include <cctype>         // for toupper()
#include <cstdio>         // for sprintf()
#include <cstdlib>        // for strtoul()
#include <cstring>        // for strchr()
 
 
void Def::
create_wchar_utf8_table ()
{
    if (!wchar_utf8_to_ext_.empty())
        return;
 
    typedef std::map<std::string, wchar_t>::const_iterator n_cmap_citer2;
 
    const n_cmap_citer2 n_cmap_end = charmap_.get_mb_cmap ().end ();
 
    for (n_cmap_citer2 n_cmap_it = charmap_.get_mb_cmap ().begin ();
         n_cmap_it != n_cmap_end; ++n_cmap_it) {
 
        const std::string wchar_utf8 = utf8_encode (n_cmap_it->second);
 
        wchar_utf8_to_ext_.insert (std::make_pair (wchar_utf8,
                                                   n_cmap_it->first));
    }
}
 
 
void Def::process_weights (collate_entry_t& entry)
{
    // iterator of weights tokens
    token_list_t::iterator w_it = entry.second.begin ();
 
    w_cmap_iter   wcmap_it;
    ce_map_iter   ce_map_it;
    coll_map_iter coll_map_it;
 
    Weights_t* weights = new Weights_t[collate_out_.num_weights];
 
    int i = 0;
    for (i = 0; i < collate_out_.num_weights && w_it != entry.second.end (); 
         ++i, ++w_it){
        get_weight (*w_it, weights, i);
    }
 
    const std::map<std::string, wchar_t>& w_cmap = charmap_.get_w_cmap ();
 
    // the remaining weights - see above - are given as weight the
    // collating value of the symbol at start of line
    while (i < collate_out_.num_weights) {
        weights[i].size = 1;
        wcmap_it = w_cmap.find (entry.first.name);
        if (wcmap_it != w_cmap.end()) {
            coll_map_it = coll_map_.find (wcmap_it->second);
            weights[i].weight[0] = coll_map_it->second.coll_val;
        }
        else if ((ce_map_it = ce_map_.find (entry.first.name)) != 
                 ce_map_.end()) {
           weights[i].weight[0] = ce_map_it->second.coll_val;
        }
        else {
            warnings_occurred_ = 
                issue_diag (W_COLSYM, false, &entry.first,
                            "symbolic name %s not defined as character or "
                            "collating-element, ignoring\n", 
                            entry.first.name.c_str()) || warnings_occurred_;
        }
        i++;
    }
 
    wcmap_it = w_cmap.find (entry.first.name);
    if (wcmap_it != w_cmap.end()) {
        coll_map_it = coll_map_.find (wcmap_it->second);
        coll_map_it->second.weights = weights;
    } else if ((ce_map_it = ce_map_.find (entry.first.name)) != ce_map_.end()) {
        ce_map_it->second.weights = weights;
    } else {
        warnings_occurred_ =  
            issue_diag (W_COLSYM, false, &entry.first,
                        "symbolic name %s not defined as character or "
                        "collating-element, ignoring\n", 
                        entry.first.name.c_str()) || warnings_occurred_;
    }
}
 
 
unsigned int Def::process_order_stmt (collate_section_t& section)
{
    // number of orders in the section
    collate_out_.num_weights = unsigned (section.order.size ());
    if (collate_out_.num_weights == 0) {
        collate_out_.num_weights = 1;
        collate_out_.weight_type[0] = 0;
    } else {
        collate_out_.num_weights = 0;
        token_list_t::iterator ord_it = section.order.begin ();
        for (; ord_it != section.order.end (); 
             ++ord_it, ++collate_out_.num_weights) {
            if (ord_it->token == Scanner::tok_forward) {
                collate_out_.weight_type[collate_out_.num_weights] = 0;
            } else if (ord_it->token == Scanner::tok_backward) {
                collate_out_.weight_type[collate_out_.num_weights] = 1;
            } else if (ord_it->token == Scanner::tok_position) {
                // this is equivalent to "forward,position"
                if (no_position_)
                    collate_out_.weight_type[collate_out_.num_weights] = 0;
                else
                    collate_out_.weight_type[collate_out_.num_weights] = 2;
            } else {
                std::string::size_type comma_pos = ord_it->name.find (',');
                if (comma_pos != std::string::npos) {
                    std::string first  = ord_it->name.substr (0, comma_pos);
                    std::string second = ord_it->name.substr (comma_pos + 1);
 
                    // verify a valid weight ordering
                    if (   (first == "forward" && second != "position")
                        || (first == "backward" && second != "position")) {
                        issue_diag (E_COLORD, true,
                                    &*ord_it, "unrecognized collating order\n");
                        break;
                    } else if (first == "forward") {
                        if (no_position_)
                            collate_out_.weight_type
                                [collate_out_.num_weights] = 0;
                        else
                            collate_out_.weight_type
                                [collate_out_.num_weights] = 2;
                    }
                    else
                        collate_out_.weight_type[collate_out_.num_weights] = 3;
                }
                else {
                    issue_diag (E_COLORD, true,
                                &*ord_it, "unrecognized collating order\n");
                }
            }
        }
    }
 
    // build the order value;
    unsigned int order = 0;
    for (int i = 0; i < collate_out_.num_weights; ++i) {
        order <<= 2;
        order |= collate_out_.weight_type[i];
    }
 
    return order;
}
 
 
// decimally increment the symbolic name, turning something like
// <U1234> into <U1245>, or <jis234> to <jis235>
static std::string
dec_increment (const std::string &sym)
{
    const char *pdig = sym.c_str ();
 
    for (; !(std::isdigit (*pdig)); ++pdig);
 
    char *end;
 
    const unsigned long sym_val = std::strtoul (pdig, &end, 10);
 
    char numstr [64];
    std::sprintf (numstr, "%lu", sym_val + 1);
 
    std::string next_sym = sym.substr (0, pdig - sym.c_str ());
    next_sym.append (numstr);
    next_sym.append (end);
 
    return next_sym;
}
 
 
// hexadecimally increment the symbolic name
static std::string
hex_increment (const std::string& sym)
{
    const char *pdig = sym.c_str ();
 
    for (; !(std::isxdigit (*pdig)); ++pdig);
 
    char *end;
 
    const unsigned long sym_val = std::strtoul (pdig, &end, 16);
 
    char numstr [64];
    std::sprintf (numstr, "%lx", sym_val + 1);
 
    std::string next_sym = sym.substr (0, pdig - sym.c_str ());
    next_sym.append (numstr);
    next_sym.append (end);
 
    return next_sym;
}
 
 
// scanning ahead and adding all symbols to the collating symbols map
void Def::preprocess_collation_definitions()
{
    // start with collation values from 2
    unsigned int coll_value = 2;
 
    // previous_elm and next_elm are used for processing ellipsis.
    std::string prev_elm;
    std::string next_elm;
 
    token_list_t::iterator cs_it = sym_list_.begin ();
    for (; cs_it != sym_list_.end (); ++cs_it) {
        cs_map_iter csm_it = cs_map_.find (cs_it->name);
        if (csm_it != cs_map_.end ())
            csm_it->second = coll_value++;
    }
 
    static token_list_t::size_type max_orders = 0;
 
    std::list<collate_section_t>::iterator s_it = section_list_.begin ();
 
    if (s_it != section_list_.end()) {
        // get maximum weights count
        max_orders = s_it->order.size ();
    }
 
    for (; s_it != section_list_.end (); ++s_it) {
        if (max_orders != s_it->order.size ())
            issue_diag (E_COLNUM, true, &*s_it->order.begin (),
                        "number of collation orders "
                        "different for this section\n");
    }
 
    const std::map<std::string, wchar_t>& w_cmap = charmap_.get_w_cmap ();
 
    // preprocess sections' collation definitions
    for (s_it = section_list_.begin (); 
         s_it != section_list_.end (); ++s_it) {
 
        // process the order statement and get the weight
        unsigned int order = process_order_stmt (*s_it);
 
        std::list<collate_entry_t>::iterator e_it = 
            s_it->entries.begin ();
 
        for (; e_it != s_it->entries.end (); ++e_it) {
            switch (e_it->first.token) {
            case Scanner::tok_sym_name:
                // store the previous name
                prev_elm = e_it->first.name;
 
                // process each of the collation definitions
                process_collation_definition (false, *e_it, 
                                              coll_value++, order);
 
                break;
 
            case Scanner::tok_abs_ellipsis:   // "<FOO>...<BAR>"
            case Scanner::tok_hex_ellipsis:   // "<FOO>..<BAR>"
            case Scanner::tok_dec_ellipsis:   // "<FOO>....<BAR>"
            {
 
                if (++e_it != s_it->entries.end ())
                    next_elm = e_it->first.name;
                else {
                    issue_diag (E_SYNTAX, true, &e_it->first,
                                "unexpected end of collation section while"
                                " processing ellipsis during scan_ahead\n");
                }
 
                if (e_it->first.token == Scanner::tok_abs_ellipsis) {
                    wchar_t w_cmap_cur_val 
                        = w_cmap.find(prev_elm)->second;
                    wchar_t w_cmap_end_val
                        = w_cmap.find(next_elm)->second;
 
                    // the first value has already been 
                    // added so don't add it again
                    w_cmap_cur_val = charmap_.increment_wchar (w_cmap_cur_val);
                    while (w_cmap_cur_val != w_cmap_end_val) {
                        // process iteration
                        collate_info_t ci = {UINT_MAX, UINT_MAX, 0, 0};
                        ci.coll_val = coll_value++;
                        ci.order = order;
 
                        coll_map_.insert (std::make_pair (w_cmap_cur_val, ci));
                        w_cmap_cur_val = 
                            charmap_.increment_wchar (w_cmap_cur_val);
                    }
 
                    // add last element "next_elm" to array
                    collate_info_t ci = {UINT_MAX, UINT_MAX, 0, 0};
                    ci.coll_val = coll_value++;
                    ci.order = order;
                    coll_map_.insert (std::make_pair (w_cmap_cur_val, ci));
 
                }
                else {
                    // we are incrementing the symbolic names
                    std::string next_name = prev_elm;
                    do {
                        if (e_it->first.token == Scanner::tok_hex_ellipsis) 
                            next_name = hex_increment (next_name);
                        else 
                            next_name = dec_increment (next_name);
 
                        w_cmap_iter it = w_cmap.find (next_name);
                        if (it != w_cmap.end()) {
                            // process iteration
                            collate_info_t ci = {UINT_MAX, UINT_MAX, 0, 0};
                            ci.coll_val = coll_value++;
                            ci.order = order;
                            coll_map_.insert (
                                std::make_pair (it->second, ci));
                        }
                    } while (next_name != next_elm);
                }
 
                prev_elm = next_elm;
                break;
            }
 
            case Scanner::tok_dbl_ellipsis:
                    issue_diag (W_NOTSUP, true, &e_it->first,
                                "ellipsis not supported"
                                " processing ellipsis during scan_ahead\n");
            default:
                break;
            }
        }
    }
}
 
 
void Def::process_collation_definition ( bool do_weights,
                                         collate_entry_t& entry,
                                         unsigned int coll_value,
                                         unsigned int order)
{
    const std::map<std::string, wchar_t>& w_cmap = charmap_.get_w_cmap ();
 
    // iterators
    w_cmap_iter w_cmap_pos = 
        w_cmap.find(entry.first.name);
    ce_map_iter ce_map_pos = 
        ce_map_.find(entry.first.name);
 
    // look up the symbolic name in the wide character map
    if (w_cmap_pos != w_cmap.end()) {
 
        wchar_t wval = w_cmap_pos->second;
        coll_map_iter coll_map_pos = coll_map_.find (wval);
 
        coll_map_pos->second.coll_val = coll_value;
        coll_map_pos->second.order = order;
 
        // process the weights
        if (do_weights) {
            unsigned int offset = next_offset_++;
            coll_map_pos->second.offset = offset;
            off_mapr_.insert(std::make_pair(offset, entry.first.name));
 
            process_weights (entry); 
        }
 
    }
    else if (ce_map_pos != ce_map_.end()) {
 
        ce_map_pos->second.coll_val = coll_value;
        ce_map_pos->second.order    = order;
                
        // process the weights
        if (do_weights) {
            unsigned int offset = next_offset_++;
            ce_map_pos->second.offset = offset;
            off_mapr_.insert(std::make_pair(offset, entry.first.name));     
 
            process_weights (entry);
        }
 
    }
    else if (cs_map_.find(entry.first.name) != cs_map_.end()) {
 
        cs_map_.find(entry.first.name)->second = coll_value;
 
    }
    else {
        // the symbol is missing from all three maps; 
        // add it in cs map as "sym <-> collating_value" pair
        cs_map_.insert (std::make_pair(entry.first.name, coll_value));
 
        warnings_occurred_ =  
            issue_diag (W_COLSYM, false, &entry.first, 
                        "unknown symbol name %s found in "
                        "LC_COLLATE definition\n", 
                        entry.first.name.c_str()) || warnings_occurred_;
    }
}
 
void Def::process_order(collate_section_t& section, 
                        unsigned int& coll_value)
{
    std::string sym;
 
    // get the orders
    unsigned int section_order = process_order_stmt (section);
 
    // current element
    std::string curr_elm;
 
    // previous_elm and next_elm are used for processing ellipsis.
    std::string prev_elm;
    std::string next_elm;
 
    const std::map<std::string, wchar_t>& w_cmap = charmap_.get_w_cmap ();
 
    // iterate thru section's entries
    std::list<collate_entry_t>::iterator e_it = section.entries.begin ();
    for (; e_it != section.entries.end (); ++e_it) {
        if (e_it->first.token == Scanner::tok_sym_name) {
            prev_elm = e_it->first.name;
            process_collation_definition (true, *e_it, 
                                          coll_value++, section_order);
 
        }
        else if (   e_it->first.token == Scanner::tok_abs_ellipsis
                 || e_it->first.token == Scanner::tok_dec_ellipsis
                 || e_it->first.token == Scanner::tok_hex_ellipsis) {
 
            // process line with ellipsis
            Weights_t* weights_template = 
                new Weights_t[collate_out_.num_weights];
            std::vector<bool> ordinal_weights;
 
            token_list_t::iterator w_it = e_it->second.begin ();
            for (int i = 0; i < collate_out_.num_weights 
                     && w_it != e_it->second.end (); ++i, ++w_it) {
                ordinal_weights.push_back (
                    get_weight (*w_it,weights_template, i));
            }
            while (ordinal_weights.size() < collate_out_.num_weights)
                ordinal_weights.push_back (true);
 
            // next line
            if (++e_it != section.entries.end ())
                next_elm = e_it->first.name;
            else {
                issue_diag (E_SYNTAX, true, &e_it->first,
                            "unexpected end of collation section while"
                            " processing ellipsis\n");
            }
 
            if (e_it->first.token == Scanner::tok_abs_ellipsis) {
                wchar_t w_cmap_cur_val 
                    = w_cmap.find(prev_elm)->second;
                wchar_t w_cmap_end_val
                    = w_cmap.find(next_elm)->second;
            
                // the first value has already been added so don't add it again
                w_cmap_cur_val = charmap_.increment_wchar (w_cmap_cur_val);
            
                while (w_cmap_cur_val != w_cmap_end_val) {
 
                    add_to_coll (w_cmap_cur_val, weights_template,
                                 coll_value++, ordinal_weights, false);
   
                    w_cmap_cur_val = charmap_.increment_wchar (w_cmap_cur_val);
                }
 
                // add the end element to the collation array.
                add_to_coll (w_cmap_end_val, weights_template, coll_value++,
                             ordinal_weights, false);
 
            } else { 
 
                // we are incrementing the symbolic names
                std::string next_name = prev_elm;
                do {
                    if (e_it->first.token == Scanner::tok_hex_ellipsis) 
                        next_name = hex_increment (next_name);
                    else 
                        next_name = dec_increment (next_name);
 
                    w_cmap_iter it = w_cmap.find (next_name);
                    if (it != w_cmap.end()) {
                        add_to_coll (it->second, weights_template, 
                                     coll_value++, ordinal_weights, false);
                    }
                } while (next_name != next_elm);
            }
 
            // store current left-hand symbol
            next_elm = prev_elm;
        
            delete[] weights_template;
 
        } else if (e_it->first.token == Scanner::tok_undefined) {
 
            // add all characters that were not explicitely given a collation
            // value in increasing order based on their encoded values
            undefined_keyword_found_ = true;
            collate_out_.undefined_optimization = true;
            undef_char_info_.order = section_order;
            Weights_t* weights_template 
                = new Weights_t[collate_out_.num_weights];
            std::vector<bool> ordinal_weights;
 
            token_list_t::iterator w_it = e_it->second.begin ();
            for (int i = 0; i < collate_out_.num_weights 
                     && w_it != e_it->second.end(); ++i, ++w_it) {
 
                if (w_it->token == Scanner::tok_abs_ellipsis) 
                    collate_out_.undefined_optimization = false;
 
                ordinal_weights.push_back (
                    !get_weight (*w_it, weights_template, i));
            }
 
            while (ordinal_weights.size() < collate_out_.num_weights)
                ordinal_weights.push_back (false);
 
            add_missing_values (ordinal_weights, weights_template,
                                coll_value, false);
 
            delete[] weights_template;
        } 
    }
}
 
void Def::add_missing_values (const std::vector<bool> &ordinal_weights,
                              const Weights_t* weights_template,
                              unsigned int &coll_value, bool give_warning)
{
    const std::map<std::string, wchar_t>& w_cmap = charmap_.get_w_cmap ();
 
    // we want to print a warning message once if there are characters that 
    // were not given collation values.
    bool warning_issued = false;
 
    symnames_list_iter symnames_it;
    for (symnames_it = charmap_.get_symnames_list ().begin ();
         symnames_it != charmap_.get_symnames_list ().end ();
         ++symnames_it) {
 
        std::map<std::string, wchar_t>::const_iterator w_cmap_it;
        if ((w_cmap_it = w_cmap.find(*symnames_it)) != w_cmap.end()) {
            wchar_t wchar_val = (*w_cmap_it).second;
 
            coll_map_iter coll_map_it;
            if ((coll_map_it = coll_map_.find(wchar_val)) != coll_map_.end()) {
                if (coll_map_it->second.offset == UINT_MAX) {
                    if (give_warning && !warning_issued) {
                        warning_issued = true;
                        warnings_occurred_ = 
                            issue_diag (W_MISSING, false,
                                        0, "some characters in the codeset "
                                        "were not explicitly given a "
                                        "collation value\n")
                            || warnings_occurred_;
                    }
                    if (!collate_out_.undefined_optimization) {
                        add_to_coll (wchar_val, weights_template, 
                                     coll_value++, ordinal_weights, true);
                    }
                }
            }
        }
    }
    if (collate_out_.undefined_optimization){
        collate_out_.undefined_weight_idx = next_offset_++;
        undef_char_info_.offset = collate_out_.undefined_weight_idx;
        undef_char_info_.coll_val = coll_value++;
        // initialize the weight template with the undefined collation info
        Weights_t* weights = new Weights_t[collate_out_.num_weights];
 
        if (0 != weights_template) {
            for (int k = 0; k < collate_out_.num_weights; ++k) {
                weights[k].size = weights_template[k].size; 
                for (int j = 0; j < 256; ++j)
                    weights[k].weight[j] = weights_template[k].weight[j];
            }
        }
 
 
        std::vector<bool>::const_iterator it;
                
        // now go through the vector that tells us if a particular 
        // weight is based on the ordinal position of the element
        // and place the correct ordinal value in the weight vector
        // if it is.
        int i = 0;
        for (it = ordinal_weights.begin();
             it != ordinal_weights.end(); ++it, ++i)
        {
            // FIXME: this may not be correct
            if (!*it){
                weights[i].size = 1;
                weights[i].weight[0] = undef_char_info_.coll_val;
            }
        }
        undef_char_info_.weights = weights;
        
        
    }
}
 
void Def::add_to_coll (const wchar_t            w_cmap_cur_val, 
                       const Weights_t*         weights_template,
                       const unsigned int       coll_value,
                       const std::vector<bool>& ordinal_weights,
                       bool                     undefined_value)
{
    Weights_t* weights = new Weights_t[collate_out_.num_weights];
 
    if (0 != weights_template) {
        for (int k = 0; k < collate_out_.num_weights; ++k) {
            weights[k].size = weights_template[k].size; 
            for (int j = 0; j < 256; ++j)
                weights[k].weight[j] = weights_template[k].weight[j];
        }
    }
 
    // get the symbolic name from the value
    std::string name 
        = charmap_.get_rw_cmap().find (w_cmap_cur_val)->second;
    coll_map_.find (w_cmap_cur_val)->second.coll_val = coll_value;
 
    unsigned int offset;
    if (undefined_value && collate_out_.undefined_optimization)
        offset = next_offset_;
    else
        offset = next_offset_++;
 
    const std::map<std::string, wchar_t>& w_cmap = charmap_.get_w_cmap ();
 
    w_cmap_iter w_cmap_pos = w_cmap.find(name);
    coll_map_iter coll_it = coll_map_.find (w_cmap_pos->second);
    coll_it->second.offset = offset;
 
    off_mapr_.insert(std::make_pair(offset, name));
 
    std::vector<bool>::const_iterator it;
                
    // now go through the vector that tells us if a particular 
    // weight is based on the ordinal position of the element
    // and place the correct ordinal value in the weight vector
    // if it is.
    int i = 0;
    for (it = ordinal_weights.begin();
         it != ordinal_weights.end(); ++it, ++i)
    {
        if (*it){
            weights[i].size = 1;
            weights[i].weight[0] = coll_value;
        }
    }
    coll_it->second.weights = weights;
}
 
// if undefined optimization is on then only those characters that have 
// been defined should go into the valid set, otherwise all characters go
// into the set.
void Def::gen_valid_coll_mb_set ()
{
    if (!valid_coll_mb_set_.empty())
        return;
    
    for (coll_map_iter coll_it = coll_map_.begin(); 
         coll_it != coll_map_.end(); ++coll_it) {
        if (coll_it->second.offset != UINT_MAX 
            || !collate_out_.undefined_optimization){
            rmb_cmap_iter rn_cmap_it 
                = charmap_.get_rmb_cmap().find (coll_it->first);
            std::string valid = rn_cmap_it->second.substr 
                (0, rn_cmap_it->second.size() - 1);
            
            while (valid.size() > 0){
                valid_coll_mb_set_.insert (valid);
                valid = valid.substr(0, valid.size() - 1); 
            }
        }
    }
}
 
 
// In processing the collate section of the locale definition file
// we will filter out the characters that are not present in the 
// charmap file; these characters when encountered on the left-hand side
// of a weight definition for a symbolic name will be left in place
// and used as a mere collating symbol for the rest of the parsing - 
// i.e. its weights ignored and assigned the weight corresponding to
// its position in the collate section.
void Def::process_collate()
{
    issue_diag (I_STAGE, false, 0, "processing LC_COLLATE section\n");
 
    // update flags
    collate_def_found_ = true;
    // initialization
    init_coll_map ();
 
    // first preprocess the collate section
    preprocess_collate ();
 
    const std::map<std::string, wchar_t>& w_cmap = charmap_.get_w_cmap ();
 
    // process the collating symbols list
    token_list_t::iterator cs_it = cs_list_.begin ();
    for (;cs_it != cs_list_.end (); ++cs_it) {
        if (w_cmap.find (cs_it->name) != 
            w_cmap.end())
            issue_diag (E_SYNTAX, true, &*cs_it,
                        "collating-symbol %s found in character map\n",
                        cs_it->name.c_str ());
        else if (ce_map_.find (cs_it->name) != ce_map_.end())
            issue_diag (E_SYNTAX, true, &*cs_it,
                        "collating-symbol %s has already been defined as "
                        "a collating-element\n", cs_it->name.c_str ());
 
        cs_map_.insert (std::make_pair (cs_it->name, 0));
    }
 
    // process the collating elements list
    std::list<collate_elem_t>::iterator ce_it = 
        ce_list_.begin ();
    for (; ce_it != ce_list_.end (); ++ce_it) {
        if (w_cmap.find (ce_it->first.name) != 
            w_cmap.end()) {
            issue_diag (E_SYNTAX, true, &ce_it->first,
                        "collating element %s found in charmap\n", 
                        ce_it->first.name.c_str());
        }
 
        // collating element info to be stored
        ce_info_t ce_tmp;
        ce_tmp.offset   = UINT_MAX;
        ce_tmp.coll_val = UINT_MAX;
        ce_tmp.order    = 0;
        ce_tmp.weights  = 0;
 
        // array of symbolic names
        std::vector<std::string> ce_sym_array;
 
        // get the definition of this collating element
        token_list_t& ce_def_list = ce_it->second;
        token_list_t::iterator ce_def_it = ce_def_list.begin ();
        bool invalid = false;
 
        if (ce_def_it->token == Scanner::tok_string) {
            if ((ce_tmp.ce_wstr = convert_wstring (*ce_def_it)).empty())
                invalid = true;
        } else if (ce_def_it->token == Scanner::tok_sym_name) {
            for (; ce_def_it != ce_def_list.end (); ++ce_def_it) 
                ce_sym_array.push_back (ce_def_it->name);
            if ((ce_tmp.ce_wstr = convert_wstring (ce_sym_array)).empty())
                invalid = true;
        } else {
            issue_diag (E_SYNTAX, true, &*ce_def_it, 
                        "illegal collating-element expression: %s\n",
                        ce_def_it->name.c_str ());
        }
        if (invalid)
            issue_diag (W_SYNTAX, false, &*ce_def_it,
                        "character in collating element definition "
                        "not found in character map\n");
 
        if (!invalid) {
            ce_map_.insert (std::make_pair(ce_it->first.name, ce_tmp));
 
            std::string ce_str;
            std::string ce_utf8_wstr;
            for (unsigned int i = 0; i < ce_tmp.ce_wstr.size(); ++i){
                ce_str += convert_to_ext(ce_tmp.ce_wstr[i]);
                ce_utf8_wstr += utf8_encode (ce_tmp.ce_wstr[i]); 
            }
            ce_sym_map_.insert (std::make_pair(ce_str, ce_it->first.name));
            ce_wsym_map_.insert (std::make_pair (ce_utf8_wstr, 
                                                 ce_it->first.name));
        }
    }
 
    // preprocess the collation definitions and make known all
    // the symbolic names available
    preprocess_collation_definitions();
 
    // collation values 0 and 1 are reserved
    unsigned int coll_value = 2;
 
    // FIXME: assign values to the collating-elements AGAIN
    cs_it = sym_list_.begin ();
    for (; cs_it != sym_list_.end (); ++cs_it) {
        cs_map_iter csm_it = cs_map_.find (cs_it->name);
        if (csm_it != cs_map_.end ())
            csm_it->second = coll_value++;
    }
 
    // process the sections
    std::list<collate_section_t>::iterator sect_it = 
        section_list_.begin ();
    for (; sect_it != section_list_.end (); ++sect_it)
        process_order (*sect_it, coll_value);
 
    // check to make sure that all the elements in the codeset were added
    if (!undefined_keyword_found_) {
        std::vector<bool> ordinal_weights;
        for (int i = 0; i < collate_out_.num_weights; ++i) 
            ordinal_weights.push_back (false);
        collate_out_.undefined_optimization = true;
        add_missing_values (ordinal_weights, 0, coll_value, true);
    }
 
}
 
 
void Def::gen_n_to_w_coll_tables (const std::string &charp, 
                                  unsigned int tab_num)
{
    gen_valid_coll_mb_set();
    offset_tab_t tab;
    for (unsigned int k = 0; k <= UCHAR_MAX; ++k) {
        tab.off[k] = UINT_MAX;
    }
 
    tab.first_offset = -1;
    mb_cmap_iter n_cmap_it;
 
    for (unsigned int i = 0; i <= UCHAR_MAX; ++i){
        unsigned char cur_char = (unsigned char)i;
        std::string mb_char = charp;
        mb_char += (char)cur_char;
        if ((n_cmap_it = charmap_.get_mb_cmap().find (mb_char)) 
            != charmap_.get_mb_cmap().end()) {
            coll_map_iter coll_map_it = coll_map_.find (n_cmap_it->second);
            if (coll_map_it->second.offset != UINT_MAX 
                || !collate_out_.undefined_optimization) {
                tab.off[cur_char] = coll_map_it->second.offset;
                if (tab.first_offset == -1)
                    tab.first_offset = cur_char;
            }
        }
        else {
            if (valid_coll_mb_set_.find (mb_char) != valid_coll_mb_set_.end()){
                ++next_tab_num_;
                tab.off[cur_char] = next_tab_num_ | 0x80000000;
                gen_n_to_w_coll_tables (mb_char, next_tab_num_);             
                if (tab.first_offset == -1)
                    tab.first_offset = cur_char;
 
            }
            else {
                tab.off[cur_char] = UINT_MAX;
            }
        }
    }
    char_offs_.insert (std::make_pair (tab_num, tab));
}
 
 
void Def::gen_w_to_n_coll_tables (const std::string &charp, 
                                  unsigned int tab_num)
{
    // initialize a table used in ctype and collate
    create_wchar_utf8_table();
    gen_valid_coll_wchar_set();
 
    offset_tab_t tab;
    for (unsigned int k = 0; k <= UCHAR_MAX; ++k) 
        tab.off[k] = UINT_MAX;
 
    tab.first_offset = -1;
    wchar_utf8_iter wu_it;
    mb_cmap_iter n_cmap_it;
 
    for (unsigned int i = 0; i <= UCHAR_MAX; ++i) {
        unsigned char cur_char = (unsigned char)i;
        std::string   mb_char = (charp);
 
        mb_char += (char)cur_char;
 
        wu_it = wchar_utf8_to_ext_.find (mb_char);
        if (wu_it != wchar_utf8_to_ext_.end()) {
            n_cmap_it = charmap_.get_mb_cmap().find (wu_it->second);
            if (n_cmap_it != charmap_.get_mb_cmap().end ()) {
                
                coll_map_iter coll_map_it = coll_map_.find (n_cmap_it->second);
                if (coll_map_it->second.offset != UINT_MAX || 
                    !collate_out_.undefined_optimization) {            
                    tab.off[cur_char] = coll_map_it->second.offset;
                    if (tab.first_offset == -1)
                        tab.first_offset = cur_char;
                }
            }
        }
        else 
            if (valid_coll_wchar_set_.find (mb_char) != 
                valid_coll_wchar_set_.end()) {
 
                ++next_wchar_coll_tab_num_;
                tab.off[cur_char] = next_wchar_coll_tab_num_ | 0x80000000;
                gen_w_to_n_coll_tables (mb_char, 
                                        next_wchar_coll_tab_num_);
                if (tab.first_offset == -1)
                    tab.first_offset = cur_char;
            } 
            else
                tab.off[cur_char] = UINT_MAX;
    }
    
    w_to_n_coll_.insert (std::make_pair (tab_num, tab));
}
 
 
void Def::gen_n_ce_tables (const std::set<std::string>ce_elms,
                           unsigned int idx, unsigned int tab_num)
{
    if (ce_elms.size() > 0) {
        ce_offset_tab_t tab;
        for (unsigned int k = 0; k <= UCHAR_MAX; ++k) {
            tab.off[k] = UINT_MAX;
        }
        tab.first_offset = -1;
        tab.last_offset = 0;
        ce_sym_map_iter ce_sym_map_it;
        ce_map_iter ce_map_it;
 
        std::set<std::string>::const_iterator ce_elms_it;
        std::set<std::string> next_elms;
 
        for (unsigned int i = 0; i <= UCHAR_MAX; ++i) {
            next_elms.clear();
            for (ce_elms_it = ce_elms.begin(); ce_elms_it != ce_elms.end(); 
                 ce_elms_it ++) {
 
                if ((unsigned char)i == (unsigned char)((*ce_elms_it)[idx])) {
                    if ((*ce_elms_it).size() == idx + 1) {
                        if ((ce_sym_map_it = ce_sym_map_.find (*ce_elms_it))
                            != ce_sym_map_.end()) {
                            ce_map_it = ce_map_.find(ce_sym_map_it->second);
                            tab.off[i] = ce_map_it->second.offset;
                            if (tab.first_offset == -1)
                                tab.first_offset = i;
                            if ((unsigned int)tab.last_offset < i)
                                tab.last_offset = i;
                        }
                    }
                    else {
                    
                        next_elms.insert (*ce_elms_it);
                        if (tab.off[i] == UINT_MAX) {
                            ++ next_n_ce_tab_num_;
                            tab.off[i] = next_n_ce_tab_num_ | 0x80000000;
                        }
                        if (tab.first_offset == -1)
                            tab.first_offset = i;
                        if ((unsigned int)tab.last_offset < i)
                            tab.last_offset = i;
                    }
                }
            }
            // now recursively call gen_n_ce_tables if any collating
            // elements with this character were found;
            if (next_elms.size() != 0)
                gen_n_ce_tables (next_elms, idx + 1, next_n_ce_tab_num_);
        }
        n_ce_offs_.insert(std::make_pair(tab_num, tab));
 
    }
}
 
 
void Def::gen_w_ce_tables (const std::set<std::string>ce_elms,
                           unsigned int idx, unsigned int tab_num)
{
    if (ce_elms.size() > 0) {
        ce_offset_tab_t tab;
        for (unsigned int k = 0; k <= UCHAR_MAX; ++k) {
            tab.off[k] = UINT_MAX;
        }
        tab.first_offset = -1;
        tab.last_offset = 0;
        ce_sym_map_iter ce_sym_map_it;
        ce_map_iter ce_map_it;
 
        std::set<std::string>::const_iterator ce_elms_it;
        std::set<std::string> next_elms;
 
        for (unsigned int i = 0; i <= UCHAR_MAX; ++i) {
            next_elms.clear();
            for (ce_elms_it = ce_elms.begin(); ce_elms_it != ce_elms.end(); 
                 ce_elms_it ++) {
 
                if ((unsigned char)i == (unsigned char)(*ce_elms_it)[idx]) {
                    if ((*ce_elms_it).size() == idx + 1) {
                        if ((ce_sym_map_it = ce_wsym_map_.find (*ce_elms_it))
                            != ce_wsym_map_.end()) {
                            ce_map_it = ce_map_.find(ce_sym_map_it->second);
                            tab.off[i] = ce_map_it->second.offset;
                            if (tab.first_offset == -1)
                                tab.first_offset = i;
                            if ((unsigned int)tab.last_offset < i)
                                tab.last_offset = i;
                        }
                    }
                    else {
                        next_elms.insert (*ce_elms_it);
                        if (tab.off[i] == UINT_MAX) {
                            ++ next_w_ce_tab_num_;
                            tab.off[i] = next_w_ce_tab_num_ | 0x80000000;
                        }
                        if (tab.first_offset == -1)
                            tab.first_offset = i;
                        if ((unsigned int)tab.last_offset < i)
                            tab.last_offset = i;
                    }
                }
            }
            // now recursively call gen_w_ce_tables if any collating
            // elements with this character were found;
            if (next_elms.size() != 0)
                gen_w_ce_tables (next_elms, idx + 1, next_w_ce_tab_num_);
        }
        w_ce_offs_.insert(std::make_pair(tab_num, tab));
    }
}
 
 
void Def::dump_collate ()
{
    std::cout << "LC_COLLATE\n";
 
    const std::map<std::string, wchar_t>& w_cmap = charmap_.get_w_cmap ();
 
    token_list_t::iterator sl_it = sym_list_.begin ();
    for (; sl_it != sym_list_.end (); ++sl_it) {
        std::cout << sl_it->name << " % "; 
        cs_map_iter cs_it = cs_map_.find (sl_it->name);
        if (cs_it != cs_map_.end ()) {
            std::cout << cs_it->second << '\n';
            continue;
        }
 
        ce_map_iter ce_it = ce_map_.find (sl_it->name);
        if (ce_it != ce_map_.end ()) {
            std::cout << ce_it->second.coll_val << '\n';
            continue;
        }
        
        std::map<std::string, wchar_t >::const_iterator cw_it = 
            w_cmap.find (sl_it->name);
        if (cw_it != w_cmap.end ()) {
            coll_map_iter cm_it = 
                coll_map_.find (cw_it->second);
            if (cm_it != coll_map_.end ()) {
                std::cout << cm_it->second.coll_val << '\n';
                continue;
            }
        }
        std::cout << '\n';
    }
 
    std::cout << "\n\n";
 
    std::list<collate_section_t>::iterator sect_it = 
        section_list_.begin ();
    while (sect_it != section_list_.end ()) {
 
        // dump the orders
        std::cout << "order_start " << sect_it->name;
        token_list_t::iterator o_it = sect_it->order.begin ();
        for (; o_it != sect_it->order.end (); ++o_it)
            std::cout << ";" << o_it->name;
        std::cout << '\n';
 
        // for each entry in the entries list
        collate_entry_list_t::iterator e_it = sect_it->entries.begin ();
        for (; e_it != sect_it->entries.end (); ++e_it) {
 
            // dump the collation definition (weights included)
            std::cout << e_it->first.name << " ";
            if (!e_it->second.empty ()) {
                token_list_t::iterator w_it = e_it->second.begin ();
                std::cout << (w_it++)->name;
                for (; w_it != e_it->second.end (); ++w_it) 
                    std::cout << ";" << w_it->name;
            }
 
            // lookup the value associated with this collation symbol
            cs_map_iter cs_it = cs_map_.find (e_it->first.name);
            if (cs_it != cs_map_.end ()) {
                std::cout << " % " << cs_it->second << '\n';
                continue;
            }
 
            ce_map_iter ce_it = ce_map_.find (e_it->first.name);
            if (ce_it != ce_map_.end ()) {
                std::cout << " % " << ce_it->second.coll_val << '\n';
                continue;
            }
        
            std::map<std::string, wchar_t >::const_iterator cw_it = 
                w_cmap.find (e_it->first.name);
            if (cw_it != w_cmap.end ()) {
                coll_map_iter cm_it = 
                    coll_map_.find (cw_it->second);
                if (cm_it != coll_map_.end ()) {
                    std::cout << " % " << cm_it->second.coll_val << '\n';
                    continue;
                }
 
            }
            std::cout << '\n';
        }
 
        std::cout << "order_end\n";
        ++sect_it;
    }
    std::cout << "END LC_COLLATE\n";
}
 
 
void Def::write_collate (std::string dir_name)
{
    assert (!dir_name.empty());
 
    static const char lc_name[] = "LC_COLLATE";
 
    if (collate_written_)
        return;
 
    if (!collate_def_found_) {
        issue_diag (I_SKIP, false, 0,
                    "%s section not found, skipping\n", lc_name);
        return;
    }
 
    next_tab_num_ = 0;
    next_wchar_coll_tab_num_ = 0;
    std::set<std::string> ce_strs;
    ce_sym_map_iter it;
    ce_map_iter ce_mit;
    for (it = ce_sym_map_.begin (); it != ce_sym_map_.end (); ++it) {
        if ((ce_mit = ce_map_.find (it->second))->second.coll_val 
            != UINT_MAX) {
            // check to see of the largest_ce needs to be changed
            if (ce_mit->second.ce_wstr.size() + 1 
                > collate_out_.largest_ce)
                collate_out_.largest_ce =
                    unsigned (ce_mit->second.ce_wstr.size());
            
            ce_strs.insert (it->first);
        }
    }
    next_n_ce_tab_num_ = 0;
    gen_n_ce_tables(ce_strs, 0, 0);
    ce_strs.clear();
    for (it = ce_wsym_map_.begin (); it != ce_wsym_map_.end (); ++it) {
        if (ce_map_.find (it->second)->second.coll_val != UINT_MAX)
            ce_strs.insert (it->first);
    }
    next_w_ce_tab_num_ = 0;
    gen_w_ce_tables (ce_strs, 0, 0);
 
    gen_n_to_w_coll_tables ("", next_tab_num_);
    gen_w_to_n_coll_tables ("", next_wchar_coll_tab_num_);
 
    (dir_name += _RWSTD_PATH_SEP) += lc_name;
 
    issue_diag (I_OPENWR, false, 0, "writing %s\n", dir_name.c_str ());
 
    std::ofstream out (dir_name.c_str(), std::ios::binary);
    out.exceptions (std::ios::failbit | std::ios::badbit);
 
    // calculate the size of an individual weight element
    collate_out_.elm_size = collate_out_.num_weights 
        * collate_out_.longest_weight * unsigned (sizeof (unsigned int)) + 
        unsigned (sizeof (unsigned int));
 
    // the first section of the collate database is the collating
    // element information
    collate_out_.n_ce_tab_off = 0;
    collate_out_.w_ce_tab_off = collate_out_.n_ce_tab_off;
    n_ce_offs_iter n_ce_offs_it;
    for (n_ce_offs_it = n_ce_offs_.begin(); 
         n_ce_offs_it != n_ce_offs_.end (); ++n_ce_offs_it) {
        collate_out_.w_ce_tab_off += (n_ce_offs_it->second.last_offset 
                                      - n_ce_offs_it->second.first_offset + 1)
                                     * unsigned (sizeof (int));
    }
 
    // next comes the weight information
    collate_out_.weight_tab_off = collate_out_.w_ce_tab_off;
    w_ce_offs_iter w_ce_offs_it;
    for (w_ce_offs_it = w_ce_offs_.begin(); 
         w_ce_offs_it != w_ce_offs_.end(); ++w_ce_offs_it) {
        collate_out_.weight_tab_off += (w_ce_offs_it->second.last_offset 
                                        - w_ce_offs_it->second.first_offset + 1)
                                       * unsigned (sizeof (int));
    }
 
    coll_map_iter coll_map_pos;
 
    collate_out_.num_elms = unsigned (off_mapr_.size());
    if (collate_out_.undefined_optimization)
        ++collate_out_.num_elms;
 
    // now calculate the offset for the first narrow character table
    collate_out_.n_char_tab_off = collate_out_.weight_tab_off
        + collate_out_.num_elms * collate_out_.elm_size;
 
    // now calculate the offset fo the fist wide character table
    // but first we need the size of the narrow tables
    char_offs_iter char_offs_it;
    unsigned int char_offs_size = 0;
    for (char_offs_it = char_offs_.begin(); 
         char_offs_it != char_offs_.end(); ++char_offs_it) {
        char_offs_size += (UCHAR_MAX + 1 
                           - char_offs_it->second.first_offset)
            * unsigned (sizeof (unsigned int));
    }
        
    collate_out_.w_char_tab_off = collate_out_.n_char_tab_off
        + char_offs_size;
 
    // now calculate the offset for the narrow character offset table
    // but first we need the size of the wide tables
    w_to_n_coll_iter w_to_n_coll_it;
    unsigned int w_to_n_size = 0;
    for (w_to_n_coll_it = w_to_n_coll_.begin();
         w_to_n_coll_it != w_to_n_coll_.end(); ++w_to_n_coll_it) {
        w_to_n_size += (UCHAR_MAX + 1 
                        - w_to_n_coll_it->second.first_offset)
            * unsigned (sizeof (unsigned int));
    }
 
    collate_out_.n_char_off_tab_off = collate_out_.w_char_tab_off
        + w_to_n_size;
 
    // now calculate the offset for the wide character offset table
    collate_out_.w_char_off_tab_off = collate_out_.n_char_off_tab_off
        + unsigned (char_offs_.size() * sizeof (unsigned int));
 
    // calculate the offset for the narrow collating element offset table
    collate_out_.n_ce_off_tab_off = collate_out_.w_char_off_tab_off
        + unsigned (w_to_n_coll_.size() * sizeof (unsigned int));
 
    // calculate the offset for the wide collating element offset table
    collate_out_.w_ce_off_tab_off = collate_out_.n_ce_off_tab_off
        + unsigned (n_ce_offs_.size() * sizeof (unsigned int));
 
    // now calculate the offset of the first character information
    collate_out_.n_char_first_char_off = collate_out_.w_ce_off_tab_off
        + unsigned (w_ce_offs_.size() * sizeof (unsigned int));
        
    // now calculate the offset of the wide table first char info
    collate_out_.w_char_first_char_off = collate_out_.n_char_first_char_off
        + unsigned (char_offs_.size() * sizeof (unsigned char));
        
    // now calculate the offset of the narrow ce first character info
    collate_out_.n_ce_first_char_off = collate_out_.w_char_first_char_off
        + unsigned (w_to_n_coll_.size() * sizeof (unsigned char));
 
    // now calculate the offset of the wide ce first character info
    collate_out_.w_ce_first_char_off = collate_out_.n_ce_first_char_off
        + unsigned (n_ce_offs_.size() * sizeof (unsigned char));
 
    // now calculate the offset of the narrow ce last character info
    collate_out_.n_ce_last_char_off = collate_out_.w_ce_first_char_off
        + unsigned (w_ce_offs_.size() * sizeof (unsigned char));
 
    // now calculate the offset of the wide ce last character info
    collate_out_.w_ce_last_char_off = collate_out_.n_ce_last_char_off
        + unsigned (n_ce_offs_.size() * sizeof (unsigned char));
 
    // now calculate the offset of the codeset name
    collate_out_.codeset_off = collate_out_.w_ce_last_char_off
        + unsigned (w_ce_offs_.size() * sizeof (unsigned char));
 
    // finally calculate the offset of the charmap name
    collate_out_.charmap_off = collate_out_.codeset_off 
        + unsigned (charmap_.get_code_set_name().size()) + 1;
        
 
 
    // print out the collate struct
    out.write ((char*)&collate_out_, sizeof(collate_out_));
 
    unsigned int i;
    for (n_ce_offs_it = n_ce_offs_.begin(); 
         n_ce_offs_it != n_ce_offs_.end(); ++n_ce_offs_it) {
        for (i = (unsigned int)n_ce_offs_it->second.first_offset; 
             i <= (unsigned int)n_ce_offs_it->second.last_offset; ++i)
            out.write ((char*)&n_ce_offs_it->second.off[i], 
                       sizeof (n_ce_offs_it->second.off[i]));
    }
 
    for (w_ce_offs_it = w_ce_offs_.begin(); 
         w_ce_offs_it != w_ce_offs_.end(); ++w_ce_offs_it) {
        for (i = (unsigned int)w_ce_offs_it->second.first_offset; 
             i <= (unsigned int)w_ce_offs_it->second.last_offset; ++i)
            out.write ((char*)&w_ce_offs_it->second.off[i], 
                       sizeof (w_ce_offs_it->second.off[i]));
    }
 
        
    // now print out the weight array
    unsigned int maxw = UINT_MAX;
    Weights_t* weights;
    bool undefined_written = false;
 
    const std::map<std::string, wchar_t>& w_cmap = charmap_.get_w_cmap ();
 
    std::size_t off_idx = 0;
    off_mapr_iter current_off = off_mapr_.begin();
    for (; current_off != off_mapr_.end(); ++off_idx) {
 
        // check the current offset
        if (current_off->first != off_idx 
            && off_idx != undef_char_info_.offset)
            issue_diag (E_RANGE, true, 0, 
                        "current_off : %d, index : %d, giving up\n",
                        current_off->first, off_idx);
 
        if (collate_out_.undefined_optimization 
            && off_idx == undef_char_info_.offset) {
            undefined_written = true;
            weights = undef_char_info_.weights;
            out.write ((char*)&undef_char_info_.order, 
                       sizeof (undef_char_info_.order));
            for (int j = 0; j < collate_out_.num_weights; ++j) {
                for (int k = 0; k < weights[j].size; ++k) {
                    out.write ((char*)&weights[j].weight[k],
                               sizeof (weights[j].weight[k]));
                }
                for (int c = weights[j].size; 
                     c < collate_out_.longest_weight; ++c)
                    out.write ((char*)&maxw, sizeof (maxw)); 
            }
            continue;
        }
 
        w_cmap_iter w_cmap_pos = w_cmap.find 
            (current_off->second);
        if (w_cmap_pos != w_cmap.end()) {
            coll_map_pos = coll_map_.find (w_cmap_pos->second);
            out.write ((char*)&coll_map_pos->second.order, 
                       sizeof (coll_map_pos->second.order));
            weights = coll_map_pos->second.weights;
            for (int j = 0; j < collate_out_.num_weights; ++j) {
                for (int k = 0; k < weights[j].size; ++k) {
                    out.write ((char*)&weights[j].weight[k],
                               sizeof (weights[j].weight[k]));
                }
                for (int c = weights[j].size; 
                     c < collate_out_.longest_weight; ++c)
                    out.write ((char*)&maxw, sizeof (maxw)); 
            }
            ++current_off;
        }
 
        else {
            ce_map_iter ce_map_it = ce_map_.find (current_off->second);
            if (ce_map_it != ce_map_.end() 
                && ce_map_it->second.coll_val != UINT_MAX) {
                out.write ((char*)&ce_map_it->second.order, 
                           sizeof (ce_map_it->second.order));
                weights = ce_map_it->second.weights;
                for (int j = 0; j < collate_out_.num_weights; ++j) {
                    for (int k = 0; k < weights[j].size; ++k) {
                        out.write ((char*)&weights[j].weight[k],
                                   sizeof (weights[j].weight[k]));
                    }
                    for (int c = weights[j].size; 
                         c < collate_out_.longest_weight; ++c)
                        out.write ((char*)&maxw, sizeof (maxw)); 
                }
 
            }
            ++current_off;
        }
    }            
 
    // maske sure that we got to the undefined value
    if (!undefined_written && collate_out_.undefined_optimization) {
        if (off_idx != undef_char_info_.offset)
            issue_diag (E_RANGE, true, 0, 
                        "current_off : %d, index : %d, giving up\n",
                        undef_char_info_.offset, off_idx);
 
        weights = undef_char_info_.weights;
        out.write ((char*)&undef_char_info_.order, 
                   sizeof (undef_char_info_.order));
        for (int j = 0; j < collate_out_.num_weights; ++j) {
            for (int k = 0; k < weights[j].size; ++k) {
                out.write ((char*)&weights[j].weight[k],
                           sizeof (weights[j].weight[k]));
            }
            for (int c = weights[j].size; 
                 c < collate_out_.longest_weight; ++c)
                out.write ((char*)&maxw, sizeof (maxw)); 
        }
    }
 
    // now print out the narrow character tables
    for (char_offs_it = char_offs_.begin(); 
         char_offs_it != char_offs_.end(); ++char_offs_it) {
        for (unsigned int c = char_offs_it->second.first_offset; 
             c <= UCHAR_MAX; ++c) {
            out.write ((const char*)&char_offs_it->second.off[c],
                       sizeof (char_offs_it->second.off[c]));
        }
    }
 
    // now print out the wide character tables
    for (w_to_n_coll_it = w_to_n_coll_.begin(); 
         w_to_n_coll_it != w_to_n_coll_.end(); ++w_to_n_coll_it) {
        for (unsigned int c = w_to_n_coll_it->second.first_offset; 
             c <= UCHAR_MAX; ++c) {
            out.write ((const char*)&w_to_n_coll_it->second.off[c],
                       sizeof (w_to_n_coll_it->second.off[c]));
        }
    }
        
    // now print the narrow character table offsets
    unsigned int current_offset = 0;
    for (char_offs_it = char_offs_.begin();
         char_offs_it != char_offs_.end(); ++char_offs_it) {
        out.write ((const char*)&current_offset, sizeof (current_offset));
        current_offset += (UCHAR_MAX + 1 
                           - char_offs_it->second.first_offset);
    }
 
    // now print the wide character table offsets
    current_offset = 0;
    for (w_to_n_coll_it = w_to_n_coll_.begin();
         w_to_n_coll_it != w_to_n_coll_.end(); ++w_to_n_coll_it) {
        out.write ((const char*)&current_offset, sizeof (current_offset));
        current_offset += (UCHAR_MAX + 1 
                           - w_to_n_coll_it->second.first_offset);
    }
 
    // now print the narrow ce table offsets
    current_offset = 0;
    for (n_ce_offs_it = n_ce_offs_.begin();
         n_ce_offs_it != n_ce_offs_.end(); ++n_ce_offs_it) {
        out.write ((const char*)&current_offset, sizeof (current_offset));
        current_offset += (n_ce_offs_it->second.last_offset 
                           - n_ce_offs_it->second.first_offset + 1);
    }
 
    // now print the wide ce table offsets
    current_offset = 0;
    for (w_ce_offs_it = w_ce_offs_.begin();
         w_ce_offs_it != w_ce_offs_.end(); ++w_ce_offs_it) {
        out.write ((const char*)&current_offset, sizeof (current_offset));
        current_offset += (w_ce_offs_it->second.last_offset 
                           - w_ce_offs_it->second.first_offset + 1);
    }
 
 
    // now print out the narrow character tables starting character
    for (char_offs_it = char_offs_.begin();
         char_offs_it != char_offs_.end(); ++char_offs_it) {
        const char off = char ((char_offs_it->second).first_offset);
        out << off;
    }        
        
    // now print out the wide character tables starting character
    for (w_to_n_coll_it = w_to_n_coll_.begin();
         w_to_n_coll_it != w_to_n_coll_.end(); ++w_to_n_coll_it) {
        const char off = char ((w_to_n_coll_it->second).first_offset);
        out << off;
    }        
        
    // now print out the narrow ce tables starting character
    for (n_ce_offs_it = n_ce_offs_.begin();
         n_ce_offs_it != n_ce_offs_.end(); ++n_ce_offs_it) {
        const char off = char ((n_ce_offs_it->second).first_offset);
        out << off;
    }        
 
    // now print out the wide ce tables starting character
    for (w_ce_offs_it = w_ce_offs_.begin();
         w_ce_offs_it != w_ce_offs_.end(); ++w_ce_offs_it) {
        const char off = char ((w_ce_offs_it->second).first_offset);
        out << off;
    }        
 
    // now print out the narrow ce tables ending character
    for (n_ce_offs_it = n_ce_offs_.begin();
         n_ce_offs_it != n_ce_offs_.end(); ++n_ce_offs_it) {
        const char off = char ((n_ce_offs_it->second).last_offset);
        out << off;
    }        
 
    // now print out the wide ce tables ending character
    for (w_ce_offs_it = w_ce_offs_.begin();
         w_ce_offs_it != w_ce_offs_.end(); ++w_ce_offs_it) {
        const char off = char ((w_ce_offs_it->second).last_offset);
        out << off;
    }        
        
    // finally write the codeset and charmap names
    out << charmap_.get_code_set_name() << std::ends
        << charmap_.get_charmap_name() << std::ends;
}
 
 
void Def::init_coll_map() {
 
    rw_cmap_iter rw_cmap_pos;
    collate_info_t tmp = {UINT_MAX, UINT_MAX, 0, 0};
 
    for (rw_cmap_pos = charmap_.get_rw_cmap().begin(); 
         rw_cmap_pos != charmap_.get_rw_cmap().end();
         ++rw_cmap_pos) {
        coll_map_.insert (std::make_pair (rw_cmap_pos->first, tmp));
    }
}
 
 
// In the preprocess_collate member function the collate section is 
// preprocessed and the result of the preprocessing is stored in-memory
// as a number of lists; the content of these lists is then preprocessed 
// acoording to the reorder statements; the result is then passed
// on to process_collate.
void Def::preprocess_collate ()
{
    int nesting_level = 0;
 
    do {
        // fetch next token
        next = scanner_.next_token();
        
        switch (next.token) {
        case Scanner::tok_end:
            next = scanner_.next_token();
            if (next.token == Scanner::tok_collate) {
                // end of collation block
                if (nesting_level == 0) 
                    return;
 
                nesting_level--;
                scanner_.close ();
            } else
                issue_diag (E_SYNTAX, true, &next,
                            "wrong section name in END directive\n");
 
            break;
        case Scanner::tok_copy: {
            next = scanner_.next_token();
            if (next.token != Scanner::tok_string)
                issue_diag (E_SYNTAX, true, &next,
                            "expected string following \"copy\" directive\n");
 
            // bump up the nesting level
            ++nesting_level;
 
            issue_diag (I_STAGE, false, 0, "processing copy directive\n");
 
            // open the file
            scanner_.open (get_pathname (strip_quotes (next.name), next.file));
 
            // get comment char and escape char; 
            // these informations are stored by the scanner
            while ((next = scanner_.next_token ()).token 
                   != Scanner::tok_collate ){
                // the LC_IDENTIFICATION section may also have a 
                // LC_COLLATE token that will mess up the parsing
                if (next.token == Scanner::tok_ident) {
                    while ((next = scanner_.next_token()).token
                           != Scanner::tok_end );
                    next = scanner_.next_token();
                }
            }
            
            break;
        }
 
        // a collating element definition defines a symbolic name that
        // represents symbolically the congregation of two other symbolic
        // names which have to be present in the character map;
        // the form of the phrase is:
        // collating-element sym from ("string" | (sym sym+))
        case Scanner::tok_coll_elem: {
            next = scanner_.next_token();
            // we expect a symbolic name
            if (next.token != Scanner::tok_sym_name) 
                issue_diag (E_SYNTAX, true, &next,
                            "symbolic name expected following "
                            "collating-element\n");
            // symbolic name
            token_t sym (next);
 
            next = scanner_.next_token ();
            if (next.token != Scanner::tok_from)
                issue_diag (E_SYNTAX, true, &next,
                            "illegal collating-element expression\n");
 
            token_list_t ce_def_list;
            next = scanner_.next_token ();
            if (next.token == Scanner::tok_string) {
                ce_def_list.push_back (next);
                ce_list_.push_back (std::make_pair(sym,ce_def_list));
            } else if (next.token == Scanner::tok_sym_name) {
                do {
                    ce_def_list.push_back (next);
                    next = scanner_.next_token();
                } while (next.token != Scanner::tok_nl);
                ce_list_.push_back (std::make_pair(sym,ce_def_list));
            } else 
                issue_diag (E_SYNTAX, true, &next,
                            "illegal collating-element expression\n");
            break;
        }
        case Scanner::tok_coll_sym:
            next = scanner_.next_token();
            if (next.token != Scanner::tok_sym_name)
                issue_diag (E_SYNTAX, true, &next,
                            "symbolic name expected following "
                            "collating-symbol\n");
            cs_list_.push_back (next);
            break;
        case Scanner::tok_script:
            next = scanner_.next_token ();
            if (next.token != Scanner::tok_sym_name)
                issue_diag (W_SYNTAX, false, &next,
                            "expecting script name, got %s\n", 
                            next.name.c_str ());
            script_list_.push_back(next);
            break;
        case Scanner::tok_sym_name:
            sym_list_.push_back (next);
            break;
        case Scanner::tok_order_start:
            preprocess_order();
            break;
 
        case Scanner::tok_reorder:
            preprocess_reorder ();
            break;
 
        case Scanner::tok_reorder_section:
            preprocess_reorder_section ();
            break;
 
        default:
            break;
        }
    }
    while (Scanner::tok_end_tokens != next.token);
}
 
 
// The task of preprocess_order is to parse and model the content of the 
// order sections in the input files
void Def::preprocess_order ()
{
    // one order-start keyword has been encountered; 
    // push a collate_section down the list and use it
    while (next.token != Scanner::tok_order_end) {
        section_list_.push_back (collate_section_t());
        collate_section_t& section = section_list_.back ();
 
        next = scanner_.next_token ();
        
        // store the orders
        while (next.token != Scanner::tok_nl) {
            if (next.token == Scanner::tok_sym_name) {
                // symbolic name, has to be a section name
                section.name = next.name;
            } else {
                // might be a combination of order and position
                // do the same until a better way is found
                section.order.push_back (next);
            }
 
            // fetch next token
            next = scanner_.next_token ();
        }
 
        // unnamed sections
        if (section.name == "")
            section.name = "unnamed";
 
        issue_diag (I_STAGE, false, 0,
                    "processing %s order\n", section.name.c_str ());
 
        // store the collation statements
        while (!(next.token == Scanner::tok_order_end || 
                 next.token == Scanner::tok_order_start )) {
 
            next = scanner_.next_token ();
            switch (next.token) {
            case Scanner::tok_end:
                issue_diag (E_SYNTAX, true, &next,
                            "unexpected END directive while "
                            "parsing collate section\n");
            case  Scanner::tok_nl:
                continue;
            case Scanner::tok_comment:
                scanner_.ignore_line ();
                break;
            case Scanner::tok_undefined:
            case Scanner::tok_sym_name: 
            {
                token_t sym (next);
                section.entries.push_back (
                    std::make_pair(sym,token_list_t()));
                collate_entry_t& entry = section.entries.back ();
                
                next = scanner_.next_token ();
                while (next.token != Scanner::tok_nl) {
                    entry.second.push_back (next);
                    next = scanner_.next_token ();
                }
                break;
            }
                        default:
                                break;
            }
        }
    }
}
 
 
// Handles the reorder statements in the form:
//    reorder-after <sym>
//    <sym> <sym><sym>...
// OR
//    reorder-after <sym>
//    ..  <sym><sym>.....
// followed by reorder-end statement or another reorder-after
void Def::preprocess_reorder ()
{
    while (true) {
        // process one or more statements grouped 
        // under a reorder section
        
        // retrieve the symbol
        next = scanner_.next_token ();
        if (next.token != Scanner::tok_sym_name)
            issue_diag (E_SYNTAX, true, &next,
                       "expecting symbolic name, got %s\n",
                       next.name.c_str ());
            
        // store the symbolic name
        token_t sym (next);
        collate_entry_list_t entries_list;
        while (true) {
 
            // process the statements
            next = scanner_.next_token ();
 
            if (next.token == Scanner::tok_nl )
                continue;
            else if (   next.token == Scanner::tok_sym_name
                     || next.token == Scanner::tok_hex_ellipsis) {
                // the line will contain one single symbolic name 
                // or a complete collation statement
                collate_entry_t entry;
                entry.first = next;
 
                next = scanner_.next_token ();
                while (next.token != Scanner::tok_nl) { 
                    entry.second.push_back (next);
                    next = scanner_.next_token ();
                }
 
                // add the entry to the list
                entries_list.push_back (entry);
            } else if (next.token == Scanner::tok_reorder) {
                // call insert_entry
                if (!entries_list.empty ())
                    insert_entries (sym, entries_list);
                entries_list.clear ();
                break;
            } else if (next.token == Scanner::tok_reorder_end ) {
                // call insert_entry
                // call insert_entry
                if (!entries_list.empty ())
                    insert_entries (sym, entries_list);
                return;
            } else {
                issue_diag (E_SYNTAX, true, &next,
                            "unexpected token : %s in a reorder block\n",
                            next.name.c_str ());
            }
        }
    }
}
 
 
void Def::preprocess_reorder_section ()
{
    while (next.token != Scanner::tok_reorder_section_end) {
        // process one or more statements grouped 
        // under a reorder section
        
        // retrieve the symbol
        next = scanner_.next_token ();
        token_t sym (next);
 
        // process the statements
        while (!(next.token == Scanner::tok_reorder_section || 
                 next.token == Scanner::tok_reorder_section_end)) {
 
            next = scanner_.next_token ();
            switch (next.token) {
            case Scanner::tok_nl:
                continue;
            case Scanner::tok_sym_name: {
                // the line will contain one single symbolic name 
                // which is the name of a section
                next = scanner_.next_token ();
                if (next.token != Scanner::tok_sym_name) {
                    issue_diag (E_SYNTAX, true, &next,
                                "expecting section name, got %s\n",
                                next.name.c_str ());
                }
 
                std::list<collate_section_t>::iterator beg = 
                    section_list_.begin ();
                std::list<collate_section_t>::iterator end = 
                    section_list_.end ();
                std::list<collate_section_t>::iterator ref_it, mov_it;
                for (ref_it = beg; 
                     ref_it->name != sym.name && ref_it != end; 
                     ++ref_it);
                for (mov_it = beg; 
                     mov_it->name != next.name && mov_it != end; 
                     ++mov_it);
 
                if (ref_it == end || mov_it == end)
                    issue_diag (E_SYNTAX, true, &next,
                                "missing sections %s %s requested\n",
                                sym.name.c_str (), next.name.c_str ());
 
                // replace it
                section_list_.splice (++ref_it, section_list_,
                                      mov_it, mov_it);
 
                // replace the insertion point
                sym = next;
                break;
            }
            default:
                issue_diag (E_SYNTAX, true, &next,
                            "unexpected token : %s in a "
                            "reorder section block\n",
                            next.name.c_str ());
                break;
            }
 
        }
    }
}
 
 
void Def::list_collate ()
{
    // print the script list
    token_list_t::iterator script_it = script_list_.begin ();
    while (script_it != script_list_.end ()) {
        std::cout << "script  " << (script_it++)->name << '\n';
    }
    std::cout << '\n';
 
    // print the collating symbols' list
    token_list_t::iterator cs_it = cs_list_.begin ();
    while (cs_it != cs_list_.end ()) {
        std::cout << "collating-symbol     " << (cs_it++)->name << '\n';
    }
    std::cout << '\n';
 
    // print the collating elements list
    std::list<collate_elem_t>::iterator ce_it = ce_list_.begin ();
    while (ce_it != ce_list_.end ()) {
        std::cout << "collating-element " << ce_it->first.name
                  << " from ";
        token_list_t::iterator ce_def_it = ce_it->second.begin ();
        for (;ce_def_it != ce_it->second.end (); ce_def_it++) {
            std::cout << ce_def_it->name << ";";
        }
        std::cout << '\n';
    }    
    std::cout << '\n';
 
    // print out general/global symbols' list
    token_list_t::iterator sym_it = sym_list_.begin ();
    while (sym_it != sym_list_.end ()) {
        std::cout << (sym_it++)->name << '\n';
    }
    std::cout << '\n';
 
    // print out sections
    std::list<collate_section_t>::iterator sc_it = 
        section_list_.begin ();
    while (sc_it != section_list_.end ()) {
        // print prolog
        std::cout << "order_start ";
        if (sc_it->name != "unnamed")
            std::cout << sc_it->name << ";";
        
        token_list_t::iterator it = sc_it->order.begin ();
        while (it != sc_it->order.end ())
            std::cout << (it++)->name << ";";
        std::cout << '\n';
 
        std::list<collate_entry_t>::iterator e_it = 
            sc_it->entries.begin ();
        while (e_it != sc_it->entries.end ()) {
            std::cout << e_it->first.name << "   ";
            token_list_t::iterator w_it =  e_it->second.begin ();
            while (w_it != e_it->second.end ())
                std::cout << (w_it++)->name << ";";
            std::cout << '\n';
            ++e_it;
        }
 
        ++sc_it;
    }
    std::cout << '\n' << "order_end\n";
}
 
 
// FIXME - modify the algorithm to get log(N) complexity
// Inserts an entry inside the section information;
// searching for the appropriate entry in the list is
// done in linear time 
bool Def::insert_entries (token_t& s, collate_entry_list_t& e)
{
    // first remove these entries if found
    collate_entry_list_t::iterator r_it = e.begin ();
    for (; r_it != e.end (); ++r_it) 
        remove_entry (*r_it);
 
    // first search through the symbols list; if found check
    // the collate_entry object; it should not have weights
    token_list_t::iterator it = sym_list_.begin ();
    for (; it != sym_list_.end (); ++it) {
        if (it->name != s.name) 
            continue;
 
        // FIXME - all statements have to have no weights if they are
        // to be inserted after collation symbols outside section boundaries;
        // found it;  check one collate_entry_t object
        if (!e.begin ()->second.empty ()) {
            issue_diag (E_REORD, true, &s,
                        "requested reorder-after: reference "
                        "symbol %s was found outside section boundaries "
                        "and the objects to be reordered are collation "
                        "definitions\n", s.name.c_str ());
        }
 
        // insert the symbolic name there
        collate_entry_list_t::iterator e_it = e.begin ();
        for (; e_it != e.end (); ++e_it) 
            it = sym_list_.insert (++it, e_it->first);
 
        return true;
    }
 
    // if not successful then search in each section
    std::list<collate_section_t>::iterator sect_it = section_list_.begin ();
    while (sect_it != section_list_.end ()) {
        collate_entry_list_t::iterator e_it = sect_it->entries.begin ();
        for (; e_it != sect_it->entries.end (); ++e_it) {
            if (e_it->first.name != s.name)
                continue;
 
            // found it; insert entry
            sect_it->entries.insert (++e_it, e.begin (), e.end ());
            return true;
        }
        ++sect_it;
    }
 
    issue_diag (W_REORD, false, &s,
                "requested reorder-after: reference "
                "symbol %s was not found \n", s.name.c_str ());
 
    return false;
}
 
 
// FIXME - modify the algorithm to get log(N) complexity
void Def::remove_entry (collate_entry_t& e)
{
    // search in the sym_list_ and in the sections
    if (e.second.empty ()) {
        token_list_t::iterator it = sym_list_.begin ();
        for (; it != sym_list_.end (); ++it) {
            if (it->name != e.first.name) 
                continue;
 
            sym_list_.erase (it);
            return;
        }
    }
 
    // search in the sections
    std::list<collate_section_t>::iterator sect_it = section_list_.begin ();
    while (sect_it != section_list_.end ()) {
        std::list<collate_entry_t>::iterator e_it = sect_it->entries.begin ();
        for (; e_it != sect_it->entries.end (); ++e_it) {
            if (e_it->first.name != e.first.name)
                continue;
 
            sect_it->entries.erase (e_it);
            return;
        }
        ++sect_it;
    }
    return ;
}
 
 
 
bool Def::get_weight ( token_t&     w,
                       Weights_t*   weights, 
                       int          weight_num)
{
    const std::map<std::string, wchar_t>& w_cmap = charmap_.get_w_cmap ();
 
    bool ret = false;
    weights[weight_num].size = 1;
    ce_map_iter ce_map_it;
    if(w.token == Scanner::tok_sym_name) {
        w_cmap_iter w_cmap_pos = w_cmap.find (w.name);
        if (w_cmap_pos != w_cmap.end()){
            coll_map_iter coll_map_pos = coll_map_.find(w_cmap_pos->second);
 
            if (coll_map_pos->second.coll_val == UINT_MAX) 
                issue_diag (E_SYMUSED, true, 
                            &w, "symbolic name %s "
                            "used as weight before being assigned a "
                            "collation value\n", w.name.c_str());
 
            // store weight
            weights[weight_num].weight[0] = coll_map_pos->second.coll_val;
        }
        else if ((ce_map_it = ce_map_.find (w.name)) 
                 != ce_map_.end())
            weights[weight_num].weight[0] = ce_map_it->second.coll_val;
        else {
            cs_map_iter cs_it = cs_map_.find (w.name);
            if (cs_it == cs_map_.end ()) 
                issue_diag (E_SYNTAX, true, 
                            &w, "symbolic name %s not found\n", 
                            w.name.c_str());
              
            weights[weight_num].weight[0] = cs_it->second;
        }
    }
    else if (w.token == Scanner::tok_char_value) {
        // the weight is given in numerical form
        const char* next_val =
            std::strchr (w.name.c_str (), scanner_.escape_char ());
 
        assert (0 != next_val);
 
        const char* next_wt = std::strchr (w.name.c_str (), ';');
 
        while (weight_num < collate_out_.num_weights) {
 
            unsigned c;
 
            for (c = 0; *next_val && (!next_wt || next_val < next_wt); ++c) {
 
                const char* end = 0;
 
                weights [weight_num].weight [c] =
                    unsigned (scanner_.convert_escape (next_val, &end, true));
 
                assert (0 != end);
 
                next_val = end;
            }
 
            weights [weight_num++].size = c;
 
            if (next_wt)
                next_wt = std::strchr (next_val, ';');
                
        }
 
    }
    else if(w.token == Scanner::tok_ignore) {
        // use the special weight 0 for IGNORE weights
        weights[weight_num].size = 1;
        weights[weight_num].weight[0] = 0;
    }
    else if (w.token == Scanner::tok_string) {
        // the weights are given either in symbolic name form (e.g.,
        // "<symbolic-name>" or in the form of a quoted multibyte
        // character string (e.g., "\001\d010\x16\")
        const std::string tmp (w.name.substr (1, w.name.size() - 2));
 
        // keeps track of the length of the weight
        unsigned char k = 0;
 
        // iterate thru the string content and retrieve the symbols
        std::string::const_iterator it = tmp.begin ();
 
        const char escape = scanner_.escape_char ();
 
        while (it != tmp.end ()) {
            // weight in string form
            std::string wsym;
 
            // next comes a symbolic name
            if (*it == '<') {
                while (*it != '>') {
                    if (*it == escape) 
                        ++it;
 
                    wsym += *it++;
                }
                wsym += *it++;
 
                // wsym has the symbolic name, lookup for it in 
                // the character map, collating-symbol map, 
                // and collating-element map
                w_cmap_iter w_cm_pos = w_cmap.find(wsym);
                cs_map_iter cs_it = cs_map_.find (wsym);
                ce_map_iter ce_it = ce_map_.find (wsym);
 
                if (w_cm_pos != w_cmap.end()) {
                    // is in the character map, check its associated
                    // collation value
                    coll_map_iter coll_it = coll_map_.find(w_cm_pos->second);
                    if (coll_it == coll_map_.end())
                        issue_diag (E_SYMUSED, true, &w, 
                                    "weight %s not defined\n", 
                                    wsym.c_str());
                    if (coll_it->second.coll_val == UINT_MAX) 
                        issue_diag (E_SYMUSED, true, 
                                    &w, "symbolic name %s "
                                    "used as weight before being assigned a "
                                    "collation value\n", wsym.c_str());
 
                    weights[weight_num].weight[k++] = coll_it->second.coll_val;
                }
                else if (ce_it != ce_map_.end()) {
                    // it is in the collating-element map
                    weights[weight_num].weight[k++] = ce_it->second.coll_val;
                }
                else if (cs_it != cs_map_.end()) {
                    // it is in the collating-symbol map
                    weights[weight_num].weight[k++] = cs_it->second;
                }
                else {
                    // it is not in any of the maps, that's an error
                    issue_diag (E_SYNTAX, true, &w, 
                                "symbolic name %s not found\n", 
                                wsym.c_str());
                }
            }
            else if (*it == escape) {
 
                // weight is given in a quoted escape form
                const char* const beg = tmp.c_str () + (it - tmp.begin ());
                const char*       end = 0;
 
                weights [weight_num].weight [k++] =
                    unsigned (scanner_.convert_escape (beg, &end, true));
 
                assert (0 != end);
 
                it += end - beg;
 
                break;
            }
            else 
                issue_diag (E_SYNTAX, true, &w,
                            "illegal string content as a weight");
        }
 
        weights[weight_num].size = k;
        if (k > collate_out_.longest_weight)
            collate_out_.longest_weight = k;
 
    }
    else if (w.token == Scanner::tok_abs_ellipsis)
        // return true if ellipsis are embedded in the weight
        ret = true;
    else
        warnings_occurred_ = 
            issue_diag (W_SYNTAX, false, &w,
                        "illegal token %s found in collation definition\n",
                        w.name.c_str()) || warnings_occurred_;
 
    return ret;
}
 
 
// if undefined optimization is on then only those characters that have 
// been defined should go into the valid set, otherwise all characters go
// into the set.
void Def::gen_valid_coll_wchar_set () {
 
    if (!valid_coll_wchar_set_.empty())
        return;
 
    create_wchar_utf8_table();
 
    for (coll_map_iter coll_it = coll_map_.begin(); 
         coll_it != coll_map_.end(); ++coll_it) {
        if (coll_it->second.offset != UINT_MAX 
            || !collate_out_.undefined_optimization){
 
            std::string valid = utf8_encode (coll_it->first);
            valid = valid.substr (0, valid.size() - 1);
            
            while (valid.size() > 0){
                valid_coll_wchar_set_.insert (valid);
                valid = valid.substr(0, valid.size() - 1); 
            }
        }
    }
}