/* tag: Tom Lord Tue Dec 4 14:41:49 2001 (unidata-generate.c) */ /* unidata-generate.c - * **************************************************************** * Copyright (C) 2000 Tom Lord * * See the file "COPYING" for further information about * the copyright and warranty status of this work. */ #include "hackerlab/arrays/pow2-array-compact.h" #include "hackerlab/arrays/pow2-array-print.h" #include "hackerlab/bitsets/bits.h" #include "hackerlab/bitsets/bits-print.h" #include "hackerlab/bitsets/uni-bits.h" #include "hackerlab/rx-posix/regexps.h" #include "hackerlab/uni/unidata.h" #include "hackerlab/unidata/db-macros.h" #include "hackerlab/unidata/case-db-macros.h" #include "hackerlab/cmd/main.h" static t_uchar * program_name = "unidata-generate"; static t_uchar * usage = "[options] input-file"; static t_uchar * version_string = "1.0"; #define OPTS(OP, OP2) \ OP (opt_help_msg, "h", "help", 0, \ "Display a help message and exit.") \ OP (opt_long_help, "H", 0, 0, \ "Display a verbose help message and exit.") \ OP (opt_version, "V", "version", 0, \ "Display a release identifier string") \ OP2 (opt_version, 0, 0, 0, "and exit.") \ OP (opt_verbose, "v", "verbose", 0, \ "Display information about the unidata database on stderr.") static t_uchar long_help[] = ("Generate C source code from teh unidata database.\n" "This program is used during the build process of \n" "the hackerlab C library.\n"); enum options { OPTS (OPT_ENUM, OPT_IGN) }; struct opt_desc opts[] = { OPTS (OPT_DESC, OPT_DESC) {-1, 0, 0, 0, 0} }; struct unidata { t_unicode code_value; t_uchar * character_name; enum uni_general_category general_category; t_uint canonical_combining_class; enum uni_bidi_category bidi_category; struct uni_decomposition_mapping character_decomposition_mapping; int decimal_digit_value; int digit_value; struct uni_numeric_value numeric_value; int mirrored; t_uchar * unicode_1_name; t_uchar * comment_10646; t_unicode uppercase_mapping; t_unicode lowercase_mapping; t_unicode titlecase_mapping; }; AR_TYPEDEF (t_unicode, unicode); AR_TYPEDEF (struct uni_decomposition_mapping, uni_decomposition_mapping); #define UNIDATA_FIELDS \ UNIDATA_FIELD(unidata_code_value, "[0-9a-fA-F]\\+") \ UNIDATA_FIELD(unidata_character_name, "[^;]\\+") \ UNIDATA_FIELD(unidata_general_category, "[A-Z][a-z]") \ UNIDATA_FIELD(unidata_canonical_combining_class, "[0-9]\\+") \ UNIDATA_FIELD(unidata_bidi_category, "[A-Z]\\{1,3\\}") \ UNIDATA_FIELD(unidata_character_decomposition_mapping, "[^;]*") \ UNIDATA_FIELD(unidata_decimal_digit_value, "[0-9]*") \ UNIDATA_FIELD(unidata_digit_value, "[0-9]*") \ UNIDATA_FIELD(unidata_numeric_value, "[0-9/]*") \ UNIDATA_FIELD(unidata_mirrored, "[YN]") \ UNIDATA_FIELD(unidata_unicode_1_name, "[^;]*") \ UNIDATA_FIELD(unidata_comment_10646, "[^;]*") \ UNIDATA_FIELD(unidata_uppercase_mapping, "[0-9a-fA-F]*") \ UNIDATA_FIELD(unidata_lowercase_mapping, "[0-9a-fA-F]*") \ UNIDATA_FIELDX(unidata_titlecase_mapping, "[0-9a-fA-F]*") /* positions within pmatch data of fields: */ enum unidata_field_positions { #undef UNIDATA_FIELD #define UNIDATA_FIELD(A,B) A, #undef UNIDATA_FIELDX #define UNIDATA_FIELDX(A,B) A, unidata_entire_line = 0, UNIDATA_FIELDS n_unidata_fields }; /* regexp for unidata fields */ static char unidata_regexp_source[] = #undef UNIDATA_FIELD #define UNIDATA_FIELD(A,B) "\\(" B "\\)" ";" #undef UNIDATA_FIELDX #define UNIDATA_FIELDX(A,B) "\\(" B "\\)" "^" UNIDATA_FIELDS "$"; static void unidata_parse (struct unidata * unidata, int * in_range, int line_no, t_uchar * line, long len); /* see the implementation static void unidata_free (struct unidata * ud);*/ static int unidata_next (struct unidata * data, int * in_range, int * line_no, int fd); static regex_t * unidata_regexp (void) { static int done = 0; static regex_t answer; if (done) return &answer; if (regcomp (&answer, unidata_regexp_source, REG_NEWLINE)) panic ("internal regcomp error for unidata_regexp_source"); done = 1; return &answer; } /* Parsed format of a unidata line. */ #undef UNI_DECOMPOSITION_TYPE #define UNI_DECOMPOSITION_TYPE(NAME) \ "<" #NAME ">" "[[:cut %:]]\\|" static char uni_decomposition_type_regexp_source[] = "^" "[[:(" UNI_DECOMPOSITION_TYPES "[[:cut 2:]]" "):]]"; static regex_t * uni_decomposition_type_regexp (void) { static int done = 0; static regex_t answer; if (done) return &answer; if (regcomp (&answer, uni_decomposition_type_regexp_source, 0)) panic ("internal regcomp error for uni_decomposition_type_regexp_source"); done = 1; return &answer; } static char uni_range_first_regexp_source[] = "^[^;]*;[^;]*[[:([[:( First>;[[:cut 1:]]):]]\\|[[:( Last>;[[:cut 2:]]):]]):]]"; static regex_t * uni_range_first_regexp (void) { static int done = 0; static regex_t answer; if (done) return &answer; if (regcomp (&answer, uni_range_first_regexp_source, 0)) panic ("internal regcomp error for uni_range_first_regexp"); done = 1; return &answer; } void unidata_parse (struct unidata * unidata, int * in_range, int line_no, t_uchar * line, long len) { int errn; int match; regmatch_t pmatch[n_unidata_fields]; regmatch_t * pmatch_p = pmatch; t_uint n; t_uchar * syntax_error; if (len && (line[len - 1] == '\n')) --len; if (len && (line[len - 1] == '\r')) --len; match = regnexec (unidata_regexp (), (char *)line, len, n_unidata_fields, &pmatch_p, 0); if (match) { syntax_error = "parsing entire line into fields"; syntax_exit: safe_printfmt (2, "unicode database:%d: syntax error (%s)\n", line_no, syntax_error); safe_printfmt (2, "\t%.*s\n", (int)len, line); panic ("unrecoverable error parsing unicode database"); } { regmatch_t range_pmatch[1]; regmatch_t * range_pmatch_p = range_pmatch; match = regnexec (uni_range_first_regexp (), line, len, 1, &range_pmatch_p, 0); switch (match) { case 0: *in_range = range_pmatch[0].final_tag; break; case REG_NOMATCH: *in_range = 0; break; default: safe_printfmt (2, "unicode database:%d:\n", line_no); panic ("internal regexp error"); break; } } if (cvt_hex_to_uint (&errn, &n, line + pmatch[unidata_code_value].rm_so, pmatch[unidata_code_value].rm_eo - pmatch[unidata_code_value].rm_so)) { syntax_error = "parsing code value"; goto syntax_exit; } else unidata->code_value = (t_unicode)n; unidata->character_name = str_save_n (lim_use_must_malloc, line + pmatch[unidata_character_name].rm_so, pmatch[unidata_character_name].rm_eo - pmatch[unidata_character_name].rm_so); unidata->general_category = uni_general_category_lookup_n (line + pmatch[unidata_general_category].rm_so, pmatch[unidata_general_category].rm_eo - pmatch[unidata_general_category].rm_so); if (cvt_decimal_to_uint (&errn, &unidata->canonical_combining_class, line + pmatch[unidata_canonical_combining_class].rm_so, pmatch[unidata_canonical_combining_class].rm_eo - pmatch[unidata_canonical_combining_class].rm_so)) { syntax_error = "parsing canonical combining class"; goto syntax_exit; } unidata->bidi_category = uni_bidi_category_lookup_n (line + pmatch[unidata_bidi_category].rm_so, pmatch[unidata_bidi_category].rm_eo - pmatch[unidata_bidi_category].rm_so); if (pmatch[unidata_character_decomposition_mapping].rm_eo == pmatch[unidata_character_decomposition_mapping].rm_so) { unidata->character_decomposition_mapping.type = uni_decomposition_none; unidata->character_decomposition_mapping.decomposition = 0; } else { regmatch_t decomp_pmatch[1]; regmatch_t * decomp_pmatch_p = decomp_pmatch; t_uchar * str; size_t len; match = regnexec (uni_decomposition_type_regexp (), line + pmatch[unidata_character_decomposition_mapping].rm_so, pmatch[unidata_character_decomposition_mapping].rm_eo - pmatch[unidata_character_decomposition_mapping].rm_so, 1, &decomp_pmatch_p, 0); if (match) { syntax_error = "parsing character decomposition mapping type"; goto syntax_exit; } unidata->character_decomposition_mapping.type = decomp_pmatch[0].final_tag - 1; unidata->character_decomposition_mapping.decomposition = 0; str = line + pmatch[unidata_character_decomposition_mapping].rm_so + decomp_pmatch[0].rm_eo; len = (pmatch[unidata_character_decomposition_mapping].rm_eo - pmatch[unidata_character_decomposition_mapping].rm_so) - decomp_pmatch[0].rm_eo; while (1) { t_uint d; t_uchar * d_start; size_t d_len; while (len && char_is_space (*str)) { ++str; --len; } if (!len) break; d_start = str; d_len = 0; while (len && char_is_xdigit (*str)) { ++d_len; ++str; --len; } if (cvt_hex_to_uint (&errn, &d, d_start, d_len)) { syntax_error = "parsing decomposition value"; goto syntax_exit; } ar_push_unicode (&unidata->character_decomposition_mapping.decomposition, d); } } if (pmatch[unidata_decimal_digit_value].rm_so == pmatch[unidata_decimal_digit_value].rm_eo) unidata->decimal_digit_value = 10; else if (cvt_decimal_to_int (&errn, &unidata->decimal_digit_value, line + pmatch[unidata_decimal_digit_value].rm_so, pmatch[unidata_decimal_digit_value].rm_eo - pmatch[unidata_decimal_digit_value].rm_so)) { syntax_error = "parsing decimal digit value"; goto syntax_exit; } if (pmatch[unidata_digit_value].rm_so == pmatch[unidata_digit_value].rm_eo) unidata->digit_value = -1; else if (cvt_decimal_to_int (&errn, &unidata->digit_value, line + pmatch[unidata_digit_value].rm_so, pmatch[unidata_digit_value].rm_eo - pmatch[unidata_digit_value].rm_so)) { syntax_error = "parsing digit value"; goto syntax_exit; } if (pmatch[unidata_numeric_value].rm_so == pmatch[unidata_numeric_value].rm_eo) unidata->numeric_value.numerator = -1; else { t_uchar * slash; t_uchar * str; size_t len; str = line + pmatch[unidata_numeric_value].rm_so; len = pmatch[unidata_numeric_value].rm_eo - pmatch[unidata_numeric_value].rm_so; slash = str_chr_index_n (str, len, '/'); if (!slash) { unidata->numeric_value.denominator = 1; if (cvt_decimal_to_int (&errn, &unidata->numeric_value.numerator, str, len)) { syntax_error = "parsing numeric value"; goto syntax_exit; } } else { if (cvt_decimal_to_uint (&errn, &unidata->numeric_value.numerator, str, slash - str)) { syntax_error = "parsing numerator of numeric value"; goto syntax_exit; } if (cvt_decimal_to_int (&errn, &unidata->numeric_value.denominator, slash + 1, len - (slash - str) - 1)) { syntax_error = "parsing denominator of numeric value"; goto syntax_exit; } } } switch (line[pmatch[unidata_mirrored].rm_so]) { case 'Y': unidata->mirrored = 1; break; case 'N': unidata->mirrored = 0; break; default: syntax_error = "parsing mirrored"; goto syntax_exit; } if (pmatch[unidata_unicode_1_name].rm_so == pmatch[unidata_unicode_1_name].rm_eo) unidata->unicode_1_name = 0; else unidata->unicode_1_name = str_save_n (lim_use_must_malloc, line + pmatch[unidata_unicode_1_name].rm_so, pmatch[unidata_unicode_1_name].rm_eo - pmatch[unidata_unicode_1_name].rm_so); if (pmatch[unidata_comment_10646].rm_so == pmatch[unidata_comment_10646].rm_eo) unidata->comment_10646 = 0; else unidata->comment_10646 = str_save_n (lim_use_must_malloc, line + pmatch[unidata_comment_10646].rm_so, pmatch[unidata_comment_10646].rm_eo - pmatch[unidata_comment_10646].rm_so); if (pmatch[unidata_uppercase_mapping].rm_so == pmatch[unidata_uppercase_mapping].rm_eo) unidata->uppercase_mapping = 0; else if (cvt_hex_to_uint (&errn, &n, line + pmatch[unidata_uppercase_mapping].rm_so, pmatch[unidata_uppercase_mapping].rm_eo - pmatch[unidata_uppercase_mapping].rm_so)) { syntax_error = "parsing uppercase mapping"; goto syntax_exit; } else unidata->uppercase_mapping = n; if (pmatch[unidata_lowercase_mapping].rm_so == pmatch[unidata_lowercase_mapping].rm_eo) unidata->lowercase_mapping = 0; else if (cvt_hex_to_uint (&errn, &n, line + pmatch[unidata_lowercase_mapping].rm_so, pmatch[unidata_lowercase_mapping].rm_eo - pmatch[unidata_lowercase_mapping].rm_so)) { syntax_error = "parsing lowercase mapping"; goto syntax_exit; } else unidata->lowercase_mapping = n; if (pmatch[unidata_titlecase_mapping].rm_so == pmatch[unidata_titlecase_mapping].rm_eo) unidata->titlecase_mapping = 0; else if (cvt_hex_to_uint (&errn, &n, line + pmatch[unidata_titlecase_mapping].rm_so, pmatch[unidata_titlecase_mapping].rm_eo - pmatch[unidata_titlecase_mapping].rm_so)) { syntax_error = "parsing titlecase mapping"; goto syntax_exit; } else unidata->titlecase_mapping = n; } #if 0 /* Not used, adding calls to it might heisenbug.. */ void unidata_free (struct unidata * ud) { lim_free (lim_use_must_malloc, ud->character_name); ar_free ((void **)&ud->character_decomposition_mapping.decomposition, lim_use_must_malloc); lim_free (lim_use_must_malloc, ud->unicode_1_name); lim_free (lim_use_must_malloc, ud->comment_10646); } #endif int unidata_next (struct unidata * data, int * in_range, int * line_no, int fd) { int errn; t_uchar * line; long len; ++*line_no; if (0 > vfdbuf_next_line (&errn, &line, &len, fd)) { safe_printfmt (2, "unicode database (%d): %s\n", errn, errno_to_string (errn)); panic ("unrecoverable error parsing unicode database\n"); } if (!line) return 0; unidata_parse (data, in_range, *line_no, line, (size_t)len); return 1; } static void print_t_uint16 (int fd, void * elt) { safe_printfmt (fd, "%d", (int)(*(t_uint16 *)elt)); } static void print_t_case (int fd, void * elt) { struct uni_case_mapping * mapping; mapping = (struct uni_case_mapping *)elt; safe_printfmt (fd, "{ 0x%04lX, 0x%04lX, 0x%04lX }", (unsigned long)mapping->upper, (unsigned long)mapping->lower, (unsigned long)mapping->title); /* safe_printfmt (fd, "{ 0x%l04X, 0x%l04X }", (unsigned long)mapping->upper, (unsigned long)mapping->lower); */ /* safe_printfmt (fd, "{ 0x%l04X }", (unsigned long)mapping->upper); */ } static void print_t_uint8 (int fd, void * elt) { safe_printfmt (fd, "%d", (int)(*(t_uint8 *)elt)); } static void print_t_int16 (int fd, void * elt) { safe_printfmt (fd, "%d", (int)(*(t_int16 *)elt)); } int main (int argc, char * argv[]) { int errn; t_uchar * input_file; t_uchar * bits_file; t_uchar * bits_h_file; t_uchar * db_file; t_uchar * db_h_file; t_uchar * case_db_file; t_uchar * case_db_h_file; t_uchar * combine_db_file; t_uchar * combine_db_h_file; t_uchar * decomp_db_file; t_uchar * decomp_db_h_file; int input_fd; int bits_fd; int bits_h_fd; int db_fd; int db_h_fd; int case_db_fd; int case_db_h_fd; int combine_db_fd; int combine_db_h_fd; int decomp_db_fd; int decomp_db_h_fd; bits * sets; bits all_chars; int x; int line_no; int has_decomp; int max_decomp; int total_decomp; t_unicode worst_decomp = 0; int non0_combine; int uppers; int lowers; int titles; int uppers_and_lowers; int uppers_and_title; int two_case; int three_case; int have_case[256]; int have_case2[512]; int numerics; int non_dec_digits; pow2_array_rules db_rules; pow2_array db_array; pow2_array_rules case_rules; pow2_array case_db_array; pow2_array_rules combine_rules; pow2_array combine_db_array; pow2_array_rules decomp_rules; pow2_array decomp_db_array; union { struct uni_decomposition_mapping * dmp; void *void_ptr; } decompositions; int verbose; int o; struct opt_parsed * option; verbose = 0; option = 0; while (1) { o = opt_standard (lim_use_must_malloc, &option, opts, &argc, argv, program_name, usage, version_string, long_help, opt_help_msg, opt_long_help, opt_version); if (o == opt_none) break; switch (o) { default: safe_printfmt (2, "unhandled option `%s'\n", option->opt_string); panic ("internal error parsing arguments"); usage_error: opt_usage (2, argv[0], program_name, usage, 1); panic_exit (); #if 0 bogus_arg: safe_printfmt (2, "ill-formed argument for `%s' (`%s')\n", option->opt_string, option->arg_string); goto usage_error; #endif case opt_verbose: verbose = 1; break; } } rx_set_dfa_cache_threshold (2 * 2097152); if (argc != 2) goto usage_error; input_file = argv[1]; bits_file = "bitsets.c"; bits_h_file = "bitsets.h"; db_file = "db.c"; db_h_file = "db.h"; case_db_file = "case-db.c"; case_db_h_file = "case-db.h"; combine_db_file = "combine-db.c"; combine_db_h_file = "combine-db.h"; decomp_db_file = "decomp-db.c"; decomp_db_h_file = "decomp-db.h"; input_fd = safe_open (input_file, O_RDONLY, 0); bits_fd = safe_open (bits_file, O_WRONLY | O_CREAT | O_EXCL, 0644); bits_h_fd = safe_open (bits_h_file, O_WRONLY | O_CREAT | O_EXCL, 0644); db_fd = safe_open (db_file, O_WRONLY | O_CREAT | O_EXCL, 0644); db_h_fd = safe_open (db_h_file, O_WRONLY | O_CREAT | O_EXCL, 0644); case_db_fd = safe_open (case_db_file, O_WRONLY | O_CREAT | O_EXCL, 0644); case_db_h_fd = safe_open (case_db_h_file, O_WRONLY | O_CREAT | O_EXCL, 0644); combine_db_fd = safe_open (combine_db_file, O_WRONLY | O_CREAT | O_EXCL, 0644); combine_db_h_fd = safe_open (combine_db_h_file, O_WRONLY | O_CREAT | O_EXCL, 0644); decomp_db_fd = safe_open (decomp_db_file, O_WRONLY | O_CREAT | O_EXCL, 0644); decomp_db_h_fd = safe_open (decomp_db_h_file, O_WRONLY | O_CREAT | O_EXCL, 0644); if (vfdbuf_buffer_fd (&errn, input_fd, 0, O_RDONLY, 0)) panic ("unable to buffer input file"); if (vfdbuf_buffer_fd (&errn, bits_fd, 0, O_WRONLY, 0)) panic ("unable to buffer bitset output file"); if (vfdbuf_buffer_fd (&errn, bits_h_fd, 0, O_WRONLY, 0)) panic ("unable to buffer bitset header output file"); if (vfdbuf_buffer_fd (&errn, db_fd, 0, O_WRONLY, 0)) panic ("unable to buffer db output file"); if (vfdbuf_buffer_fd (&errn, db_h_fd, 0, O_WRONLY, 0)) panic ("unable to buffer db header output file"); if (vfdbuf_buffer_fd (&errn, case_db_fd, 0, O_WRONLY, 0)) panic ("unable to buffer db output file"); if (vfdbuf_buffer_fd (&errn, case_db_h_fd, 0, O_WRONLY, 0)) panic ("unable to buffer db header output file"); if (vfdbuf_buffer_fd (&errn, combine_db_fd, 0, O_WRONLY, 0)) panic ("unable to buffer db output file"); if (vfdbuf_buffer_fd (&errn, combine_db_h_fd, 0, O_WRONLY, 0)) panic ("unable to buffer db header output file"); if (vfdbuf_buffer_fd (&errn, decomp_db_fd, 0, O_WRONLY, 0)) panic ("unable to buffer db output file"); if (vfdbuf_buffer_fd (&errn, decomp_db_h_fd, 0, O_WRONLY, 0)) panic ("unable to buffer db header output file"); sets = (bits *)must_malloc (uni_n_categories * sizeof (bits)); for (x = 0; x < uni_n_categories; ++x) sets[x] = bits_alloc (0, uni_bits_tree_rule); all_chars = bits_alloc (0, uni_bits_tree_rule); { static t_uint16 db_default_page[1 << 11]; static struct uni_case_mapping case_default_page[16]; static t_uint8 combine_default_page[16]; static t_int16 decomp_default_page[16]; { int i; t_uint16 v; v = unidata__assemble_db (0, 10, 0, uni_bidi_ON, uni_general_category_Cn); for (i = 0; i < (sizeof (db_default_page) / sizeof (db_default_page[0])); ++i) { db_default_page[i] = v; } } db_rules = make_pow2_array_rules (lim_use_must_malloc, sizeof (t_uint16), (void *)db_default_page, 11, (size_t)0x3ff, 0, (size_t)0x7ff); case_rules = make_pow2_array_rules (lim_use_must_malloc, sizeof (struct uni_case_mapping), (void *)case_default_page, 16, (size_t)0x1f, 12, (size_t)0xf, 8, (size_t)0xf, 4, (size_t)0xf, 0, (size_t)0xf); combine_rules = make_pow2_array_rules (lim_use_must_malloc, sizeof (t_uint8), (void *)combine_default_page, 16, (size_t)0x1f, 12, (size_t)0xf, 8, (size_t)0xf, 4, (size_t)0xf, 0, (size_t)0xf); decomp_rules = make_pow2_array_rules (lim_use_must_malloc, sizeof (t_int16), (void *)decomp_default_page, 16, (size_t)0x1f, 12, (size_t)0xf, 8, (size_t)0xf, 4, (size_t)0xf, 0, (size_t)0xf); } { struct uni_decomposition_mapping decomp; decomp.type = uni_decomposition_none; decomp.decomposition = 0; decompositions.dmp = 0; ar_push_uni_decomposition_mapping (&decompositions.dmp, decomp); } db_array = pow2_array_alloc (lim_use_must_malloc, db_rules); case_db_array = pow2_array_alloc (lim_use_must_malloc, case_rules); combine_db_array = pow2_array_alloc (lim_use_must_malloc, combine_rules); decomp_db_array = pow2_array_alloc (lim_use_must_malloc, decomp_rules); line_no = 0; has_decomp = 0; max_decomp = 0; total_decomp = 0; non0_combine = 0; uppers = 0; uppers_and_title = 0; uppers_and_lowers = 0; lowers = 0; titles = 0; for (x = 0; x < 256; ++x) have_case[x] = 0; for (x = 0; x < 512; ++x) have_case2[x] = 0; two_case = 0; three_case = 0; numerics = 0; non_dec_digits = 0; while (1) { struct unidata data; int in_range; struct unidata data_2; if (!unidata_next (&data, &in_range, &line_no, input_fd)) break; /* eof */ if (verbose && !(line_no % 500)) safe_printfmt (2, "line %d\n", line_no); if (data.general_category == uni_general_category_Cn) { safe_printfmt (2, "Character U+%X is an unassigned character in unidata.txt!", data.code_value); panic ("unidata.txt is broken"); } { t_uint16 dbv; dbv = unidata__assemble_db (1, data.decimal_digit_value, data.mirrored, data.bidi_category, data.general_category); *(t_uint16 *)pow2_array_ref (db_array, data.code_value) = dbv; } if (data.digit_value >= 0) ++non_dec_digits; if (data.numeric_value.numerator >= 0) { ++numerics; } if (data.character_decomposition_mapping.decomposition) { size_t size; ++has_decomp; size = ar_size ((void *)data.character_decomposition_mapping.decomposition); total_decomp += (int)size; if (size > max_decomp) { max_decomp = (int) size; worst_decomp = data.code_value; } } if (data.uppercase_mapping) { ++uppers; if (data.lowercase_mapping) ++uppers_and_lowers; if (data.titlecase_mapping) ++uppers_and_title; } if (data.lowercase_mapping) ++lowers; if (data.titlecase_mapping) ++titles; { int q; q = !!data.uppercase_mapping + !!data.lowercase_mapping + !!data.titlecase_mapping; if (q == 2) ++two_case; else if (q == 3) ++three_case; } if (data.uppercase_mapping || data.lowercase_mapping || data.titlecase_mapping) { have_case[0xff & (data.code_value >> 8)] = 1; have_case2[0x1ff & (data.code_value >> 7)] = 1; } if (data.uppercase_mapping || data.lowercase_mapping || data.titlecase_mapping) { struct uni_case_mapping * mapping; mapping = (struct uni_case_mapping *)pow2_array_ref (case_db_array, data.code_value); mapping->upper = data.uppercase_mapping; mapping->lower = data.lowercase_mapping; mapping->title = data.titlecase_mapping; } if (data.canonical_combining_class) { ++non0_combine; *(t_uint8 *)pow2_array_ref (combine_db_array, data.code_value) = data.canonical_combining_class; } if (data.character_decomposition_mapping.type != uni_decomposition_none) { struct uni_decomposition_mapping decomp; t_int16 index; if ((1 << 16) <= ar_size (decompositions.void_ptr)) panic ("too many characters have decomposition mappings -- unidata-generate needs to be modified\n"); index = (t_int16)ar_size (decompositions.void_ptr); decomp.type = data.character_decomposition_mapping.type; decomp.decomposition = (t_unicode *)ar_copy ((void *)data.character_decomposition_mapping.decomposition, sizeof (t_unicode)); ar_push_uni_decomposition_mapping (&decompositions.dmp, decomp); *(t_int16 *)pow2_array_ref (decomp_db_array, data.code_value) = index; } bits_adjoin (sets[data.general_category], data.code_value); if ((data.general_category != uni_general_category_Cs) && (data.general_category != uni_general_category_Co)) bits_adjoin (all_chars, data.code_value); if (in_range && (in_range != 1)) { safe_printfmt (2, "unicode database:%d: found end of range (\"..., Last>;\") without start of range\n", line_no); panic ("unrecoverable error parsing unicode database"); } else if (in_range) { if (!unidata_next (&data_2, &in_range, &line_no, input_fd)) { safe_printfmt (2, "unicode database:%d: end of line encountered looking for range end\n", line_no); panic ("unrecoverable error parsing unicode database"); } /* safe_printfmt (2, "line %d (range end)\n", line_no); */ if (in_range != 2) { safe_printfmt (2, "unicode database:%d: missing end of range (\"..., Last>;\")\n", line_no); panic ("unrecoverable error parsing unicode database"); } bits_fill_range (sets[data.general_category], data.code_value, data_2.code_value + 1); if ((data.general_category != uni_general_category_Cs) && (data.general_category != uni_general_category_Co)) { bits_fill_range (all_chars, data.code_value, data_2.code_value + 1); } { t_uint16 dbv; int q; dbv = unidata__assemble_db (1, data.decimal_digit_value, data.mirrored, data.bidi_category, data.general_category); for (q = data.code_value; q <= data_2.code_value; ++q) *(t_uint16 *)pow2_array_ref (db_array, q) = dbv; if (data.uppercase_mapping || data.lowercase_mapping || data.titlecase_mapping) { struct uni_case_mapping * mapping; mapping = (struct uni_case_mapping *)pow2_array_ref (case_db_array, data.code_value); mapping->upper = data.uppercase_mapping; mapping->lower = data.lowercase_mapping; mapping->title = data.titlecase_mapping; for (q = data.code_value; q <= data_2.code_value; ++q) *(struct uni_case_mapping *)pow2_array_ref (case_db_array, q) = *mapping; } if (data.canonical_combining_class) { for (q = data.code_value; q <= data_2.code_value; ++q) { ++non0_combine; *(t_uint8 *)pow2_array_ref (combine_db_array, q) = data.canonical_combining_class; } } if (data.character_decomposition_mapping.type != uni_decomposition_none) { t_int16 index; index = (t_int16)(ar_size (decompositions.void_ptr) - 1); for (q = data.code_value; q <= data_2.code_value; ++q) *(t_int16 *)pow2_array_ref (decomp_db_array, q) = index; } } } } /* "The Private Use character outside of the BMP (U+F0000..U+FFFFD, * U+100000..U+10FFFD) are not listed. These correspond to surrogate * pairs where the first surrogate is in the High Surrogate Private * Use section." - The UnicodeData File Format Version 3.0.0 */ bits_fill_range (sets[uni_general_category_Co], 0xf0000, 0xffffe); bits_fill_range (all_chars, 0xf0000, 0xffffe); bits_fill_range (sets[uni_general_category_Co], 0x100000, 0x10fff2); bits_fill_range (all_chars, 0x100000, 0x10fff2); /* These should appear to be unassigned characters in the database. * * If you encounter a file with private-use characters you don't * recognize, that's an error. * * If you have an application that uses private use characters, * you should make a modified unidata.txt assigning them appropriate * categories (not Co). */ { enum uni_general_category cat; for (cat = uni_first_synthetic_category; cat < uni_n_categories; ++cat) { int first_char; int x; bits it; first_char = uni_general_category_names[cat].name[0]; it = bits_alloc (0, uni_bits_tree_rule); for (x = 0; uni_general_category_names[x].name; ++x) { if ( (uni_general_category_names[x].name[0] == first_char) && (sets[x])) { bits_union (it, sets[x]); } } sets[cat] = it; } safe_printfmt (bits_fd, "/* This file automatically generated by unidata-generate */\n\n"); safe_printfmt (bits_fd, "#include \"bitsets.h\"\n"); safe_printfmt (bits_fd, "\n\n"); safe_printfmt (bits_h_fd, "/* This file automatically generated by unidata-generate */\n\n"); safe_printfmt (bits_h_fd, "#include \"hackerlab/bitsets/bits.h\"\n"); safe_printfmt (bits_h_fd, "\n\n"); for (x = 0; x < uni_n_categories; ++x) { t_uchar * name; t_uchar * stub; name = str_alloc_cat (lim_use_must_malloc, "unidata_bitset_", uni_general_category_names[x].name); stub = str_alloc_cat (lim_use_must_malloc, name, "_"); bits_compact (sets[x]); bits_print (bits_fd, sets[x], name, stub, 0, 0, 0); safe_printfmt (bits_fd, "\n\f\n"); bits_print (bits_h_fd, sets[x], name, stub, 0, 1, 0); safe_printfmt (bits_h_fd, "\n\f\n"); } } bits_compact (all_chars); bits_print (bits_fd, all_chars, "unidata_bitset_universal", "unidata_bitset_universal_", 0, 0, 0); safe_printfmt (bits_fd, "\n\n"); bits_print (bits_h_fd, all_chars, "unidata_bitset_universal", "unidata_bitset_universal_", 0, 1, 0); safe_printfmt (bits_h_fd, "\n\n"); if (verbose) { safe_printfmt (2, "%d characters have a decomposition mapping\n", has_decomp); safe_printfmt (2, "%d characters in the widest decomp mapping\n", max_decomp); safe_printfmt (2, "U+%X is the code value of the widest decomp mapping\n", worst_decomp); safe_printfmt (2, "%d characters (total) in decomp mappings\n", total_decomp); safe_printfmt (2, "%d have a non-0 canonical combining class\n", non0_combine); safe_printfmt (2, "%d have uppercase mappings\n", uppers); safe_printfmt (2, "%d have lowercase mappings\n", lowers); safe_printfmt (2, "%d have titlecase mappings\n", titles); safe_printfmt (2, "%d have upper and lower mappings\n", uppers_and_lowers); safe_printfmt (2, "%d have upper and title mappings\n", uppers_and_title); safe_printfmt (2, "%d have lower and title mappings\n", two_case - (uppers_and_lowers + uppers_and_title)); } if (verbose) { { int case_pages; int case_half_pages; case_pages = 0; case_half_pages = 0; for (x = 0; x < 256; ++x) if (have_case[x]) ++case_pages; for (x = 0; x < 512; ++x) if (have_case2[x]) ++case_half_pages; safe_printfmt (2, "%d pages (256 characters/page) have case mappings\n", case_pages); safe_printfmt (2, "%d half pages (128 characters/page) have case mappings\n", case_half_pages); safe_printfmt (2, "%d characters have exactly two case mappings\n", two_case); safe_printfmt (2, "%d characters have exactly three case mappings\n", three_case); } safe_printfmt (2, "%d characters have a numeric value\n", numerics); safe_printfmt (2, "%d characters are non-decimal digits\n", non_dec_digits); } safe_printfmt (db_fd, "/* This file automatically generated by unidata-generate */\n\n"); safe_printfmt (db_fd, "#include \"db.h\"\n"); safe_printfmt (db_fd, "\n\n"); pow2_array_compact (db_array, 0, 0, 0); pow2_array_print (db_fd, db_array, "unidata__db", "unidata__db", 0, 0, 0, "t_uint16", print_t_uint16); safe_printfmt (db_fd, "\n\n"); safe_printfmt (db_h_fd, "/* This file automatically generated by unidata-generate */\n\n"); safe_printfmt (db_h_fd, "#include \"hackerlab/arrays/pow2-array.h\"\n"); safe_printfmt (db_h_fd, "\n\n"); pow2_array_print (db_h_fd, db_array, "unidata__db", "unidata__db", 1, "unidata__db_ref", 0, "t_uint16", 0); safe_printfmt (db_h_fd, "\n\n"); safe_printfmt (case_db_fd, "/* This file automatically generated by unidata-generate */\n\n"); safe_printfmt (case_db_fd, "#include \"case-db.h\"\n"); safe_printfmt (case_db_fd, "\n\n"); pow2_array_compact (case_db_array, 0, 0, 0); pow2_array_print (case_db_fd, case_db_array, "unidata__case_db", "unidata__case_db", 0, 0, 0, "struct uni_case_mapping", print_t_case); safe_printfmt (case_db_fd, "\n\n"); safe_printfmt (case_db_h_fd, "/* This file automatically generated by unidata-generate */\n\n"); safe_printfmt (case_db_h_fd, "#include \"hackerlab/arrays/pow2-array.h\"\n"); safe_printfmt (case_db_h_fd, "#include \"hackerlab/unidata/case-db-macros.h\"\n"); safe_printfmt (case_db_h_fd, "\n\n"); pow2_array_print (case_db_h_fd, case_db_array, "unidata__case_db", "unidata__case_db", 1, "unidata__case_db_ref", 0, "struct uni_case_mapping", 0); safe_printfmt (case_db_h_fd, "\n\n"); safe_printfmt (combine_db_fd, "/* This file automatically generated by unidata-generate */\n\n"); safe_printfmt (combine_db_fd, "#include \"combine-db.h\"\n"); safe_printfmt (combine_db_fd, "\n\n"); pow2_array_compact (combine_db_array, 0, 0, 0); pow2_array_print (combine_db_fd, combine_db_array, "unidata__combine_db", "unidata__combine_db", 0, 0, 0, "t_uint8", print_t_uint8); safe_printfmt (combine_db_fd, "\n\n"); safe_printfmt (combine_db_h_fd, "/* This file automatically generated by unidata-generate */\n\n"); safe_printfmt (combine_db_h_fd, "#include \"hackerlab/arrays/pow2-array.h\"\n"); safe_printfmt (combine_db_h_fd, "#include \"hackerlab/unidata/combine-db-macros.h\"\n"); safe_printfmt (combine_db_h_fd, "\n\n"); pow2_array_print (combine_db_h_fd, combine_db_array, "unidata__combine_db", "unidata__combine_db", 1, "unidata__combine_db_ref", 0, "t_uint8", 0); safe_printfmt (combine_db_h_fd, "\n\n"); safe_printfmt (decomp_db_fd, "/* This file automatically generated by unidata-generate */\n\n"); safe_printfmt (decomp_db_fd, "#include \"decomp-db.h\"\n"); safe_printfmt (decomp_db_fd, "\n\n"); pow2_array_compact (decomp_db_array, 0, 0, 0); pow2_array_print (decomp_db_fd, decomp_db_array, "unidata__decomp_db", "unidata__decomp_db", 0, 0, 0, "t_int16", print_t_int16); safe_printfmt (decomp_db_fd, "\n\n"); { size_t d; size_t n_d; size_t off; n_d = ar_size (decompositions.void_ptr); safe_printfmt (decomp_db_fd, "static t_unicode unidata_decomposition_data[] =\n"); safe_printfmt (decomp_db_fd, "{\n"); for (d = 0; d < n_d; ++d) { size_t c; size_t n_c; n_c = ar_size (decompositions.dmp[d].decomposition); for (c = 0; c < n_c; ++c) { safe_printfmt (decomp_db_fd, " 0x%04X,\n", decompositions.dmp[d].decomposition[c]); } safe_printfmt (decomp_db_fd, " 0x0,\n"); } safe_printfmt (decomp_db_fd, "};\n\n"); safe_printfmt (decomp_db_fd, "struct uni_decomposition_mapping unidata_decomposition_table[] =\n"); safe_printfmt (decomp_db_fd, "{\n"); off = 0; for (d = 0; d < n_d; ++d) { safe_printfmt (decomp_db_fd, " { %d, unidata_decomposition_data + %lu },\n", decompositions.dmp[d].type, (unsigned long)off); off += ar_size ((void *)decompositions.dmp[d].decomposition) + 1; } safe_printfmt (decomp_db_fd, "};\n\n"); } safe_printfmt (decomp_db_h_fd, "/* This file automatically generated by unidata-generate */\n\n"); safe_printfmt (decomp_db_h_fd, "#include \"hackerlab/arrays/pow2-array.h\"\n"); safe_printfmt (decomp_db_h_fd, "#include \"hackerlab/unidata/decomp-db-macros.h\"\n"); safe_printfmt (decomp_db_h_fd, "\n\n"); pow2_array_print (decomp_db_h_fd, decomp_db_array, "unidata__decomp_db", "unidata__decomp_db", 1, "unidata__decomp_db_ref", 0, "t_int16", 0); safe_printfmt (decomp_db_h_fd, "\n\n"); safe_printfmt (decomp_db_h_fd, "extern struct uni_decomposition_mapping unidata_decomposition_table[];\n\n\n"); safe_close (input_fd); safe_close (bits_fd); safe_close (bits_h_fd); safe_close (db_fd); safe_close (db_h_fd); safe_close (case_db_fd); safe_close (case_db_h_fd); safe_close (combine_db_fd); safe_close (combine_db_h_fd); safe_close (decomp_db_fd); safe_close (decomp_db_h_fd); return 0; }