ARB
|
#include <climits>
#include <netinet/in.h>
#include <arb_file.h>
#include <arb_diff.h>
#include <arbdbt.h>
#include "gb_key.h"
#include "gb_compress.h"
#include "gb_dict.h"
#include "arb_progress.h"
Go to the source code of this file.
Classes | |
struct | O_gbdByKey |
union | DictTree |
struct | FullDictTree |
struct | SingleDictTree |
Macros | |
#define | COMPRESSIBLE(type) ((type) >= GB_BYTES && (type)<=GB_STRING) |
#define | DICT_MEM_WEIGHT 4 |
#define | WORD_HELPFUL(wordlen, occurrences) |
#define | MIN_WORD_LEN 8 |
#define | MAX_WORD_LEN 50 |
#define | MAX_BROTHERS |
#define | MAX_DIFFER |
#define | INCR_DIFFER 1 |
#define | DICT_STRING_INCR 1024 |
#define | LEN_BITS 4 |
#define | INDEX_BITS 2 |
#define | INDEX_LEN_BITS 1 |
#define | LEN_SHIFT 0 |
#define | INDEX_SHIFT (LEN_SHIFT+LEN_BITS) |
#define | INDEX_LEN_SHIFT (INDEX_SHIFT+INDEX_BITS) |
#define | BITMASK(bits) ((1<<(bits))-1) |
#define | GETVAL(tag, typ) (((tag)>>typ##_SHIFT)&BITMASK(typ##_BITS)) |
#define | MIN_SHORTLEN 6 |
#define | MAX_SHORTLEN (BITMASK(LEN_BITS)+MIN_SHORTLEN-1) |
#define | MIN_LONGLEN (MAX_SHORTLEN+1) |
#define | MAX_LONGLEN (MIN_LONGLEN+255) |
#define | SHORTLEN_DECR (MIN_SHORTLEN-1) |
#define | LONGLEN_DECR MIN_LONGLEN |
#define | MIN_COMPR_WORD_LEN MIN_SHORTLEN |
#define | MAX_COMPR_WORD_LEN MAX_LONGLEN |
#define | MAX_SHORT_INDEX BITMASK(INDEX_BITS+8) |
#define | MAX_LONG_INDEX BITMASK(INDEX_BITS+16) |
#define | LAST_COMPRESSED_BIT 64 |
#define | DUMP_COMPRESSION_TEST 0 |
#define | test_dtree(tree) |
#define | testCounts(tree) |
#define | cmp(i1, i2) (heap2[i1]-heap2[i2]) |
#define | swap(i1, i2) |
#define | cmp(i1, i2) GB_MEMCMP(dict->text+dict->offsets[heap[i1]], dict->text+dict->offsets[heap[i2]], dict->textlen) |
#define | swap(i1, i2) do { int s = heap[i1]; heap[i1] = heap[i2]; heap[i2] = s; } while (0) |
Typedefs | |
typedef unsigned char | unsigned_char |
typedef unsigned char * | u_str |
typedef const unsigned char * | cu_str |
Enumerations | |
enum | DictNodeType { SINGLE_NODE, FULL_NODE } |
Variables | |
static int | gbdByKey_cnt |
Definition at line 69 of file adoptimize.cxx.
Referenced by build_dict_tree(), and readAndWrite().
#define DICT_MEM_WEIGHT 4 |
Definition at line 70 of file adoptimize.cxx.
#define WORD_HELPFUL | ( | wordlen, | |
occurrences | |||
) |
Definition at line 72 of file adoptimize.cxx.
Referenced by cut_useless_words().
#define MIN_WORD_LEN 8 |
Definition at line 82 of file adoptimize.cxx.
Referenced by gb_create_dictionary().
#define MAX_WORD_LEN 50 |
Definition at line 83 of file adoptimize.cxx.
Referenced by gb_create_dictionary().
#define MAX_BROTHERS |
Definition at line 84 of file adoptimize.cxx.
Referenced by add_to_dtree().
#define MAX_DIFFER |
Definition at line 86 of file adoptimize.cxx.
Referenced by build_dict_tree().
#define INCR_DIFFER 1 |
Definition at line 88 of file adoptimize.cxx.
Referenced by build_dict_tree().
#define DICT_STRING_INCR 1024 |
Definition at line 90 of file adoptimize.cxx.
Referenced by gb_create_dictionary().
#define LEN_BITS 4 |
Definition at line 312 of file adoptimize.cxx.
#define INDEX_BITS 2 |
Definition at line 313 of file adoptimize.cxx.
Referenced by gb_compress_by_dictionary().
#define INDEX_LEN_BITS 1 |
Definition at line 314 of file adoptimize.cxx.
#define LEN_SHIFT 0 |
Definition at line 316 of file adoptimize.cxx.
Referenced by gb_compress_by_dictionary().
Definition at line 317 of file adoptimize.cxx.
Referenced by gb_compress_by_dictionary().
#define INDEX_LEN_SHIFT (INDEX_SHIFT+INDEX_BITS) |
Definition at line 318 of file adoptimize.cxx.
Referenced by gb_compress_by_dictionary().
#define BITMASK | ( | bits | ) | ((1<<(bits))-1) |
Definition at line 320 of file adoptimize.cxx.
Referenced by gb_compress_by_dictionary().
#define GETVAL | ( | tag, | |
typ | |||
) | (((tag)>>typ##_SHIFT)&BITMASK(typ##_BITS)) |
Definition at line 321 of file adoptimize.cxx.
Referenced by gb_uncompress_by_dictionary_internal().
#define MIN_SHORTLEN 6 |
Definition at line 323 of file adoptimize.cxx.
Referenced by gb_compress_by_dictionary().
#define MAX_SHORTLEN (BITMASK(LEN_BITS)+MIN_SHORTLEN-1) |
Definition at line 324 of file adoptimize.cxx.
Referenced by gb_compress_by_dictionary().
#define MIN_LONGLEN (MAX_SHORTLEN+1) |
Definition at line 325 of file adoptimize.cxx.
#define MAX_LONGLEN (MIN_LONGLEN+255) |
Definition at line 326 of file adoptimize.cxx.
#define SHORTLEN_DECR (MIN_SHORTLEN-1) |
Definition at line 328 of file adoptimize.cxx.
Referenced by gb_compress_by_dictionary(), and gb_uncompress_by_dictionary_internal().
#define LONGLEN_DECR MIN_LONGLEN |
Definition at line 329 of file adoptimize.cxx.
Referenced by gb_compress_by_dictionary(), and gb_uncompress_by_dictionary_internal().
#define MIN_COMPR_WORD_LEN MIN_SHORTLEN |
Definition at line 331 of file adoptimize.cxx.
Referenced by gb_compress_by_dictionary(), and searchWord().
#define MAX_COMPR_WORD_LEN MAX_LONGLEN |
Definition at line 332 of file adoptimize.cxx.
Referenced by searchWord().
#define MAX_SHORT_INDEX BITMASK(INDEX_BITS+8) |
Definition at line 334 of file adoptimize.cxx.
Referenced by gb_compress_by_dictionary().
#define MAX_LONG_INDEX BITMASK(INDEX_BITS+16) |
Definition at line 335 of file adoptimize.cxx.
Referenced by build_dict_tree(), gb_compress_by_dictionary(), and gb_create_dictionary().
#define LAST_COMPRESSED_BIT 64 |
Definition at line 337 of file adoptimize.cxx.
Referenced by gb_compress_by_dictionary(), and gb_uncompress_by_dictionary_internal().
#define DUMP_COMPRESSION_TEST 0 |
Definition at line 348 of file adoptimize.cxx.
#define test_dtree | ( | tree | ) |
Definition at line 1047 of file adoptimize.cxx.
Referenced by build_dict_tree().
#define testCounts | ( | tree | ) |
Definition at line 1048 of file adoptimize.cxx.
Referenced by build_dict_tree(), and gb_create_dictionary().
#define cmp | ( | i1, | |
i2 | |||
) | (heap2[i1]-heap2[i2]) |
Definition at line 2122 of file adoptimize.cxx.
Referenced by downheap(), and downheap2().
#define swap | ( | i1, | |
i2 | |||
) |
Definition at line 2123 of file adoptimize.cxx.
Referenced by downheap(), and downheap2().
#define cmp | ( | i1, | |
i2 | |||
) | GB_MEMCMP(dict->text+dict->offsets[heap[i1]], dict->text+dict->offsets[heap[i2]], dict->textlen) |
Definition at line 2122 of file adoptimize.cxx.
Definition at line 2123 of file adoptimize.cxx.
typedef unsigned char unsigned_char |
Definition at line 29 of file adoptimize.cxx.
Definition at line 30 of file adoptimize.cxx.
Definition at line 31 of file adoptimize.cxx.
enum DictNodeType |
Enumerator | |
---|---|
SINGLE_NODE | |
FULL_NODE |
Definition at line 49 of file adoptimize.cxx.
Definition at line 94 of file adoptimize.cxx.
References GBDATA::as_entry(), gb_assert, GB_BYTES, GB_FLOATS, GB_INTS, GB_read_bytes_pntr(), GB_read_char_pntr(), GB_read_floats_pntr(), GB_read_ints_pntr(), GB_STRING, NULp, GBDATA::type(), and GBENTRY::uncompressed_size().
Referenced by build_dict_tree(), and readAndWrite().
Definition at line 113 of file adoptimize.cxx.
Referenced by gb_compress_by_dictionary(), gb_create_dictionary(), and searchWord().
|
static |
Definition at line 119 of file adoptimize.cxx.
References GBDATA::as_container(), O_gbdByKey::cnt, GBCONTAINER::d, gb_assert, GB_KEY_QUARK(), GBCONTAINER_ELEM(), O_gbdByKey::gbds, GBDATA::is_container(), and gb_data_list::nheader.
Referenced by g_b_opti_createGbdByKey().
|
static |
Definition at line 138 of file adoptimize.cxx.
References ARB_calloc(), O_gbdByKey::cnt, g_b_opti_scanGbdByKey(), GB_MAIN_TYPE::gb_main(), gbdByKey_cnt, O_gbdByKey::gbds, gb_Key::key, GB_MAIN_TYPE::keycnt, GB_MAIN_TYPE::keys, gb_Key::nref, and NULp.
Referenced by gb_create_dictionaries().
|
static |
Definition at line 171 of file adoptimize.cxx.
References gbdByKey_cnt.
Referenced by gb_create_dictionaries().
Definition at line 178 of file adoptimize.cxx.
References GBDATA::as_entry(), GBENTRY::data(), error(), gb_assert, GB_await_error(), GB_BYTES, GB_child(), GB_FLOATS, GB_INTS, GB_memdup(), GB_nextChild(), GB_OBSOLETE, GB_set_temporary(), GB_STRING, gb_uncompress_bytes(), gb_uncompress_longs_old(), GB_write_bytes(), GB_write_pntr(), GB_write_string(), GBS_global_string(), GBDATA::is_container(), NULp, GBENTRY::size(), GBDATA::type(), and GBENTRY::uncompressed_size().
Referenced by gb_convert_V2_to_V3().
Definition at line 259 of file adoptimize.cxx.
References error(), gb_convert_compression(), GB_create_container(), GB_disable_quicksave(), GB_entry(), GB_FIND, GB_search(), GB_SYSTEM_FOLDER, GB_warning(), and NULp.
Referenced by GB_login().
|
inline |
Definition at line 298 of file adoptimize.cxx.
References gb_assert, and GB_DICTIONARY::offsets.
Referenced by ALPHA_DICT_OFFSET(), and gb_uncompress_by_dictionary_internal().
|
inline |
Definition at line 302 of file adoptimize.cxx.
References gb_assert, INDEX_DICT_OFFSET(), and GB_DICTIONARY::resort.
Referenced by searchWord().
Definition at line 424 of file adoptimize.cxx.
References diff().
Referenced by gb_create_dictionary(), memstr(), and searchWord().
|
static |
Definition at line 435 of file adoptimize.cxx.
References ALPHA_DICT_OFFSET(), gb_assert, GB_MEMCMP(), MAX_COMPR_WORD_LEN, min(), MIN_COMPR_WORD_LEN, GB_DICTIONARY::resort, s, GB_DICTIONARY::text, GB_DICTIONARY::textlen, and GB_DICTIONARY::words.
Referenced by gb_compress_by_dictionary().
|
static |
Definition at line 505 of file adoptimize.cxx.
References buffer, gb_assert, GB_give_other_buffer(), GETVAL, INDEX_DICT_OFFSET(), LAST_COMPRESSED_BIT, LONGLEN_DECR, GB_DICTIONARY::offsets, SHORTLEN_DECR, GB_DICTIONARY::text, and GB_DICTIONARY::words.
Referenced by gb_compress_by_dictionary(), and gb_uncompress_by_dictionary().
char* gb_uncompress_by_dictionary | ( | GBDATA * | gbd, |
GB_CSTR | s_source, | ||
size_t | size, | ||
size_t * | new_size | ||
) |
Definition at line 578 of file adoptimize.cxx.
References error(), GB_export_error(), GB_get_db_path(), gb_get_dictionary(), GB_KEY_QUARK(), GB_MAIN(), gb_uncompress_by_dictionary_internal(), GBS_global_string(), GBDATA::is_a_string(), and NULp.
Referenced by gb_uncompress_data().
char* gb_compress_by_dictionary | ( | GB_DICTIONARY * | dict, |
GB_CSTR | s_source, | ||
size_t | size, | ||
size_t * | msize, | ||
int | last_flag, | ||
int | search_backward, | ||
int | search_forward | ||
) |
Definition at line 591 of file adoptimize.cxx.
References BITMASK, buffer, gb_assert, GB_COMPRESSION_DICTIONARY, GB_COMPRESSION_TAGS_SIZE_MAX, GB_give_other_buffer(), gb_uncompress_by_dictionary_internal(), INDEX_BITS, INDEX_LEN_SHIFT, INDEX_SHIFT, LAST_COMPRESSED_BIT, LEN_SHIFT, length, LONGLEN_DECR, MAX_LONG_INDEX, MAX_SHORT_INDEX, MAX_SHORTLEN, min(), MIN_COMPR_WORD_LEN, MIN_SHORTLEN, NULp, GB_DICTIONARY::offsets, searchWord(), and SHORTLEN_DECR.
Referenced by gb_compress_data().
Definition at line 1052 of file adoptimize.cxx.
References SingleDictTree::brother, SingleDictTree::ch, SingleDictTree::count, GBM_DICT_INDEX, gbm_get_mem(), NULp, DictTree::single, SINGLE_NODE, SingleDictTree::son, tree, and SingleDictTree::typ.
Referenced by add_to_dtree().
Definition at line 1084 of file adoptimize.cxx.
References SingleDictTree::brother, SingleDictTree::ch, FullDictTree::count, SingleDictTree::count, DictTree::exists, DictTree::full, FULL_NODE, gb_assert, GBM_DICT_INDEX, gbm_free_mem(), gbm_get_mem(), NULp, DictTree::single, SINGLE_NODE, FullDictTree::son, SingleDictTree::son, tree, FullDictTree::typ, SingleDictTree::typ, and FullDictTree::usedSons.
Referenced by add_to_dtree(), and build_dict_tree().
|
static |
Definition at line 1120 of file adoptimize.cxx.
References SingleDictTree::brother, DictTree::exists, DictTree::full, FULL_NODE, GBM_DICT_INDEX, gbm_free_mem(), DictTree::single, SINGLE_NODE, FullDictTree::son, SingleDictTree::son, and FullDictTree::typ.
Referenced by gb_create_dictionary(), remove_word_from_dtree(), and removeSubsequentString().
|
static |
Definition at line 1143 of file adoptimize.cxx.
References SingleDictTree::brother, FullDictTree::count, SingleDictTree::count, DictTree::exists, DictTree::full, FULL_NODE, GBM_DICT_INDEX, gbm_free_mem(), NULp, DictTree::single, SINGLE_NODE, FullDictTree::son, SingleDictTree::son, tree, FullDictTree::typ, and FullDictTree::usedSons.
Referenced by build_dict_tree().
Definition at line 1208 of file adoptimize.cxx.
References SingleDictTree::brother, FullDictTree::count, SingleDictTree::count, DictTree::exists, DictTree::full, FULL_NODE, GBM_DICT_INDEX, gbm_free_mem(), NULp, DictTree::single, SINGLE_NODE, FullDictTree::son, SingleDictTree::son, tree, FullDictTree::typ, FullDictTree::usedSons, and WORD_HELPFUL.
Referenced by build_dict_tree().
Definition at line 1293 of file adoptimize.cxx.
References SingleDictTree::brother, SingleDictTree::ch, FullDictTree::count, SingleDictTree::count, DictTree::exists, DictTree::full, FULL_NODE, gb_assert, GBM_DICT_INDEX, gbm_free_mem(), DictTree::single, SINGLE_NODE, FullDictTree::son, SingleDictTree::son, tree, FullDictTree::typ, SingleDictTree::typ, and FullDictTree::usedSons.
Referenced by add_to_dtree().
Definition at line 1352 of file adoptimize.cxx.
References add_dtree_to_dtree(), SingleDictTree::brother, SingleDictTree::ch, FullDictTree::count, SingleDictTree::count, DictTree::exists, DictTree::full, FULL_NODE, gb_assert, MAX_BROTHERS, new_dtree(), DictTree::single, single2full_dtree(), SINGLE_NODE, FullDictTree::son, SingleDictTree::son, tree, FullDictTree::typ, and FullDictTree::usedSons.
Referenced by build_dict_tree().
Definition at line 1413 of file adoptimize.cxx.
References SingleDictTree::brother, FullDictTree::count, SingleDictTree::count, DictTree::exists, DictTree::full, FULL_NODE, gb_assert, DictTree::single, SINGLE_NODE, FullDictTree::son, SingleDictTree::son, and FullDictTree::typ.
Referenced by build_dict_tree(), and gb_create_dictionary().
Definition at line 1448 of file adoptimize.cxx.
References SingleDictTree::brother, FullDictTree::count, DictTree::exists, DictTree::full, FULL_NODE, gb_assert, DictTree::single, SINGLE_NODE, FullDictTree::son, SingleDictTree::son, and FullDictTree::typ.
Referenced by build_dict_tree(), and gb_create_dictionary().
Definition at line 1477 of file adoptimize.cxx.
References SingleDictTree::brother, FullDictTree::count, SingleDictTree::count, DictTree::exists, DictTree::full, FULL_NODE, DictTree::single, SINGLE_NODE, and SingleDictTree::typ.
Referenced by expandBranches(), and removeSubsequentString().
|
static |
Definition at line 1500 of file adoptimize.cxx.
References SingleDictTree::brother, SingleDictTree::ch, FullDictTree::count, SingleDictTree::count, COUNT(), DictTree::exists, free_dtree(), DictTree::full, FULL_NODE, gb_assert, NULp, DictTree::single, SINGLE_NODE, FullDictTree::son, SingleDictTree::son, tree, FullDictTree::typ, and FullDictTree::usedSons.
Referenced by expandBranches().
|
static |
Definition at line 1594 of file adoptimize.cxx.
References GB_MEMCMP(), and NULp.
Referenced by expandBranches().
|
static |
Definition at line 1617 of file adoptimize.cxx.
References SingleDictTree::brother, SingleDictTree::ch, FullDictTree::count, SingleDictTree::count, COUNT(), DictTree::exists, DictTree::full, FULL_NODE, gb_assert, memstr(), NULp, removeSubsequentString(), DictTree::single, SINGLE_NODE, FullDictTree::son, SingleDictTree::son, FullDictTree::typ, and FullDictTree::usedSons.
Referenced by build_dict_tree().
|
static |
Definition at line 1747 of file adoptimize.cxx.
References add_to_dtree(), calcCounts(), O_gbdByKey::cnt, COMPRESSIBLE, FullDictTree::count, count_dtree_leafs(), cut_dtree(), cut_useless_words(), DictTree::exists, expandBranches(), DictTree::full, FULL_NODE, gb_assert, O_gbdByKey::gbds, GBM_DICT_INDEX, gbm_free_mem(), gbm_get_mem(), get_data_n_size(), INCR_DIFFER, GBDATA::is_a_string(), MAX_DIFFER, MAX_LONG_INDEX, NULp, single2full_dtree(), FullDictTree::son, test_dtree, testCounts, tree, FullDictTree::typ, and GBDATA::type().
Referenced by gb_create_dictionary().
|
static |
Definition at line 1884 of file adoptimize.cxx.
References SingleDictTree::brother, SingleDictTree::ch, FullDictTree::count, SingleDictTree::count, DictTree::exists, free_dtree(), DictTree::full, FULL_NODE, gb_assert, GBM_DICT_INDEX, gbm_free_mem(), NULp, DictTree::single, SINGLE_NODE, FullDictTree::son, SingleDictTree::son, tree, FullDictTree::typ, and FullDictTree::usedSons.
Referenced by gb_create_dictionary().
Definition at line 2094 of file adoptimize.cxx.
References cmp, gb_assert, and swap.
Referenced by sort_dict_offsets().
|
static |
Definition at line 2125 of file adoptimize.cxx.
References cmp, gb_assert, and swap.
Referenced by sort_dict_offsets().
|
static |
Definition at line 2151 of file adoptimize.cxx.
References downheap(), downheap2(), GB_DICTIONARY::offsets, GB_DICTIONARY::resort, and GB_DICTIONARY::words.
Referenced by gb_create_dictionary().
|
static |
Definition at line 2199 of file adoptimize.cxx.
References buffer, build_dict_tree(), calcCounts(), count_dtree_leafs(), DICT_STRING_INCR, DictTree::exists, free_dtree(), gb_assert, GB_MEMCMP(), GBM_DICT_INDEX, gbm_free_mem(), gbm_get_mem(), MAX_LONG_INDEX, MAX_WORD_LEN, min(), MIN_WORD_LEN, NULp, offset, GB_DICTIONARY::offsets, remove_word_from_dtree(), GB_DICTIONARY::resort, sort_dict_offsets(), testCounts, GB_DICTIONARY::text, GB_DICTIONARY::textlen, and GB_DICTIONARY::words.
Referenced by gb_create_dictionaries().
|
static |
Definition at line 2322 of file adoptimize.cxx.
References GBM_DICT_INDEX, gbm_free_mem(), NULp, GB_DICTIONARY::offsets, GB_DICTIONARY::resort, GB_DICTIONARY::text, GB_DICTIONARY::textlen, and GB_DICTIONARY::words.
Referenced by gb_create_dictionaries().
|
static |
Definition at line 2331 of file adoptimize.cxx.
References GBDATA::as_entry(), O_gbdByKey::cnt, COMPRESSIBLE, error(), gb_assert, GB_BYTES, GB_FLOATS, GB_INTS, GB_STRING, GB_write_bytes(), GB_write_floats(), GB_write_ints(), GB_write_string(), O_gbdByKey::gbds, GBM_DICT_INDEX, gbm_free_mem(), gbm_get_mem(), get_data_n_size(), GBENTRY::memsize(), NULp, and GBDATA::type().
Referenced by gb_create_dictionaries().
|
static |
Definition at line 2383 of file adoptimize.cxx.
References gb_Key::compression_mask, error(), g_b_opti_createGbdByKey(), g_b_opti_freeGbdByKey(), gb_assert, GB_begin_transaction(), GB_commit_transaction(), GB_COMPRESSION_DICTIONARY, gb_create_dictionary(), GB_end_transaction(), gb_free_dictionary(), gb_get_compression_mask(), gb_get_dictionary(), gb_load_dictionary_data(), gb_main, GB_MAIN_TYPE::gb_main(), gb_save_dictionary_data(), gbdByKey_cnt, O_gbdByKey::gbds, GBM_DICT_INDEX, gbm_free_mem(), gbm_get_mem(), arb_progress::inc_and_check_user_abort(), gb_Key::key, GB_MAIN_TYPE::keys, nint, NULp, GB_DICTIONARY::offsets, quark2key(), readAndWrite(), GB_DICTIONARY::resort, GB_DICTIONARY::text, GB_DICTIONARY::textlen, type, GBDATA::type(), and GB_DICTIONARY::words.
Referenced by GB_optimize().
Definition at line 2556 of file adoptimize.cxx.
References ASSERT_NO_ERROR, error(), gb_create_dictionaries(), GB_disable_quicksave(), GB_get_requested_undo_type(), GB_get_usable_memory(), GB_MAIN(), GB_request_undo_type(), GB_UNDO_KILL, and NULp.
Referenced by NT_database_optimization().
|
static |
Definition at line 33 of file adoptimize.cxx.
Referenced by g_b_opti_createGbdByKey(), g_b_opti_freeGbdByKey(), and gb_create_dictionaries().