ARB
ad_load.cxx
Go to the documentation of this file.
1 // =============================================================== //
2 // //
3 // File : ad_load.cxx //
4 // Purpose : //
5 // //
6 // Institute of Microbiology (Technical University Munich) //
7 // http://www.arb-home.de/ //
8 // //
9 // =============================================================== //
10 
11 #include <cctype>
12 #include <ctime>
13 #include <netinet/in.h>
14 #include <sys/stat.h>
15 
16 #include <arbdbt.h>
17 #include <arb_str.h>
18 #include <arb_file.h>
19 #include <arb_defs.h>
20 #include <arb_progress.h>
21 #include <arb_zfile.h>
22 #include <BufferedFileReader.h>
23 
24 #include "gb_key.h"
25 #include "gb_localdata.h"
26 #include "gb_map.h"
27 #include "gb_load.h"
28 #include "ad_io_inline.h"
29 
30 static int gb_verbose_mode = 0;
32  gb_verbose_mode = 1;
33 }
34 
35 #define FILESIZE_GRANULARITY 1024 // progress will work with DB files up to 2Tb
36 
37 /* ----------------------------------------
38  * ASCII format
39  *
40  * Versions:
41  *
42  * V0 - 20.6.95
43  * V1 Full save
44  * V2 Differential save
45  * V3 May read from stdin. Skipped support for old ASCII format.
46  */
47 
48 static const char *getToken(char **line) {
49  char *token = *line;
50  (*line) += strcspn(*line, " \t");
51 
52  if ((*line)[0]) { // sth follows behind token
53  (*line)[0] = 0; // terminate token
54  (*line)++;
55  (*line) += strspn(*line, " \t"); // goto next token
56  }
57 
58  return token;
59 }
60 
61 static GB_ERROR set_protection_level(GB_MAIN_TYPE *Main, GBDATA *gbd, const char *p) {
62  int secr, secw, secd, lu;
64 
65  secr = secw = secd = 0;
66  lu = 0;
67 
68  if (p && p[0] == ':') {
69  long i;
70 
71  secd = p[1]; A_TO_I(secd);
72  secw = p[2]; A_TO_I(secw);
73  secr = p[3]; A_TO_I(secr);
74 
75  if (secd<0 || secd>7) error = GBS_global_string("Illegal protection level %i", secd);
76  else if (secw<0 || secw>7) error = GBS_global_string("Illegal protection level %i", secw);
77  else if (secr<0 || secr>7) error = GBS_global_string("Illegal protection level %i", secr);
78 
79  lu = atoi(p+4);
80 
81  for (i=Main->last_updated; i<=lu; ++i) {
83  Main->dates[i] = ARB_strdup("unknown date");
84  Main->last_updated = lu+1;
85  }
86  }
87 
88  if (!error) {
89  gbd->flags.security_delete = secd;
90  gbd->flags.security_write = secw;
91  gbd->flags.security_read = secr;
92  gbd->flags2.last_updated = lu;
93  }
94 
95  return error;
96 }
97 
98 static GB_ERROR gb_parse_ascii_rek(LineReader& r, GBCONTAINER *gb_parent, const char *parent_name) {
99  // if parent_name == 0 -> we are parsing at root-level
100  GB_ERROR error = NULp;
101  int done = 0;
102  GB_MAIN_TYPE *Main = GBCONTAINER_MAIN(gb_parent);
103 
104  while (!error && !done) {
105  string LINE;
106  if (!r.getLine(LINE)) break;
107 
108  char *line = (char*)(LINE.c_str()); // HACK: code below will modify the content of the string
109 
110  rest :
111  line += strspn(line, " \t"); // goto first non-whitespace
112 
113  if (line[0]) { // not empty
114  if (line[0] == '/' && line[1] == '*') { // comment
115  char *eoc = strstr(line+2, "*/");
116  if (eoc) {
117  line = eoc+2;
118  goto rest;
119  }
120  error = "expected '*/'";
121  }
122  else { // real content
123  const char *name = getToken(&line);
124 
125  if (name[0] == '%' && name[1] == ')' && !name[2]) { // close container
126  if (!parent_name) {
127  error = "Unexpected '%)' (not allowed outside container)";
128  }
129  else {
130  if (line[0] == '/' && line[1] == '*') { // comment at container-end
131  char *eoc = strstr(line+2, "*/");
132  if (!eoc) {
133  error = "expected '*/'";
134  }
135  else {
136  line += 2;
137  *eoc = 0;
138  if (strcmp(line, parent_name) != 0) {
139  fprintf(stderr,
140  "Warning: comment at end of container ('%s') does not match name of container ('%s').\n"
141  " (might be harmless if you've edited the file and did not care about these comments)\n",
142  line, parent_name);
143  }
144  line = eoc+2;
145  }
146  }
147  done = 1;
148  }
149  }
150  else {
151  const char *protection = NULp;
152  if (line[0] == ':') { // protection level
153  protection = getToken(&line);
154  }
155 
156  bool readAsString = true;
157  if (line[0] == '%') {
158  readAsString = false;
159 
160  const char *type = getToken(&line);
161 
162  if (type[1] == 0 || type[2] != 0) {
163  error = GBS_global_string("Syntax error in type '%s' (expected %% and 1 more character)", type);
164  }
165  else {
166  if (type[1] == '%' || type[1] == '$') { // container
167  if (line[0] == '(' && line[1] == '%') {
168  char *cont_name = ARB_strdup(name);
169  GBCONTAINER *gbc = gb_make_container(gb_parent, cont_name, -1, 0);
170 
171  char *protection_copy = nulldup(protection);
172  bool marked = type[1] == '$';
173  error = gb_parse_ascii_rek(r, gbc, cont_name);
174  // CAUTION: most buffer variables are invalidated NOW!!!
175 
176  if (!error) error = set_protection_level(Main, gbc, protection_copy);
177  if (marked) GB_write_flag(gbc, 1);
178 
179  free(protection_copy);
180  free(cont_name);
181  }
182  else {
183  error = "Expected '(%' after '%%'";
184  }
185  }
186  else {
187  GB_TYPES gb_type = GB_NONE;
188 
189  switch (type[1]) {
190  case 'i': gb_type = GB_INT; break;
191  case 'l': gb_type = GB_OBSOLETE; break;
192  case 'y': gb_type = GB_BYTE; break;
193  case 'f': gb_type = GB_FLOAT; break;
194  case 'I': gb_type = GB_BITS; break;
195 
196  case 'Y': gb_type = GB_BYTES; break;
197  case 'N': gb_type = GB_INTS; break;
198  case 'F': gb_type = GB_FLOATS; break;
199  case 's': gb_type = GB_STRING; readAsString = true; break; // special case (string starts with '%')
200 
201  default:
202  error = GBS_global_string("Unknown type '%s'", type);
203  break;
204  }
205 
206  if (!error && !readAsString) {
207  gb_assert(gb_type != GB_NONE);
208 
209  if (gb_type == GB_OBSOLETE) {
210  fputs("Skipped entry (obsolete type GB_LINK)\n", stderr);
211  }
212  else {
213  GBENTRY *gb_new = gb_make_entry(gb_parent, name, -1, 0, gb_type);
214  if (!gb_new) error = GB_await_error();
215  else {
216  switch (type[1]) {
217  case 'i': error = GB_write_int(gb_new, atoi(line)); break;
218  case 'y': error = GB_write_byte(gb_new, atoi(line)); break;
219  case 'f': error = GB_write_float(gb_new, GB_atof(line)); break;
220  case 'I': {
221  int len = strlen(line);
222  gb_assert(line[0] == '\"');
223  gb_assert(line[len-1] == '\"');
224  error = GB_write_bits(gb_new, line+1, len-2, "-"); break;
225  }
226 
227  case 'Y': if (gb_ascii_2_bin(line, gb_new)) error = "syntax error in byte-array"; break;
228  case 'N': if (gb_ascii_2_bin(line, gb_new)) error = "syntax error in int-array"; break;
229  case 'F': if (gb_ascii_2_bin(line, gb_new)) error = "syntax error in float-array"; break;
230 
231  default: gb_assert(0); // forgot a case ?
232  }
233 
234  if (!error) error = set_protection_level(Main, gb_new, protection);
235  }
236  }
237  }
238  }
239  }
240  }
241 
242  if (readAsString) {
243  if (line[0] != '\"') {
244  error = GBS_global_string("Unexpected content '%s'", line);
245  }
246  else { // string entry
247  char *string_start = line+1;
248  char *end = GBS_fconvert_string(string_start);
249 
250  if (!end) error = "Cannot convert string (contains zero char)";
251  else {
252  GBDATA *gb_string = gb_make_entry(gb_parent, name, -1, 0, GB_STRING);
253  if (!gb_string) error = GB_await_error();
254  else {
255  error = GB_write_string(gb_string, string_start);
256  if (!error) error = set_protection_level(Main, gb_string, protection);
257  }
258  }
259  }
260  }
261  }
262  }
263  }
264  }
265 
266  return error;
267 }
268 
270  GB_ERROR error = gb_parse_ascii_rek(r, gb_parent, NULp);
271  if (error) {
272  error = GBS_global_string("%s in line %zu", error, r.getLineNumber()); // @@@ consider using r.lineError
273  }
274  return error;
275 }
276 
278  BufferedPipeReader(const string& pipename, FILE *in)
279  : BufferedFileReader(pipename, in)
280  {}
282  gb_assert(!get_fp()); // you HAVETO call close() or dont_close() manually
283  // (to check the error and to close the pipe)
284  }
285 
287  FILE*& pipe = get_fp();
288  GB_ERROR error = ARB_zfclose(pipe);
289  pipe = NULp;
290  return error;
291  }
292 
293  void dont_close() {
294  FILE*& f = get_fp();
295  f = NULp;
296  }
297 };
298 
299 static GB_ERROR gb_read_ascii_beyond_header(FILE *in, const char *path, GBCONTAINER *gbc) {
300  // This loads an ASCII database (header line has already been read!)
301  GB_ERROR error = NULp;
302  BufferedPipeReader r(path, in);
303 
304  GB_search(gbc, GB_SYSTEM_FOLDER, GB_CREATE_CONTAINER); // Switch to Version 3
305  error = gb_parse_ascii(r, gbc);
306  r.dont_close(); // done by caller
307 
308  return error;
309 }
310 
311 // --------------------------
312 // Read binary files
313 
314 static long gb_recover_corrupt_file(GBCONTAINER *gbc, FILE *in, GB_ERROR recovery_reason, bool loading_quick_save) {
315  // search pattern dx xx xx xx string 0
316  // returns 0 if recovery was able to resync
317  static FILE *old_in = NULp;
318  static unsigned char *file = NULp;
319  static long size = 0;
320 
321  if (!GBCONTAINER_MAIN(gbc)->allow_corrupt_file_recovery) {
322  if (!recovery_reason) { recovery_reason = GB_await_error(); }
323  char *reason = ARB_strdup(recovery_reason);
324  const char *located_reason = GBS_global_string("%s (inside '%s')", reason, GB_get_db_path(gbc));
325 
326  if (loading_quick_save) {
327  GB_export_error(located_reason);
328  }
329  else {
330  GB_export_errorf("%s\n"
331  "(parts of your database might be recoverable using 'arb_repair yourDB.arb newName.arb')\n",
332  located_reason);
333  }
334  free(reason);
335  return -1;
336  }
337 
338  if (GB_is_fifo(in)) {
339  GB_export_error("Unable to recover from corrupt file (Reason: cannot recover from stream)\n"
340  "Note: if the file is a compressed arb-file, uncompress it manually and retry.");
341  return -1;
342  }
343 
344  long pos = ftell(in);
345  if (old_in != in) {
346  file = (unsigned char *)GB_map_FILE(in, 0);
347  old_in = in;
348  size = GB_size_of_FILE(in);
349  }
350  for (; pos<size-10; pos ++) {
351  if ((file[pos] & 0xf0) == (GB_STRING_SHRT<<4)) {
352  long s;
353  int c;
354  for (s = pos + 4; s<size && file[s]; s++) {
355  c = file[s];
356  if (! (isalnum(c) || isspace(c) || strchr("._;:,", c))) break;
357  }
358  if (s< size && s > pos+11 && !file[s]) { // we found something
360  return fseek(in, pos, 0);
361  }
362  }
363  }
364  return -1; // no short string found
365 }
366 
367 // ----------------------------------------
368 // #define DEBUG_READ
369 #if defined(DEBUG_READ)
370 
371 static void DEBUG_DUMP_INDENTED(long deep, const char *s) {
372  printf("%*s%s\n", (int)deep, "", s);
373 }
374 
375 #else
376 #define DEBUG_DUMP_INDENTED(d, s)
377 #endif // DEBUG_READ
378 // ----------------------------------------
379 
380 static long gb_read_bin_rek_V2(FILE *in, GBCONTAINER *gbc_dest, long nitems, long version,
381  long reversed, long deep, arb_progress& progress) {
382  GB_MAIN_TYPE *Main = GB_MAIN(gbc_dest);
383 
384  DEBUG_DUMP_INDENTED(deep, GBS_global_string("Reading container with %li items", nitems));
385 
386  progress.inc_to(ftell(in)/FILESIZE_GRANULARITY);
387  if (progress.aborted()) {
388  GB_export_error(progress.error_if_aborted());
389  return -1;
390  }
391 
392  gb_create_header_array(gbc_dest, (int)nitems);
393  gb_header_list *header = GB_DATA_LIST_HEADER(gbc_dest->d);
394  if (deep == 0 && GBCONTAINER_MAIN(gbc_dest)->allow_corrupt_file_recovery) {
395  GB_warning("Now reading to end of file (trying to recover data from broken database)");
396  nitems = 10000000; // read forever at highest level
397  }
398 
399  bool is_quicksave = version != 1;
400 
401  for (long item = 0; item<nitems; item++) {
402  long type = getc(in);
403  DEBUG_DUMP_INDENTED(deep, GBS_global_string("Item #%li/%li type=%02lx (filepos=%08lx)", item+1, nitems, type, ftell(in)-1));
404  if (type == EOF) {
405  if (GBCONTAINER_MAIN(gbc_dest)->allow_corrupt_file_recovery) {
406  GB_export_error("End of file reached (no error)");
407  }
408  else {
409  GB_export_error("Unexpected end of file seen");
410  }
411  return -1;
412  }
413  if (!type) {
414  int func;
415  if (version == 1) { // master file
416  if (gb_recover_corrupt_file(gbc_dest, in, "Unknown DB type 0 (DB version=1)", is_quicksave)) return -1;
417  continue;
418  }
419  func = getc(in);
420  switch (func) {
421  case 1: { // delete entry
422  int index = (int)gb_get_number(in);
423  if (index >= gbc_dest->d.nheader) {
424  gb_create_header_array(gbc_dest, index+1);
425  header = GB_DATA_LIST_HEADER(gbc_dest->d);
426  }
427 
428  GBDATA *gb2 = GB_HEADER_LIST_GBD(header[index]);
429  if (gb2) {
430  gb_delete_entry(gb2);
431  }
432  else {
433  header[index].flags.set_change(GB_DELETED);
434  }
435 
436  break;
437  }
438  default:
439  if (gb_recover_corrupt_file(gbc_dest, in, GBS_global_string("Unknown func=%i", func), is_quicksave)) return -1;
440  continue;
441  }
442  continue;
443  }
444 
445  long security = getc(in);
446  long type2 = (type>>4)&0xf;
447  GBQUARK key = (GBQUARK)gb_get_number(in);
448 
449  if (key >= Main->keycnt || !quark2key(Main, key)) {
450  const char *reason = GBS_global_string("database entry with unknown field quark %i", key);
451  if (gb_recover_corrupt_file(gbc_dest, in, reason, is_quicksave)) return -1;
452  continue;
453  }
454 
455  DEBUG_DUMP_INDENTED(deep, GBS_global_string("key='%s' type2=%li", quark2key(Main, key), type2));
456 
457  GBENTRY *gbe = NULp;
458  GBCONTAINER *gbc = NULp;
459  GBDATA *gbd = NULp;
460 
461  {
462  int index;
463  if (version == 2) {
464  index = (int)gb_get_number(in);
465  if (index >= gbc_dest->d.nheader) {
466  gb_create_header_array(gbc_dest, index+1);
467  header = GB_DATA_LIST_HEADER(gbc_dest->d);
468  }
469 
470  if (index >= 0 && (gbd = GB_HEADER_LIST_GBD(header[index]))) {
471  if ((gbd->type() == GB_DB) != (type2 == GB_DB)) {
472  GB_internal_error("Type changed, you may loose data");
473  gb_delete_entry(gbd);
474  SET_GB_HEADER_LIST_GBD(header[index], NULp);
475  }
476  else {
477  if (type2 == GB_DB) {
478  gbc = gbd->as_container();
479  }
480  else {
481  gbe = gbd->as_entry();
482  gbe->free_data();
483  }
484  }
485  }
486  }
487  else index = -1;
488 
489  if (!gbd) {
490  if (type2 == (long)GB_DB) {
491  gbd = gbc = gb_make_container(gbc_dest, NULp, index, key);
492  }
493  else {
494  gbd = gbe = gb_make_entry(gbc_dest, NULp, index, key, (GB_TYPES)type2);
495  gbe->index_check_out();
496  }
497  }
498  }
499 
500  gb_assert(implicated(gbe, gbe == gbd));
501  gb_assert(implicated(gbc, gbc == gbd));
502  gb_assert(implicated(gbd, contradicted(gbe, gbc)));
503 
504  if (version == 2) {
505  gbd->create_extended();
506  gbd->touch_creation_and_update(Main->clock);
507  header[gbd->index].flags.ever_changed = 1;
508  }
509  else {
510  Main->keys[key].nref_last_saved++;
511  }
512 
513  gbd->flags.security_delete = type >> 1;
514  gbd->flags.security_write = ((type&1) << 2) + (security >> 6);
515  gbd->flags.security_read = security >> 3;
516  gbd->flags.compressed_data = security >> 2;
517  header[gbd->index].flags.flags = (int)((security >> 1) & 1);
518  gbd->flags.unused = security >> 0;
519  gbd->flags2.last_updated = getc(in);
520 
521  switch (type2) {
522  case GB_INT: {
524  if (!fread((char*)&buffer, sizeof(GB_UINT4), 1, in)) {
525  GB_export_error("File too short, seems truncated");
526  return -1;
527  }
528  gbe->info.i = ntohl(buffer);
529  break;
530  }
531 
532  case GB_FLOAT:
533  if (!fread((char*)&gbe->info.i, sizeof(float), 1, in)) {
534  GB_export_error("File too short, seems truncated");
535  return -1;
536  }
537  break;
538  case GB_STRING_SHRT: {
539  long i = GB_give_buffer_size();
540  char *buff = GB_give_buffer(GBTUM_SHORT_STRING_SIZE+2);
541  char *p = buff;
542 
543  long size = 0;
544  while (1) {
545  for (; size<i; size++) {
546  if (!(*(p++) = getc(in))) goto shrtstring_fully_loaded;
547  }
548  i = i*3/2;
549  buff = GB_increase_buffer(i);
550  p = buff + size;
551  }
552  shrtstring_fully_loaded :
553  gbe->insert_data(buff, size, size+1);
554  break;
555  }
556  case GB_OBSOLETE:
557  case GB_STRING:
558  case GB_BITS:
559  case GB_BYTES:
560  case GB_INTS:
561  case GB_FLOATS: {
562  long size = gb_get_number(in);
563  long memsize = gb_get_number(in);
564 
565  DEBUG_DUMP_INDENTED(deep, GBS_global_string("size=%li memsize=%li", size, memsize));
566 
567  GBENTRY_memory storage(gbe, size, memsize);
568 
569  long i = fread(storage, 1, (size_t)memsize, in);
570  if (i!=memsize) {
571  gb_read_bin_error(in, gbe, "Unexpected EOF found");
572  return -1;
573  }
574  break;
575  }
576  case GB_DB: {
577  long size = gb_get_number(in);
578  // gbc->d.size is automatically incremented
579  if (gb_read_bin_rek_V2(in, gbc, size, version, reversed, deep+1, progress)) {
580  if (!GBCONTAINER_MAIN(gbc_dest)->allow_corrupt_file_recovery) {
581  return -1;
582  }
583  }
584  break;
585  }
586  case GB_BYTE:
587  gbe->info.i = getc(in);
588  break;
589  default:
590  gb_read_bin_error(in, gbd, "Unknown type");
591  if (gb_recover_corrupt_file(gbc_dest, in, NULp, is_quicksave)) {
593  ? 0 // loading stopped
594  : -1;
595  }
596 
597  continue;
598  }
599  }
600  return 0;
601 }
602 
604  GBDATA *gb2;
605  GBDATA *gb_result = NULp;
606  for (gb2 = GB_child(gbd); gb2; gb2 = GB_nextChild(gb2)) {
607  int type = GB_read_type(gb2);
608  if (type != GB_DB) continue;
609  if (!strcmp(GB_SYSTEM_FOLDER, GB_read_key_pntr(gb2))) {
610  gb_result = gb2;
611  break;
612  }
613  }
614  return gb_result;
615 }
616 
617 
619  /* Search a system folder within the database tree
620  * and copy it to main level
621  */
622  GBDATA *gb_oldsystem;
623  GB_ERROR error;
624  GBDATA *gb_system = GB_entry(gb_main, GB_SYSTEM_FOLDER);
625  if (gb_system) return;
626 
627  GB_warning("Searching system information");
628  gb_oldsystem = gb_search_system_folder_rek(gb_main);
629  if (!gb_oldsystem) {
630  GB_warning("!!!!! not found (bad)");
631  return;
632  }
633  gb_system = GB_search(gb_main, GB_SYSTEM_FOLDER, GB_CREATE_CONTAINER);
634  error = GB_copy_dropProtectMarksAndTempstate(gb_system, gb_oldsystem);
635  if (!error) error = GB_delete(gb_oldsystem);
636  if (error) GB_warning(error);
637  GB_warning("***** found (good)");
638 }
639 
640 inline bool read_keyword(const char *expected_keyword, FILE *in, GBCONTAINER *gbc) {
641  gb_assert(strlen(expected_keyword) == 4); // mandatory
642 
643  long val = gb_read_in_uint32(in, 0);
644  bool as_expected = strncmp((char*)&val, expected_keyword, 4) == 0;
645 
646  if (!as_expected) {
647  gb_read_bin_error(in, gbc, GBS_global_string("keyword '%s' not found", expected_keyword));
648  }
649  return as_expected;
650 }
651 
652 static long gb_read_bin(FILE *in, GBCONTAINER *gbc, bool allowed_to_load_diff, arb_progress& progress) {
653  int c = 1;
654  long i;
655  long error;
656  long j, k;
657  long version;
658  long nodecnt;
659  long first_free_key;
660  char *buffer, *p;
661 
662  GB_MAIN_TYPE *Main = GBCONTAINER_MAIN(gbc);
663 
664  while (c && c != EOF) {
665  c = getc(in);
666  }
667  if (c==EOF) {
668  gb_read_bin_error(in, gbc, "First zero not found");
669  return 1;
670  }
671 
672  if (!read_keyword("vers", in, gbc)) return 1;
673 
674  // detect byte order in ARB file
675  i = gb_read_in_uint32(in, 0);
676  bool reversed;
677  switch (i) {
678  case 0x01020304: reversed = false; break;
679  case 0x04030201: reversed = true; break;
680  default:
681  gb_read_bin_error(in, gbc, "keyword '^A^B^C^D' not found");
682  return 1;
683  }
684 
685  version = gb_read_in_uint32(in, reversed);
686  if (version == 0) {
687  gb_read_bin_error(in, gbc, "ARB Database version 0 no longer supported (rev [9647])");
688  return 1;
689  }
690  if (version>2) {
691  gb_read_bin_error(in, gbc, "ARB Database version > '2'");
692  return 1;
693  }
694 
695  if (version == 2 && !allowed_to_load_diff) {
696  GB_export_error("This is not a primary arb file, please select the master"
697  " file xxx.arb");
698  return 1;
699  }
700 
701  if (!read_keyword("keys", in, gbc)) return 1;
702 
704 
705  first_free_key = 0;
706  Main->free_all_keys();
707 
708  buffer = GB_give_buffer(256);
709  while (1) { // read keys
710  long nrefs = 0;
711  if (version) {
712  nrefs = gb_get_number(in);
713  }
714  p = buffer;
715  for (k=0; ; k++) {
716  c = getc(in);
717  if (!c) break;
718  if (c==EOF) {
719  gb_read_bin_error(in, gbc, "unexpected EOF while reading keys");
720  return 1;
721  }
722  *(p++) = c;
723  }
724  *p = 0;
725 
726  if (k > GB_KEY_LEN_MAX) {
727  printf("Warning: Key '%s' exceeds maximum keylength (%i)\n"
728  " Please do NOT create such long keys!\n",
729  buffer, GB_KEY_LEN_MAX);
730  }
731  if (p == buffer) break;
732 
733  if (*buffer == 1) { // empty key
734  long index = gb_create_key(Main, NULp, false);
735 
736  Main->keys[index].key = NULp;
737  Main->keys[index].nref = 0;
738  Main->keys[index].next_free_key = first_free_key;
739 
740  first_free_key = index;
741  }
742  else {
743  long index = gb_create_key(Main, buffer, false);
744 
745  Main->keys[index].nref = nrefs;
746  }
747  }
748 
749  Main->first_free_key = first_free_key;
750 
751  if (!read_keyword("time", in, gbc)) return 1;
752 
753  for (j=0; j<(ALLOWED_DATES-1); j++) { // read times
754  p = buffer;
755  for (k=0; k<256; k++) {
756  c = getc(in);
757  if (!c) break;
758  if (c==EOF) {
759  gb_read_bin_error(in, gbc, "unexpected EOF while reading times");
760  return 1;
761  }
762  *(p++) = c;
763  }
764  *p = 0;
765  if (p == buffer) break;
766  freedup(Main->dates[j], buffer);
767  }
768  if (j>=(ALLOWED_DATES-1)) {
769  gb_read_bin_error(in, gbc, "too many date entries");
770  return 1;
771  }
772  Main->last_updated = (unsigned int)j;
773 
774  if (!read_keyword("data", in, gbc)) return 1;
775 
776  nodecnt = gb_read_in_uint32(in, reversed);
777  GB_give_buffer(256);
778 
779  if (version==1) { // teste auf map file falls version == 1
780  long mode = GB_mode_of_link(Main->path); // old master
781  GB_CSTR map_path;
782  unsigned long time_of_db;
783 
784  if (S_ISLNK(mode)) {
785  char *path2 = GB_follow_unix_link(Main->path);
786  map_path = gb_mapfile_name(path2);
787  time_of_db = GB_time_of_file(path2);
788  free(path2);
789  }
790  else {
791  map_path = gb_mapfile_name(Main->path);
792  time_of_db = GB_time_of_file(Main->path);
793  }
794 
795  bool mapped = false;
796  GB_ERROR map_fail_reason = NULp;
797  gb_map_header mheader;
798 
799  switch (gb_is_valid_mapfile(map_path, &mheader, 0)) {
800  case -1: map_fail_reason = GBS_global_string("no FastLoad File '%s' found", map_path); break;
801  case 0: map_fail_reason = GB_await_error(); break;
802  case 1: {
803  unsigned long time_of_map = GB_time_of_file(map_path);
804 
805  if (time_of_map != time_of_db) { // timestamp of mapfile differs
806  unsigned long diff = time_of_map>time_of_db ? time_of_map-time_of_db : time_of_db-time_of_map;
807  fprintf(stderr, "Warning: modification times of DB and fastload file differ (DB=%lu fastload=%lu diff=%lu)\n",
808  time_of_db, time_of_map, diff);
809 
810  if (diff>5) {
811  map_fail_reason = "modification times of DB and fastload file differ (too much)";
812  }
813  else {
814  fprintf(stderr, "(accepting modification time difference of %lu seconds)\n", diff);
815  }
816  }
817 
818  if (!map_fail_reason) {
819  if (!gb_main_array[mheader.main_idx]) {
820  GBCONTAINER *new_gbc = (GBCONTAINER*)gb_map_mapfile(map_path);
821 
822  if (new_gbc) {
823  GBCONTAINER *father = GB_FATHER(gbc);
824  GB_MAIN_IDX new_idx = mheader.main_idx;
825  GB_MAIN_IDX old_idx = father->main_idx;
826 
827  long gbc_index = gbc->index;
828 
830 
831  gb_assert(new_gbc->main_idx == new_idx);
832  gb_assert((new_idx % GB_MAIN_ARRAY_SIZE) == new_idx);
833 
834  gb_main_array[new_idx] = Main;
835 
836  gbm_free_mem(Main->root_container, sizeof(GBCONTAINER), quark2gbmindex(Main, 0));
837 
838  Main->root_container = new_gbc;
839  father->main_idx = new_idx;
840 
841  SET_GBCONTAINER_ELEM(father, gbc_index, NULp); // unlink old main-entry
842 
843  gbc = new_gbc;
844  SET_GB_FATHER(gbc, father);
845 
846  SET_GBCONTAINER_ELEM(father, gbc->index, (GBDATA*)gbc); // link new main-entry
847 
848  gb_main_array[old_idx] = NULp;
849 
851  mapped = true;
852  }
853  }
854  else {
855  map_fail_reason = GBS_global_string("FastLoad-File index conflict (%s, %i)", map_path, mheader.main_idx);
856  }
857  }
858 
859  break;
860  }
861  default: gb_assert(0); break;
862  }
863 
864  gb_assert(mapped || map_fail_reason);
865 
866  if (mapped) {
867  Main->mapped = true;
868  return 0; // succeded loading mapfile -> no need to load normal DB file
869  }
870  GB_informationf("ARB: %s => loading entire DB", map_fail_reason);
871  }
872 
873  // load binary DB file
874 
875  switch (version) {
876  case 2: // quick save file
877  for (i=1; i < Main->keycnt; i++) {
878  if (Main->keys[i].key) {
879  Main->keys[i].nref_last_saved = Main->keys[i].nref;
880  }
881  }
882 
883  if (Main->clock<=0) Main->clock++;
884  // fall-through
885  case 1: // master arb file
886  error = gb_read_bin_rek_V2(in, gbc, nodecnt, version, reversed, 0, progress);
887  break;
888  default:
889  GB_internal_errorf("Sorry: This ARB Version does not support database format V%li", version);
890  error = 1;
891  }
892 
895  }
896 
897  switch (version) {
898  case 2:
899  case 1:
900  for (i=1; i < Main->keycnt; i++) {
901  if (Main->keys[i].key) {
902  Main->keys[i].nref = Main->keys[i].nref_last_saved;
903  }
904  }
905  break;
906  default:
907  break;
908  }
909 
910  return error;
911 }
912 
913 // ----------------------
914 // OPEN DATABASE
915 
917 
918 void GB_set_next_main_idx(long idx) {
919  gb_next_main_idx_for_mapfile = idx;
920 }
921 
923  static int initialized = 0;
924  GB_MAIN_IDX idx;
925 
926  if (!initialized) {
927  for (idx=0; idx<GB_MAIN_ARRAY_SIZE; idx++) gb_main_array[idx] = NULp;
928  initialized = 1;
929  }
930  if (gb_next_main_idx_for_mapfile<=0) {
931  while (1) { // search for unused array index
933  if (!gb_main_array[idx]) break;
934  }
935  }
936  else {
937  idx = (short)gb_next_main_idx_for_mapfile;
938  gb_next_main_idx_for_mapfile = 0;
939  }
940 
941  gb_assert((idx%GB_MAIN_ARRAY_SIZE) == idx);
942 
943  gb_main_array[idx] = Main;
944  return idx;
945 }
946 
948  if (dummy_father) {
950  gb_assert(gb_main_array[idx] == this);
951  gb_main_array[idx] = NULp;
952  }
953 }
954 
955 GB_ERROR GB_MAIN_TYPE::login_remote(const char *db_path, const char *opent) {
956  GB_ERROR error = NULp;
957 
958  i_am_server = false;
959  c_link = gbcmc_open(db_path);
960  if (!c_link) {
961  error = GBS_global_string("There is no ARBDB server '%s', please start one or add a filename", db_path);
962  }
963  else {
965 
967  error = initial_client_transaction();
968 
969  if (!error) {
971 
972  if (strchr(opent, 't')) error = gb_unfold(root_container, 0, -2); // tiny
973  else if (strchr(opent, 'm')) error = gb_unfold(root_container, 1, -2); // medium (no sequence)
974  else if (strchr(opent, 'b')) error = gb_unfold(root_container, 2, -2); // big (no tree)
975  else if (strchr(opent, 'h')) error = gb_unfold(root_container, -1, -2); // huge (all)
976  else error = gb_unfold(root_container, 0, -2); // tiny
977  }
978  }
979  return error;
980 }
981 
982 inline bool is_binary_db_id(int id) {
983  return (id == 0x56430176)
984  || (id == GBTUM_MAGIC_NUMBER)
985  || (id == GBTUM_MAGIC_REVERSED);
986 }
987 inline bool has_ascii_db_id(uint32_t bin_id, FILE *in) {
988  // assumes the first 4 bytes of the input have been read (into 'bin_id').
989  // will read enough bytes to check for valid ascii-header.
990  // returns true if file has an arb-ascii-database-header.
991  const int ASC_HEADER_SIZE = 15;
992  char buffer[ASC_HEADER_SIZE+1];
993 
994  size_t read_bytes = fread(buffer+4, 1, ASC_HEADER_SIZE-4, in); // 4 bytes have already been read as binary ID
995  if (read_bytes == (ASC_HEADER_SIZE-4)) {
996  buffer[ASC_HEADER_SIZE] = 0;
997 
998  const char *ascii_header = "/*ARBDB ASCII*/";
999  uint32_t *ui_buffer = (uint32_t*)buffer;
1000 
1001  *ui_buffer = bin_id; // insert these 4 bytes
1002  if (strcmp(buffer, ascii_header) == 0) return true;
1003 
1004  *ui_buffer = reverse_byteorder(bin_id); // insert them reversed
1005  if (strcmp(buffer, ascii_header) == 0) return true;
1006  }
1007  return false;
1008 }
1009 
1011  GB_ERROR error = NULp;
1012  GBDATA *gb_tmp = GB_search(gb_main, "tmp", GB_CREATE_CONTAINER);
1013  if (gb_tmp) {
1014  error = GB_set_temporary(gb_tmp);
1015  }
1016  else {
1017  error = GB_await_error();
1018  }
1019  return error;
1020 }
1021 
1022 static GBDATA *GB_login(const char *cpath, const char *opent, const char *user) {
1049  GBCONTAINER *gbc;
1050  GB_MAIN_TYPE *Main;
1051  gb_open_types opentype;
1052  GB_CSTR quickFile = NULp;
1053  int ignoreMissingMaster = 0;
1054  int loadedQuickIndex = -1;
1055  GB_ERROR error = NULp;
1056  char *path = ARB_strdup(null2empty(cpath));
1057  bool dbCreated = false;
1058 
1059  gb_assert(!strchr(opent, 'd')); // mode 'd' is deprecated. You have to use 'D' and store your defaults inside ARBHOME/lib/arb_default
1060 
1061  opentype = (!opent || strchr(opent, 'w')) ? gb_open_all : gb_open_read_only_all;
1062 
1063  if (strchr(path, ':')) {
1064  ; // remote access
1065  }
1066  else if (GBS_string_matches(path, "*.quick?", GB_MIND_CASE)) {
1067  char *ext = gb_findExtension(path);
1068  gb_assert(ext);
1069  if (isdigit(ext[6])) {
1070  loadedQuickIndex = atoi(ext+6);
1071  strcpy(ext, ".arb");
1072  quickFile = gb_oldQuicksaveName(path, loadedQuickIndex);
1073  if (strchr(opent, 'R')) ignoreMissingMaster = 1;
1074  }
1075  }
1076  else if (GBS_string_matches(path, "*.a??", GB_MIND_CASE)) {
1077 
1078  char *extension = gb_findExtension(path);
1079 
1080  if (isdigit(extension[2]) && isdigit(extension[3])) {
1081  loadedQuickIndex = atoi(extension+2);
1082  strcpy(extension, ".arb");
1083  quickFile = gb_quicksaveName(path, loadedQuickIndex);
1084  if (strchr(opent, 'R')) ignoreMissingMaster = 1;
1085  }
1086  else {
1087  char *base = ARB_strdup(path);
1088  char *ext = gb_findExtension(base);
1089  {
1090  gb_scandir dir;
1091  ext[0] = 0;
1092  gb_scan_directory(base, &dir);
1093 
1094  loadedQuickIndex = dir.highest_quick_index;
1095 
1096  if (dir.highest_quick_index!=dir.newest_quick_index) {
1097  GB_warning("The QuickSave-File with the highest index-number\n"
1098  "is not the NEWEST of your QuickSave-Files.\n"
1099  "If you didn't restore old QuickSave-File from a backup\n"
1100  "please inform your system-administrator - \n"
1101  "this may be a serious bug and you may loose your data.");
1102  }
1103 
1104  switch (dir.type) {
1105  case GB_SCAN_NO_QUICK: break;
1106  case GB_SCAN_NEW_QUICK: quickFile = gb_quicksaveName(path, dir.highest_quick_index); break;
1107  case GB_SCAN_OLD_QUICK: quickFile = gb_oldQuicksaveName(path, dir.newest_quick_index); break;
1108  }
1109  }
1110 
1111  free(base);
1112  }
1113  }
1114 
1115  if (gb_verbose_mode) {
1116  GB_informationf("ARB: Loading '%s'%s%s", path, quickFile ? " + Changes-File " : "", null2empty(quickFile));
1117  }
1118 
1119  error = GB_install_pid(1);
1120  if (error) {
1121  GB_export_error(error);
1122  return NULp;
1123  }
1124 
1125  GB_init_gb();
1126 
1127  Main = new GB_MAIN_TYPE(path);
1128  Main->mark_as_server();
1129 
1130  if (strchr(opent, 'R')) Main->allow_corrupt_file_recovery = 1;
1131 
1132  ASSERT_RESULT(long, 0, gb_create_key(Main, "main", false));
1133 
1134  Main->dummy_father = gb_make_container(NULp, NULp, -1, 0); // create "main"
1135  Main->dummy_father->main_idx = gb_make_main_idx(Main);
1137  gbc = gb_make_container(Main->dummy_father, NULp, -1, 0); // create "main"
1138 
1139  Main->root_container = gbc;
1140 
1141  error = gbcm_login(gbc, user);
1142  if (!error) {
1143  Main->opentype = opentype;
1144  Main->security_level = 7;
1145 
1146  if (path && (strchr(opent, 'r'))) {
1147  if (strchr(path, ':')) {
1148  error = Main->login_remote(path, opent);
1149  }
1150  else {
1151  int read_from_stdin = strcmp(path, "-") == 0;
1152 
1153  GB_ULONG time_of_main_file = 0; long i;
1154 
1155  Main->mark_as_server();
1156  GB_begin_transaction(gbc);
1157  Main->clock = 0; // start clock
1158 
1159  FILE *input = read_from_stdin ? stdin : ARB_zfopen(path, "rb", ZFILE_AUTODETECT, error, false);
1160  if (!input && ignoreMissingMaster) {
1161  error = NULp; // @@@ maybe need to inspect error message here
1162  goto load_quick_save_file_only;
1163  }
1164 
1165  if (!input) {
1166  if (strchr(opent, 'c')) {
1167  error = NULp; // from ARB_zfopen
1168  GB_disable_quicksave(gbc, "Database Created");
1169 
1170  if (strchr(opent, 'D')) { // use default settings
1171  GB_clear_error(); // with default-files gb_scan_directory (used above) has created an error, cause the path was a fake path
1172 
1173  gb_assert(!ARB_strBeginsWith(path, ".arb_prop/")); // do no longer pass path-prefix [deprecated!]
1174  char *found_path = GB_property_file(false, path);
1175 
1176  if (!found_path) {
1177  fprintf(stderr, "file %s not found\n", path);
1178  dbCreated = true;
1179  }
1180  else {
1181  freeset(path, found_path);
1182  // cppcheck-suppress deallocuse (false positive; path is reassigned to non-NULp above)
1183  input = ARB_zfopen(path, "rb", ZFILE_UNCOMPRESSED, error, false); // need to use ARB_zfopen here (to be able to use ARB_zfclose below)
1184  }
1185  }
1186  else {
1187  dbCreated = true;
1188  }
1189 
1190  if (dbCreated) {
1191  fprintf(stderr, "Created new database \"%s\".\n", path);
1192  }
1193  }
1194  else {
1195  gb_assert(error); // from ARB_zfopen
1196  error = GBS_global_string("Database '%s' not found (%s)", path, error);
1197  gbc = NULp;
1198  }
1199  }
1200  if (input) {
1201  if (strchr(opent, 'D')) { // we are loading properties -> be verboose
1202  fprintf(stderr, "Using properties from '%s'\n", path);
1203  }
1204  time_of_main_file = GB_time_of_file(path);
1205 
1206  i = gb_read_in_uint32(input, 0);
1207 
1208  if (is_binary_db_id(i)) {
1209  {
1210  arb_progress progress("Loading database", GB_size_of_FILE(input)/FILESIZE_GRANULARITY);
1211  i = gb_read_bin(input, gbc, false, progress); // read or map whole db
1212  progress.done();
1213  }
1214  gbc = Main->root_container;
1215  error = ARB_zfclose(input);
1216 
1217  if (i) {
1218  if (Main->allow_corrupt_file_recovery) {
1219  GB_print_error();
1220  GB_clear_error();
1221  }
1222  else {
1223  gbc = NULp;
1224  error = GBS_global_string("Failed to load database '%s'\n"
1225  "Reason: %s",
1226  path,
1227  GB_await_error());
1228  }
1229  }
1230 
1231  if (gbc && quickFile) {
1232  long err;
1233  GB_ERROR err_msg;
1234  load_quick_save_file_only :
1235  err = 0;
1236  err_msg = NULp;
1237 
1238  input = fopen(quickFile, "rb");
1239 
1240  if (input) {
1241  GB_ULONG time_of_quick_file = GB_time_of_file(quickFile);
1242  if (time_of_main_file && time_of_quick_file < time_of_main_file) {
1243  const char *warning = GBS_global_string("Your main database file '%s' is newer than\n"
1244  " the changes file '%s'\n"
1245  " That is very strange and happens only if files where\n"
1246  " moved/copied by hand\n"
1247  " Your file '%s' may be an old relict,\n"
1248  " if you ran into problems now,delete it",
1249  path, quickFile, quickFile);
1250  GB_warning(warning);
1251  }
1252  i = gb_read_in_uint32(input, 0);
1253  if (is_binary_db_id(i)) {
1254  {
1255  arb_progress progress("Loading quicksave", GB_size_of_FILE(input)/FILESIZE_GRANULARITY);
1256  err = gb_read_bin(input, gbc, true, progress);
1257  progress.done();
1258  }
1259  fclose (input);
1260 
1261  if (err) {
1262  err_msg = GBS_global_string("Loading failed (file corrupt?)\n"
1263  "[Fail-Reason: '%s']",
1264  GB_await_error());
1265  }
1266  }
1267  else {
1268  err_msg = "Wrong file format (not a quicksave file)";
1269  err = 1;
1270  }
1271  }
1272  else {
1273  err_msg = "Can't open file";
1274  err = 1;
1275  }
1276 
1277  if (err) {
1278  error = GBS_global_string("I cannot load your quick file '%s'\n"
1279  "Reason: %s\n"
1280  "\n"
1281  "Note: you MAY restore an older version by running arb with:\n"
1282  " arb <name of quicksave-file>",
1283  quickFile, err_msg);
1284 
1285  if (!Main->allow_corrupt_file_recovery) {
1286  gbc = NULp;
1287  }
1288  else {
1289  GB_export_error(error);
1290  GB_print_error();
1291  GB_clear_error();
1292  error = NULp;
1293  GB_disable_quicksave(gbc, "Couldn't load last quicksave (your latest changes are NOT included)");
1294  }
1295  }
1296  }
1297  Main->qs.last_index = loadedQuickIndex; // determines which # will be saved next
1298  }
1299  else {
1300  if (has_ascii_db_id(i, input)) {
1301  error = gb_read_ascii_beyond_header(input, path, gbc);
1302  if (input != stdin) {
1303  GB_ERROR close_error = ARB_zfclose(input);
1304  if (!error) error = close_error;
1305  }
1307  "Sorry, I cannot save differences to ascii files\n"
1308  " Save whole database in binary mode first");
1309  }
1310  else {
1311  error = GBS_global_string("'%s' is no valid arb database file", path);
1312  }
1313 
1314  }
1315  }
1316  }
1317  }
1318  else {
1319  GB_disable_quicksave(gbc, "Database not part of this process");
1320  Main->mark_as_server();
1321  GB_begin_transaction(gbc);
1322  }
1323 
1324  if (error) gbcm_logout(Main, user);
1325  }
1326 
1327  gb_assert(error || gbc);
1328 
1329  if (error) {
1331  gbc = NULp;
1332  delete Main;
1333 
1334  GB_export_error(error);
1335  }
1336  else {
1337  GB_commit_transaction(gbc);
1338  {
1339  GB_begin_transaction(gbc); // New Transaction, should be quicksaveable
1340  if (!strchr(opent, 'N')) { // new format
1341  gb_convert_V2_to_V3(gbc); // Compression conversion
1342  }
1343  error = gb_load_key_data_and_dictionaries(Main);
1344  if (!error) error = gb_resort_system_folder_to_top(Main->root_container);
1345  if (!error) error = init_tmp_branch(gbc);
1346 
1347  GB_commit_transaction(gbc);
1348 
1349  // "handle" error
1350  if (error) GBK_terminatef("PANIC in GB_login: %s", error);
1351  }
1352  Main->security_level = 0;
1353 
1354  if (Main->is_server()) {
1356  }
1357  if (gb_verbose_mode && !dbCreated) GB_informationf("ARB: Loading '%s' done\n", path);
1358  }
1359  free(path);
1360  return gbc;
1361 }
1362 
1363 GBDATA *GB_open(const char *path, const char *opent) {
1364  const char *user;
1365  user = GB_getenvUSER();
1366  return GB_login(path, opent, user);
1367 }
1368 
1369 GB_ERROR GBT_check_arb_file(const char *name) { // goes to header: __ATTR__USERESULT
1375  GB_ERROR error = NULp;
1376  if (!strchr(name, ':')) { // don't check remote DB
1377  if (GB_is_regularfile(name)) {
1378  FILE *in = ARB_zfopen(name, "rb", ZFILE_AUTODETECT, error, true); // suppress stderr (to hide "broken pipe" errors from decompressors)
1379  if (!in) {
1380  gb_assert(error);
1381  error = GBS_global_string("Cannot read file '%s' (Reason: %s)", name, error);
1382  }
1383  else {
1384  uint32_t i = gb_read_in_uint32(in, 0);
1385 
1386  if (!is_binary_db_id(i)) {
1387  if (!has_ascii_db_id(i, in)) {
1388  error = GBS_global_string("'%s' is not an arb file", name);
1389  }
1390  }
1391  ARB_zfclose(in); // Note: error ignored here (will report broken pipe from decompressor)
1392  }
1393  }
1394  else {
1395  error = GBS_global_string("'%s' is no file", name);
1396  }
1397  }
1398  return error;
1399 }
1400 
1401 
1402 // --------------------------------------------------------------------------------
1403 
1404 #ifdef UNIT_TESTS
1405 
1406 #include <test_unit.h>
1407 
1408 void TEST_io_number() {
1409  struct data {
1410  long val;
1411  short size_expd;
1412  };
1413 
1414  data DATA[] = {
1415  { 0x0, 1 },
1416  { 0x1, 1 },
1417  { 0x7f, 1 },
1418 
1419  { 0x80, 2 },
1420  { 0x81, 2 },
1421  { 0xff, 2 },
1422  { 0x100, 2 },
1423  { 0x1234, 2 },
1424  { 0x3fff, 2 },
1425 
1426  { 0x4000, 3 },
1427  { 0x4001, 3 },
1428  { 0xffff, 3 },
1429  { 0x1fffff, 3 },
1430 
1431  { 0x200000, 4 },
1432  { 0x7fffff, 4 },
1433  { 0x800002, 4 },
1434  { 0xffffff, 4 },
1435  { 0xfffffff, 4 },
1436 
1437 #if defined(ARB_64)
1438  // the following entries are negative on 32bit systems; see ad_io_inline.h@bit-hell
1439  { 0x10000000, 5 },
1440  { 0x7fffffff, 5 },
1441  { 0x80000000, 5 },
1442  { 0x80808080, 5 },
1443  { 0xffffffff, 5 },
1444 #endif
1445  };
1446 
1447  const char *numbers = "numbers.test";
1448  long writeSize = 0;
1449  long readSize = 0;
1450 
1451  {
1452  FILE *out = fopen(numbers, "wb");
1453  TEST_REJECT_NULL(out);
1454 
1455  long lastPos = 0;
1456  for (size_t i = 0; i<ARRAY_ELEMS(DATA); ++i) {
1457  data& d = DATA[i];
1458  TEST_ANNOTATE(GBS_global_string("val=0x%lx", d.val));
1459  gb_put_number(d.val, out);
1460 
1461  long pos = ftell(out);
1462  long bytes_written = pos-lastPos;
1463  TEST_EXPECT_EQUAL(bytes_written, d.size_expd);
1464 
1465  writeSize += bytes_written;
1466 
1467  lastPos = pos;
1468  }
1469  TEST_ANNOTATE(NULp);
1470 
1471  fclose(out);
1472  }
1473 
1474  {
1475  FILE *in = fopen(numbers, "rb");
1476  TEST_REJECT_NULL(in);
1477 
1478  long lastPos = 0;
1479 
1480  for (size_t i = 0; i<ARRAY_ELEMS(DATA); ++i) {
1481  data& d = DATA[i];
1482  TEST_ANNOTATE(GBS_global_string("val=0x%lx", d.val));
1483 
1484  long val = gb_get_number(in);
1485  TEST_EXPECT_EQUAL(val, d.val);
1486 
1487  long pos = ftell(in);
1488  long bytes_read = pos-lastPos;
1489  TEST_EXPECT_EQUAL(bytes_read, d.size_expd);
1490 
1491  readSize += bytes_read;
1492 
1493  lastPos = pos;
1494  }
1495  TEST_ANNOTATE(NULp);
1496 
1497  fclose(in);
1498  }
1499 
1500  TEST_EXPECT_EQUAL(GB_size_of_file(numbers), writeSize);
1501  TEST_EXPECT_EQUAL(writeSize, readSize);
1502 
1504 }
1505 
1506 void TEST_GBDATA_size() {
1507  // protect GBDATA/GBENTRY/GBCONTAINER against unwanted changes
1508  GBENTRY fakeEntry;
1509 
1510 #if defined(ARB_64)
1511 
1512  TEST_EXPECT_EQUAL(sizeof(GBDATA), 40);
1513  TEST_EXPECT_EQUAL(sizeof(fakeEntry.info), 24); // @@@ SIZOFINTERN may be increased (to use unused space); that change would need new mapfile version
1514  TEST_EXPECT_EQUAL(sizeof(fakeEntry.cache_index), 4);
1515  // 40+24+4 = 68 (where/what are the missing 4 bytes?; they seem to be at the end of the struct; maybe padding-bytes)
1516 
1517  TEST_EXPECT_EQUAL(sizeof(GBENTRY), 72);
1518  TEST_EXPECT_EQUAL(sizeof(GBCONTAINER), 104);
1519 
1520 #else // !defined(ARB_64)
1521 
1522  TEST_EXPECT_EQUAL(sizeof(GBDATA), 24);
1523  TEST_EXPECT_EQUAL(sizeof(fakeEntry.info), 12);
1524  TEST_EXPECT_EQUAL(sizeof(fakeEntry.cache_index), 4);
1525  // 24+12+4 = 40 (here nothing is missing)
1526 
1527  TEST_EXPECT_EQUAL(sizeof(GBENTRY), 40);
1528  TEST_EXPECT_EQUAL(sizeof(GBCONTAINER), 60);
1529 
1530 #endif
1531 }
1532 TEST_PUBLISH(TEST_GBDATA_size);
1533 
1534 #endif // UNIT_TESTS
GB_ERROR GB_begin_transaction(GBDATA *gbd)
Definition: arbdb.cxx:2528
static long gb_recover_corrupt_file(GBCONTAINER *gbc, FILE *in, GB_ERROR recovery_reason, bool loading_quick_save)
Definition: ad_load.cxx:314
GB_ERROR GB_copy_dropProtectMarksAndTempstate(GBDATA *dest, GBDATA *source)
Definition: arbdb.cxx:2152
void free_data()
Definition: gb_data.h:223
void gb_put_number(long b0, FILE *out)
Definition: ad_io_inline.h:49
const char * GB_ERROR
Definition: arb_core.h:25
#define GB_SYSTEM_FOLDER
Definition: arbdb.h:27
int allow_corrupt_file_recovery
Definition: gb_main.h:125
long GB_size_of_FILE(FILE *in)
Definition: arb_file.cxx:34
GB_CSTR gb_mapfile_name(GB_CSTR path)
gb_quick_save qs
Definition: gb_main.h:127
GB_ERROR GB_write_bits(GBDATA *gbd, const char *bits, long size, const char *c_0)
Definition: arbdb.cxx:1418
GBDATA * GB_open(const char *path, const char *opent)
Definition: ad_load.cxx:1363
GB_TYPES type
GB_ERROR GB_commit_transaction(GBDATA *gbd)
Definition: arbdb.cxx:2551
void GB_warning(const char *message)
Definition: arb_msg.cxx:530
char * path
Definition: gb_main.h:122
void mark_as_server()
Definition: gb_main.h:201
Definition: arbdb.h:65
void SET_GBCONTAINER_ELEM(GBCONTAINER *gbc, int idx, GBDATA *gbd)
Definition: gb_header.h:60
GBDATA * GB_child(GBDATA *father)
Definition: adquery.cxx:322
void free_all_keys()
Definition: ad_core.cxx:752
GB_MAIN_TYPE * gb_main_array[GB_MAIN_ARRAY_SIZE]
#define implicated(hypothesis, conclusion)
Definition: arb_assert.h:289
char * dates[ALLOWED_DATES]
Definition: gb_main.h:149
unsigned int security_read
Definition: gb_data.h:69
bool getLine(string &line)
Definition: arbdb.h:69
void GB_set_next_main_idx(long idx)
Definition: ad_load.cxx:918
GB_ERROR GB_write_string(GBDATA *gbd, const char *s)
Definition: arbdb.cxx:1387
const char * quark2key(GB_MAIN_TYPE *Main, GBQUARK key_quark)
Definition: gb_key.h:45
long gb_get_number(FILE *in)
Definition: ad_io_inline.h:79
gb_flag_types2 flags2
Definition: gb_data.h:135
gb_data_base_type_union info
Definition: gb_data.h:204
long GB_mode_of_link(const char *path)
Definition: arb_file.cxx:68
GB_MAIN_TYPE * GB_MAIN(GBDATA *gbd)
Definition: gb_data.h:291
void insert_data(const char *Data, long Size, long Memsize)
Definition: gb_storage.h:60
long nref
Definition: gb_key.h:31
bool search_system_folder
Definition: gb_localdata.h:57
static long gb_read_bin(FILE *in, GBCONTAINER *gbc, bool allowed_to_load_diff, arb_progress &progress)
Definition: ad_load.cxx:652
unsigned int folded_container
Definition: gb_data.h:82
long server_id
Definition: gb_data.h:130
long gb_create_key(GB_MAIN_TYPE *Main, const char *key, bool create_gb_key)
Definition: ad_core.cxx:708
void inc_to(PINT x)
Definition: arb_progress.h:362
void GB_disable_quicksave(GBDATA *gbd, const char *reason)
Definition: arbdb.cxx:2647
#define ASSERT_RESULT(Type, Expected, Expr)
Definition: arb_assert.h:336
GB_MAIN_IDX main_idx
Definition: gb_map.h:26
gbcmc_comm * c_link
Definition: gb_main.h:117
char * ARB_strdup(const char *str)
Definition: arb_string.h:27
GB_ERROR gb_convert_V2_to_V3(GBDATA *gb_main)
Definition: adoptimize.cxx:259
void GB_internal_errorf(const char *templat,...)
Definition: arb_msg.cxx:504
const char * GBS_global_string(const char *templat,...)
Definition: arb_msg.cxx:203
void warning(int warning_num, const char *warning_message)
Definition: util.cxx:61
GB_NUMHASH * GBS_create_numhash(size_t estimated_elements)
Definition: adhash.cxx:668
unsigned int unused
Definition: gb_data.h:71
void GBK_terminatef(const char *templat,...)
Definition: arb_msg.cxx:523
static void gb_search_system_folder(GBDATA *gb_main)
Definition: ad_load.cxx:618
GBDATA * GB_HEADER_LIST_GBD(gb_header_list &hl)
Definition: gb_header.h:42
CONSTEXPR_INLINE gb_header_list * GB_DATA_LIST_HEADER(gb_data_list &dl)
Definition: gb_data.h:105
static long gb_next_main_idx_for_mapfile
Definition: ad_load.cxx:916
static GB_ERROR gb_read_ascii_beyond_header(FILE *in, const char *path, GBCONTAINER *gbc)
Definition: ad_load.cxx:299
int GB_unlink(const char *path)
Definition: arb_file.cxx:188
GB_TYPES type() const
Definition: gb_data.h:139
GB_MAIN_TYPE * GBCONTAINER_MAIN(GBCONTAINER *gbc)
Definition: gb_data.h:289
GB_CSTR gb_oldQuicksaveName(GB_CSTR path, int nr)
GB_MAIN_IDX gb_make_main_idx(GB_MAIN_TYPE *Main)
Definition: ad_load.cxx:922
GBCONTAINER * root_container
Definition: gb_main.h:120
long GB_size_of_file(const char *path)
Definition: arb_file.cxx:28
unsigned int last_updated
Definition: gb_main.h:139
FILE * ARB_zfopen(const char *name, const char *mode, FileCompressionMode cmode, GB_ERROR &error, bool hideStderr)
Definition: arb_zfile.cxx:42
#define ARRAY_ELEMS(array)
Definition: arb_defs.h:19
static void gb_delete_entry(GBCONTAINER *&gbc)
Definition: ad_core.cxx:473
long clock
Definition: gb_main.h:151
char buffer[MESSAGE_BUFFERSIZE]
Definition: seq_search.cxx:34
gb_open_types opentype
Definition: gb_main.h:123
static GB_ERROR init_tmp_branch(GBDATA *gb_main)
Definition: ad_load.cxx:1010
unsigned long GB_ULONG
Definition: arbdb_base.h:42
GB_ERROR error_if_aborted()
Definition: arb_progress.h:323
static GBDATA * gb_search_system_folder_rek(GBDATA *gbd)
Definition: ad_load.cxx:603
static int gb_verbose_mode
Definition: ad_load.cxx:30
GB_ERROR GB_delete(GBDATA *&source)
Definition: arbdb.cxx:1916
void GBT_install_message_handler(GBDATA *gb_main)
Definition: adtools.cxx:223
unsigned int last_updated
Definition: gb_data.h:79
static bool initialized
Definition: AW_advice.cxx:36
POS_TREE1 * father
Definition: probe_tree.h:39
void SET_GB_HEADER_LIST_GBD(gb_header_list &hl, GBDATA *gbd)
Definition: gb_header.h:45
#define TEST_PUBLISH(testfunction)
Definition: test_unit.h:1517
#define DEBUG_DUMP_INDENTED(d, s)
Definition: ad_load.cxx:376
unsigned int GB_UINT4
Definition: arbdb_base.h:37
const int GB_REMOTE_HASH_SIZE
Definition: adtune.cxx:17
GB_ERROR GB_export_error(const char *error)
Definition: arb_msg.cxx:257
static GB_ERROR gb_parse_ascii_rek(LineReader &r, GBCONTAINER *gb_parent, const char *parent_name)
Definition: ad_load.cxx:98
size_t getLineNumber() const
GB_ERROR GB_await_error()
Definition: arb_msg.cxx:342
uint32_t reverse_byteorder(uint32_t val)
Definition: ad_io_inline.h:21
static int diff(int v1, int v2, int v3, int v4, int st, int en)
Definition: ClustalV.cxx:534
GB_ERROR gb_unfold(GBCONTAINER *gbc, long deep, int index_pos)
Definition: arbdb.cxx:586
gb_flag_types flags
Definition: gb_data.h:134
Definition: arbdb.h:67
Definition: arbdb.h:78
GB_TYPES GB_read_type(GBDATA *gbd)
Definition: arbdb.cxx:1643
#define short
Definition: ureadseq.h:8
bool has_ascii_db_id(uint32_t bin_id, FILE *in)
Definition: ad_load.cxx:987
GB_ERROR GB_install_pid(int mode)
Definition: adcomm.cxx:1957
GB_CSTR GB_read_key_pntr(GBDATA *gbd)
Definition: arbdb.cxx:1656
char * GB_property_file(bool warn_when_not_found, const char *filename)
Definition: adfile.cxx:204
char * GB_map_FILE(FILE *in, int writeable)
Definition: adsocket.cxx:323
GB_HASH * key_2_index_hash
Definition: gb_main.h:135
bool aborted()
Definition: arb_progress.h:335
bool is_server() const
Definition: gb_main.h:203
GB_BUFFER GB_give_buffer(size_t size)
Definition: arbdb.cxx:311
void GB_clear_error()
Definition: arb_msg.cxx:354
unsigned long GB_time_of_file(const char *path)
Definition: arb_file.cxx:44
int highest_quick_index
Definition: gb_load.h:25
GB_ERROR login_remote(const char *db_path, const char *opent) __ATTR__USERESULT
Definition: ad_load.cxx:955
gb_scan_quicks_types type
Definition: gb_load.h:28
#define TEST_REJECT_NULL(n)
Definition: test_unit.h:1325
static void error(const char *msg)
Definition: mkptypes.cxx:96
GB_ERROR gb_scan_directory(char *basename, gb_scandir *sd)
Definition: adfile.cxx:30
long gb_read_bin_error(FILE *in, GBDATA *gbd, const char *text)
#define GB_KEY_LEN_MAX
Definition: gb_key.h:25
void index_check_out()
Definition: adindex.cxx:65
int gb_is_valid_mapfile(const char *path, gb_map_header *mheader, int verbose)
Definition: admap.cxx:652
GB_CSTR gb_quicksaveName(GB_CSTR path, int nr)
int last_index
Definition: gb_main.h:43
#define GBTUM_MAGIC_REVERSED
Definition: gb_local.h:27
unsigned int security_delete
Definition: gb_data.h:67
bool GB_is_fifo(const char *path)
Definition: arb_file.cxx:89
GBCONTAINER * as_container() const
Definition: gb_data.h:155
int GB_random(int range)
Definition: admath.cxx:88
#define TEST_EXPECT_ZERO_OR_SHOW_ERRNO(iocond)
Definition: test_unit.h:1090
void gb_create_header_array(GBCONTAINER *gbc, int size)
Definition: ad_core.cxx:176
long first_free_key
Definition: gb_main.h:133
long nref_last_saved
Definition: gb_key.h:33
static long gb_read_bin_rek_V2(FILE *in, GBCONTAINER *gbc_dest, long nitems, long version, long reversed, long deep, arb_progress &progress)
Definition: ad_load.cxx:380
CONSTEXPR_INLINE GBCONTAINER * GB_FATHER(GBDATA *gbd)
Definition: gb_data.h:271
int cache_index
Definition: gb_data.h:206
char * key
Definition: gb_key.h:29
GB_ERROR GB_write_float(GBDATA *gbd, float f)
Definition: arbdb.cxx:1281
GB_ERROR GB_write_int(GBDATA *gbd, long i)
Definition: arbdb.cxx:1250
void GB_init_gb(void)
Definition: arbdb.cxx:512
GB_ERROR GB_print_error()
Definition: arb_msg.cxx:324
GBENTRY * gb_make_entry(GBCONTAINER *father, const char *key, long index_pos, GBQUARK keyq, GB_TYPES type)
Definition: ad_core.cxx:353
static GBDATA * GB_login(const char *cpath, const char *opent, const char *user)
Definition: ad_load.cxx:1022
GB_ERROR GB_set_temporary(GBDATA *gbd) __ATTR__USERESULT
Definition: arbdb.cxx:2282
Definition: arbdb.h:72
int newest_quick_index
Definition: gb_load.h:26
#define ALLOWED_DATES
Definition: gb_main.h:85
fputs(TRACE_PREFIX, stderr)
unsigned int security_level
Definition: gb_main.h:150
Definition: arbdb.h:63
GB_ERROR GB_export_errorf(const char *templat,...)
Definition: arb_msg.cxx:262
BufferedPipeReader(const string &pipename, FILE *in)
Definition: ad_load.cxx:278
void GB_internal_error(const char *message)
Definition: arb_msg.cxx:481
CONSTEXPR_INLINE_Cxx14 void SET_GB_FATHER(GBDATA *gbd, GBCONTAINER *father)
Definition: gb_data.h:274
GBCONTAINER * gb_make_container(GBCONTAINER *father, const char *key, long index_pos, GBQUARK keyq)
Definition: ad_core.cxx:414
#define FILESIZE_GRANULARITY
Definition: ad_load.cxx:35
bool mapped
Definition: gb_main.h:137
#define GBTUM_MAGIC_NUMBER
Definition: gb_local.h:25
#define gb_assert(cond)
Definition: arbdbt.h:11
unsigned int ever_changed
Definition: gb_header.h:23
void GB_write_flag(GBDATA *gbd, long flag)
Definition: arbdb.cxx:2773
GB_ERROR close()
Definition: ad_load.cxx:286
static const char * getToken(char **line)
Definition: ad_load.cxx:48
GB_BUFFER GB_increase_buffer(size_t size)
Definition: arbdb.cxx:316
GB_ERROR ARB_zfclose(FILE *fp)
Definition: arb_zfile.cxx:166
GB_CSTR GB_getenvUSER(void)
Definition: adsocket.cxx:545
#define A_TO_I(c)
Definition: gb_load.h:16
long index
Definition: gb_data.h:133
short GB_MAIN_IDX
Definition: gb_local.h:32
void touch_creation_and_update(long date)
Definition: gb_data.h:190
long quark2gbmindex(GB_MAIN_TYPE *Main, GBQUARK key_quark)
Definition: gb_key.h:46
GB_ERROR GB_write_byte(GBDATA *gbd, int i)
Definition: arbdb.cxx:1238
GB_ERROR gbcm_logout(GB_MAIN_TYPE *Main, const char *loginname)
Definition: adcomm.cxx:1934
void create_extended()
Definition: gb_data.h:176
GB_ERROR gb_resort_system_folder_to_top(GBCONTAINER *gb_main)
Definition: arbdb.cxx:2705
gb_open_types
Definition: gb_main.h:35
#define GB_MAIN_ARRAY_SIZE
Definition: gb_load.h:14
float GB_atof(const char *str)
Definition: arbdb.cxx:190
const char * GB_get_db_path(GBDATA *gbd)
Definition: adTest.cxx:14
bool read_keyword(const char *expected_keyword, FILE *in, GBCONTAINER *gbc)
Definition: ad_load.cxx:640
char * GBS_fconvert_string(char *buffer)
Definition: adstring.cxx:880
char * gb_findExtension(char *path)
gb_Key * keys
Definition: gb_main.h:134
NOT4PERL int GB_give_buffer_size(void)
Definition: arbdb.cxx:330
static GB_ERROR gb_parse_ascii(LineReader &r, GBCONTAINER *gb_parent)
Definition: ad_load.cxx:269
GBENTRY * as_entry() const
Definition: gb_data.h:150
void GB_set_verbose()
Definition: ad_load.cxx:31
static int line
Definition: arb_a2ps.c:296
#define NULp
Definition: cxxforward.h:116
bool GB_is_regularfile(const char *path)
Definition: arb_file.cxx:76
gb_data_list d
Definition: gb_data.h:246
bool is_binary_db_id(int id)
Definition: ad_load.cxx:982
int keycnt
Definition: gb_main.h:131
gb_header_flags flags
Definition: gb_header.h:35
void GB_informationf(const char *templat,...)
Definition: arb_msg.cxx:555
gbcmc_comm * gbcmc_open(const char *path)
Definition: adsocket.cxx:170
void release_main_idx()
Definition: ad_load.cxx:947
bool ARB_strBeginsWith(const char *str, const char *with)
Definition: arb_str.h:42
const int GBTUM_SHORT_STRING_SIZE
Definition: adtune.cxx:28
GB_TYPES
Definition: arbdb.h:62
GBDATA * GB_nextChild(GBDATA *child)
Definition: adquery.cxx:326
GBDATA * gb_main
Definition: adname.cxx:32
void gbm_free_mem(void *block, size_t size, long index)
Definition: gb_memory.h:131
Definition: arbdb.h:71
bool GBS_string_matches(const char *str, const char *expr, GB_CASE case_sens)
Definition: admatch.cxx:193
GB_ERROR GBT_check_arb_file(const char *name)
Definition: ad_load.cxx:1369
GBDATA * GB_search(GBDATA *gbd, const char *fieldpath, GB_TYPES create)
Definition: adquery.cxx:531
char * GB_follow_unix_link(const char *path)
Definition: arb_file.cxx:262
GB_NUMHASH * remote_hash
Definition: gb_main.h:152
int nheader
Definition: gb_data.h:102
GB_MAIN_IDX main_idx
Definition: gb_data.h:255
const char * GB_CSTR
Definition: arbdb_base.h:25
GBDATA * gb_map_mapfile(const char *path)
Definition: admap.cxx:706
int GBQUARK
Definition: arbdb_base.h:30
#define TEST_EXPECT_EQUAL(expr, want)
Definition: test_unit.h:1294
uint32_t gb_read_in_uint32(FILE *in, bool reversed)
Definition: ad_io_inline.h:42
gb_local_data * gb_local
Definition: arbdb.cxx:33
void set_change(GB_CHANGE val)
Definition: gb_header.h:25
GBDATA * GB_entry(GBDATA *father, const char *key)
Definition: adquery.cxx:334
unsigned int security_write
Definition: gb_data.h:68
static GB_ERROR set_protection_level(GB_MAIN_TYPE *Main, GBDATA *gbd, const char *p)
Definition: ad_load.cxx:61
GB_ERROR gbcm_login(GBCONTAINER *gb_main, const char *loginname)
Definition: adcomm.cxx:1883
unsigned int compressed_data
Definition: gb_data.h:70
long gb_ascii_2_bin(const char *source, GBENTRY *gbe)
long next_free_key
Definition: gb_key.h:32
GBCONTAINER * dummy_father
Definition: gb_main.h:119
#define ALLOWED_KEYS
Definition: gb_main.h:84
GB_HASH * GBS_create_hash(long estimated_elements, GB_CASE case_sens)
Definition: adhash.cxx:253
Definition: arbdb.h:66
void gb_delete_dummy_father(GBCONTAINER *&gbc)
Definition: ad_core.cxx:542
unsigned int flags
Definition: gb_header.h:20
GB_ERROR gb_load_key_data_and_dictionaries(GB_MAIN_TYPE *Main)
Definition: adsystem.cxx:209
GB_write_int const char s
Definition: AW_awar.cxx:154