NT_sort.cxx

Go to the documentation of this file.

// =============================================================== //
//                                                                 //
//   File      : NT_sort.cxx                                       //
//   Purpose   :                                                   //
//                                                                 //
//   Institute of Microbiology (Technical University Munich)       //
//   http://www.arb-home.de/                                       //
//                                                                 //
// =============================================================== //

#include "NT_local.h"

#include <item_sel_list.h>
#include <aw_awar.hxx>
#include <arb_progress.h>
#include <aw_msg.hxx>
#include <aw_root.hxx>
#include <TreeNode.h>
#include <arb_sort.h>
#include <arb_global_defs.h>

#define FIELD_FILTER_RESORT (1<<GB_STRING)|(1<<GB_INT)|(1<<GB_FLOAT) // field types supported by cmpByKey()

#define CUSTOM_CRITERIA 3

struct customCriterion {
    char *key;
    bool  reverse;
    bool  is_valid;

    void check_valid() {
        is_valid = key && strcmp(key, NO_FIELD_SELECTED) != 0;
    }

    customCriterion() : key(NULp), reverse(false) { check_valid(); }
    customCriterion(const char *key_, bool reverse_) : key(ARB_strdup(key_)), reverse(reverse_) { check_valid(); }
    customCriterion(const customCriterion& other) : key(nulldup(other.key)), reverse(other.reverse) { check_valid(); }
    DECLARE_ASSIGNMENT_OPERATOR(customCriterion);
    ~customCriterion() { free(key); }
};

static int cmpByKey(GBDATA *gbd1, GBDATA *gbd2, const customCriterion& by) {
    int cmp = 0;
    if (by.is_valid) {
        GBDATA *gb_field1 = GB_entry(gbd1, by.key);
        GBDATA *gb_field2 = GB_entry(gbd2, by.key);

        if (gb_field1) {
            if (gb_field2) {
                switch (GB_read_type(gb_field1)) {
                    case GB_STRING: {
                        const char *s1 = GB_read_char_pntr(gb_field1);
                        const char *s2 = GB_read_char_pntr(gb_field2);

                        cmp = strcmp(s1, s2);
                        break;
                    }
                    case GB_FLOAT: {
                        float f1 = GB_read_float(gb_field1);
                        float f2 = GB_read_float(gb_field2);

                        cmp = f1<f2 ? -1 : (f1>f2 ? 1 : 0);
                        break;
                    }
                    case GB_INT: {
                        int i1 = GB_read_int(gb_field1);
                        int i2 = GB_read_int(gb_field2);

                        cmp = i1-i2;
                        break;
                    }
                    default:
                        cmp = 0; // other field type -> no idea how to compare
                        break;
                }

                if (by.reverse) cmp = -cmp;
            }
            else cmp = -1;           // existing < missing!
        }
        else cmp = gb_field2 ? 1 : 0;
    }
    return cmp;
}

static GBDATA **gb_resort_data_list;
static long     gb_resort_data_count;

static void NT_resort_data_base_by_tree(TreeNode *tree, GBDATA *gb_species_data) {
    if (tree) {
        if (tree->is_leaf()) {
            if (tree->gb_node) {
                gb_resort_data_list[gb_resort_data_count++] = tree->gb_node;
            }
        }
        else {
            NT_resort_data_base_by_tree(tree->get_leftson(), gb_species_data);
            NT_resort_data_base_by_tree(tree->get_rightson(), gb_species_data);
        }
    }
}

static bool customOrderIsStrict = true;
static bool customDefinesOrder  = false;

static int resort_data_by_customOrder(const void *v1, const void *v2, void *cd_sortBy) {
    GBDATA *gbd1 = (GBDATA*)v1;
    GBDATA *gbd2 = (GBDATA*)v2;

    const customCriterion *sortBy = (const customCriterion *)cd_sortBy;

    int cmp = 0;
    for (int c = 0; !cmp && c<CUSTOM_CRITERIA; ++c) {
        cmp = cmpByKey(gbd1, gbd2, sortBy[c]);
    }

    if (!cmp) customOrderIsStrict = false;
    else customDefinesOrder       = true;

    return cmp;
}

static GB_ERROR resort_data_base(GBDATA *gb_main, TreeNode *tree, const customCriterion *sortBy) {
    nt_assert(contradicted(tree, sortBy));

    GB_ERROR error = GB_begin_transaction(gb_main);
    if (!error) {
        GBDATA *gb_sd     = GBT_get_species_data(gb_main);
        if (!gb_sd) error = GB_await_error();
        else {
            if (tree) {
                gb_resort_data_count = 0;
                ARB_calloc(gb_resort_data_list, GB_nsons(gb_sd) + 256);
                NT_resort_data_base_by_tree(tree, gb_sd);
            }
            else {
                gb_resort_data_list = GBT_gen_species_array(gb_main, &gb_resort_data_count);

                nt_assert(gb_resort_data_count>=0);
                nt_assert(implicated(gb_resort_data_count>0, gb_resort_data_list)); // accept NULp array (if no species exists). GB_sort can handle that and GB_resort_data_base should as well

                GB_sort((void **)gb_resort_data_list, 0, gb_resort_data_count, resort_data_by_customOrder, (void*)sortBy);
            }

            error = GB_resort_data_base(gb_main, gb_resort_data_list, gb_resort_data_count);

            free(gb_resort_data_list);
        }
    }
    return GB_end_transaction(gb_main, error);
}

static GB_ERROR strict_resort_data_base(GBDATA *gb_main, const customCriterion *sortBy) {
    customOrderIsStrict = true;
    customDefinesOrder  = false;

    GB_ERROR error_or_warning = resort_data_base(gb_main, NULp, sortBy);
    // Note: if a real error occurs, transaction was aborted inside resort_data_base.

    if (!error_or_warning) {
        if      (!customDefinesOrder)  error_or_warning = "Warning: No order is defined by the specified fields";
        else if (!customOrderIsStrict) error_or_warning = "Note: The specified fields do not define a strict order";
    }
    return error_or_warning;
}

void NT_resort_data_by_phylogeny(AW_window*, TREE_canvas *ntw) {
    arb_progress  progress("Sorting data");
    GB_ERROR      error = NULp;
    TreeNode     *tree  = NT_get_tree_root_of_canvas(ntw);

    if (!tree)  error = "Please select/build a tree first";
    if (!error) error = resort_data_base(GLOBAL.gb_main, tree, NULp);
    if (error) aw_message(error);
}

#define AWAR_TREE_SORT1 "db_sort/sort_1"
#define AWAR_TREE_SORT2 "db_sort/sort_2"
#define AWAR_TREE_SORT3 "db_sort/sort_3"

#define AWAR_TREE_REV1 "db_sort/rev1"
#define AWAR_TREE_REV2 "db_sort/rev2"
#define AWAR_TREE_REV3 "db_sort/rev3"

static void NT_resort_data_by_user_criteria(AW_window *aw) {
    arb_progress progress("Sorting data");

    AW_root *aw_root = aw->get_root();

    customCriterion sortBy[CUSTOM_CRITERIA];
    sortBy[0] = customCriterion(aw_root->awar(AWAR_TREE_SORT1)->read_char_pntr(), aw_root->awar(AWAR_TREE_REV1)->read_int());
    sortBy[1] = customCriterion(aw_root->awar(AWAR_TREE_SORT2)->read_char_pntr(), aw_root->awar(AWAR_TREE_REV2)->read_int());
    sortBy[2] = customCriterion(aw_root->awar(AWAR_TREE_SORT3)->read_char_pntr(), aw_root->awar(AWAR_TREE_REV3)->read_int());

    GB_ERROR error_or_warning = strict_resort_data_base(GLOBAL.gb_main, sortBy);
    if (error_or_warning) aw_message(error_or_warning);
}

void NT_create_resort_awars(AW_root *awr, AW_default aw_def) {
    awr->awar_string(AWAR_TREE_SORT1, "name",            aw_def);
    awr->awar_string(AWAR_TREE_SORT2, "full_name",       aw_def);
    awr->awar_string(AWAR_TREE_SORT3, NO_FIELD_SELECTED, aw_def);

    awr->awar_int(AWAR_TREE_REV1, 0, aw_def);
    awr->awar_int(AWAR_TREE_REV2, 0, aw_def);
    awr->awar_int(AWAR_TREE_REV3, 0, aw_def);
}

AW_window *NT_create_resort_window(AW_root *awr) {
    AW_window_simple *aws = new AW_window_simple;
    aws->init(awr, "SORT_DB_ENTRIES", "SORT DATABASE");
    aws->load_xfig("nt_sort.fig");

    aws->at("close");
    aws->callback(AW_POPDOWN);
    aws->create_button("CLOSE", "CLOSE", "C");

    aws->callback(makeHelpCallback("sp_sort_fld.hlp"));
    aws->at("help");
    aws->create_button("HELP", "HELP", "H");

    create_itemfield_selection_button(aws, FieldSelDef(AWAR_TREE_SORT1, GLOBAL.gb_main, SPECIES_get_selector(), FIELD_FILTER_RESORT, "1st sort field"), "key1");
    create_itemfield_selection_button(aws, FieldSelDef(AWAR_TREE_SORT2, GLOBAL.gb_main, SPECIES_get_selector(), FIELD_FILTER_RESORT, "2nd sort field"), "key2");
    create_itemfield_selection_button(aws, FieldSelDef(AWAR_TREE_SORT3, GLOBAL.gb_main, SPECIES_get_selector(), FIELD_FILTER_RESORT, "3rd sort field"), "key3");

    aws->at("rev1"); aws->label("Reverse"); aws->create_toggle(AWAR_TREE_REV1);
    aws->at("rev2"); aws->label("Reverse"); aws->create_toggle(AWAR_TREE_REV2);
    aws->at("rev3"); aws->label("Reverse"); aws->create_toggle(AWAR_TREE_REV3);

    aws->at("go");
    aws->callback(NT_resort_data_by_user_criteria);
    aws->create_button("GO", "GO", "G");

    return aws;
}


// --------------------------------------------------------------------------------

#ifdef UNIT_TESTS
#ifndef TEST_UNIT_H
#include <test_unit.h>
#endif

#define TEST_EXPECT_DATABASE_ORDER(expected_order) do{                  \
        GB_transaction ta1(gb_main);                                    \
        char *got_order = GBT_store_marked_species(gb_main, false);     \
        TEST_EXPECT_EQUAL(got_order, expected_order);                   \
        free(got_order);                                                \
    }while(0)

#define TEST_EXPECT_ANNOTATD_ORDER(expected_order,expected_values,key) do{                                      \
        GB_transaction  ta2(gb_main);                                                                           \
        TEST_EXPECT_DATABASE_ORDER(expected_order);                                                             \
        char *aci        = GBS_global_string_copy("split(\";\")|findspec(readdb(%s))|merge(\";\")", key);       \
        char *gen_values = GB_command_interpreter(expected_order, aci, gb_main);                                \
        if (!gen_values) TEST_EXPECT_NO_ERROR(GB_await_error());                                                \
        TEST_EXPECT_EQUAL(gen_values, expected_values);                                                         \
        free(gen_values);                                                                                       \
        free(aci);                                                                                              \
    }while(0)

void TEST_resort_database() {
    GB_shell  shell;
    GBDATA   *gb_main = GB_open("TEST_fields_ascii.arb", "r"); // ../UNIT_TESTER/run/TEST_fields_ascii.arb

    GBT_mark_all(gb_main, 1); // needed to test result

    const char *const BYNAME_ORDER  = "AcsCell2;AgrSp148;BurCalid;ClaMich8;CytHutc2;DstHafn2;ErwOleae;GlmApico;HrpAura2;IlyPoly2;MhyPalud;MtpKand2;OcgTerie;PesPropi;PslBats2;PurGergo;SrrAquat;StxAceti;TreIsopt;VibJapon;XanCucur;YerKrist";
    const char *const PHYLO_ORDER   = "ErwOleae;PurGergo;YerKrist;SrrAquat;GlmApico;VibJapon;MhyPalud;BurCalid;StxAceti;XanCucur;AgrSp148;PslBats2;MtpKand2;OcgTerie;PesPropi;DstHafn2;ClaMich8;AcsCell2;HrpAura2;CytHutc2;TreIsopt;IlyPoly2";
    const char *const INITIAL_ORDER = "ErwOleae;PurGergo;YerKrist;SrrAquat;GlmApico;VibJapon;MhyPalud;BurCalid;StxAceti;XanCucur;AgrSp148;PslBats2;TreIsopt;IlyPoly2;CytHutc2;ClaMich8;AcsCell2;HrpAura2;PesPropi;DstHafn2;OcgTerie;MtpKand2";

    // test initial order:
    TEST_EXPECT_DATABASE_ORDER(INITIAL_ORDER);

    // tests for NT_resort_data_by_phylogeny():
    {
        TreeNode *tree;
        {
            GB_transaction ta(gb_main);
            tree = GBT_read_tree(gb_main, "tree_LTPs132_SSU", new SimpleRoot);
            TEST_REJECT_NULL(tree);

            TEST_EXPECT_NO_ERROR(GBT_link_tree(tree, gb_main, false, NULp, NULp)); // link the tree
        }
        TEST_EXPECT_NO_ERROR(resort_data_base(gb_main, tree, NULp));

        // test resulting order:
        TEST_EXPECT_DATABASE_ORDER(PHYLO_ORDER);

        destroy(tree);
    }

    // tests for NT_resort_data_by_user_criteria():
    customCriterion csc[CUSTOM_CRITERIA];
    csc[0] = customCriterion("name", 0);

    // test resorting by dbfield (string 'name'):
    TEST_EXPECT_NO_ERROR(strict_resort_data_base(gb_main, csc));
    TEST_EXPECT_DATABASE_ORDER(BYNAME_ORDER);

    // test resorting by dbfield (int) + use reverse order:
    csc[0] = customCriterion("align_bp_score_slv", 1);
    TEST_EXPECT_ERROR_CONTAINS(strict_resort_data_base(gb_main, csc), "specified fields do not define a strict order");
    TEST_EXPECT_NO_ERROR(resort_data_base(gb_main, NULp, csc));
#if defined(LINUX)
    // the order produced by this sort is unstable and implementation dependant (e.g. it differs between LINUX and OSX).
    // Its enough to test the result for one OS. The fact that non-strictness is detected is more important and tested above.
    TEST_EXPECT_ANNOTATD_ORDER("AcsCell2;MtpKand2;ClaMich8;AgrSp148;BurCalid;PslBats2;ErwOleae;GlmApico;XanCucur;YerKrist;IlyPoly2;PurGergo;StxAceti;PesPropi;VibJapon;SrrAquat;DstHafn2;TreIsopt;MhyPalud;HrpAura2;OcgTerie;CytHutc2",
                               "124;124;122;121;121;121;120;119;119;118;117;117;117;116;116;115;114;113;112;111;105;101",
                               "align_bp_score_slv");
#endif
    csc[1] = customCriterion("align_quality_slv", 0); // add a 2nd criterion (making sort stable)
    TEST_EXPECT_NO_ERROR(strict_resort_data_base(gb_main, csc));
    // --- diff to above------: XXXXXXXXXXXXXXXXX-------------------XXXXXXXXXXXXXXXXX----------------------------------------------XXXXXXXXXXXXXXXXX---------------------------------------------------------------------------------
    TEST_EXPECT_ANNOTATD_ORDER("MtpKand2;AcsCell2;ClaMich8;AgrSp148;PslBats2;BurCalid;ErwOleae;GlmApico;XanCucur;YerKrist;IlyPoly2;StxAceti;PurGergo;PesPropi;VibJapon;SrrAquat;DstHafn2;TreIsopt;MhyPalud;HrpAura2;OcgTerie;CytHutc2",
                               //XXXX-------XXXXX----------------XXXXX--------------------------- marks those values which make the sort stable!
                               "91;96;99;95;96;97;97;92;99;99;87;93;99;86;99;99;91;72;92;83;82;88",
                               "align_quality_slv");
    csc[1] = customCriterion(); // "remove" criterion

    // test resorting by dbfield (float)
    csc[0] = customCriterion("homop_slv", 0);
    TEST_EXPECT_ERROR_CONTAINS(strict_resort_data_base(gb_main, csc), "specified fields do not define a strict order");
    TEST_EXPECT_NO_ERROR(resort_data_base(gb_main, NULp, csc));
#if defined(LINUX)
    // (see comment about 'non-strictness' above)
    TEST_EXPECT_ANNOTATD_ORDER("DstHafn2;XanCucur;AgrSp148;GlmApico;IlyPoly2;PesPropi;ClaMich8;StxAceti;OcgTerie;MhyPalud;CytHutc2;AcsCell2;BurCalid;PslBats2;ErwOleae;YerKrist;PurGergo;SrrAquat;TreIsopt;VibJapon;HrpAura2;MtpKand2",
                               "0.12;0.13;0.2;0.2;0.2;0.2;0.27;0.27;0.27;0.28;0.29;0.33;0.33;0.35;0.4;0.41;0.41;0.41;0.41;0.47;0.68;1.6",
                               "homop_slv");
#endif

    csc[0] = customCriterion("align_ident_slv", 0);
    TEST_EXPECT_NO_ERROR(strict_resort_data_base(gb_main, csc));
    TEST_EXPECT_ANNOTATD_ORDER("OcgTerie;HrpAura2;TreIsopt;IlyPoly2;PesPropi;MtpKand2;DstHafn2;CytHutc2;StxAceti;MhyPalud;BurCalid;GlmApico;AgrSp148;AcsCell2;PslBats2;ErwOleae;ClaMich8;PurGergo;SrrAquat;VibJapon;YerKrist;XanCucur",
                               "69.761269;73.481384;74.731186;76.5748;77.514793;79.776756;79.802635;82.919708;87.339195;87.470451;89.498642;89.625168;89.759888;90.149460;92.114960;94.846054;95.121948;97.172417;97.569443;99.170128;99.247093;100", // that is strict
                               "align_ident_slv");

    GB_close(gb_main);
}

#endif // UNIT_TESTS

// --------------------------------------------------------------------------------