xferset.cxx

Go to the documentation of this file.

// ========================================================= //
//                                                           //
//   File      : xferset.c                                   //
//   Purpose   : field transfer sets                         //
//                                                           //
//   Coded by Ralf Westram (coder@reallysoft.de) in Mar 19   //
//   http://www.arb-home.de/                                 //
//                                                           //
// ========================================================= //

#include "xferset.h"

#include <ConfigMapping.h>
#include <BufferedFileReader.h>
#include <arbdbt.h>
#include <arb_str.h>
#include <arb_stdstr.h>

#include <set>
#include <gb_aci.h>

using namespace std;

namespace FieldTransfer {

    typedef set<string, NoCaseCmp> StrSet;

    static void StrSet2StrArray(const StrSet& src, StrArray& dst) {
        for (StrSet::const_iterator i = src.begin(); i != src.end(); ++i) {
            dst.put(strdup(i->c_str()));
        }
    }
    static void StrArray2StrSet(const StrArray& src, StrSet& dst) {
        for (unsigned i = 0; i<src.size(); ++i) {
            dst.insert(src[i]);
        }
    }

    void RuleSet::extractUsedFields(StrArray& input, StrArray& output) const { // @@@ want flavor just filling 2 StrSets
        StrSet in, out;
        for (unsigned i = 0; i<size(); ++i) {
            const Rule&   rule      = get(i);
            const string& srcFields = rule.getSourceFields();
            if (rule.multiple_source_fields()) {
                ConstStrArray ifield;
                GBT_split_string(ifield, srcFields.c_str(), ';');
                for (unsigned f = 0; f<ifield.size(); ++f) {
                    const char *source = ifield[f];
                    if (source[0]) in.insert(source);
                }
            }
            else {
                if (!srcFields.empty()) in.insert(srcFields);
            }
            const string& target = rule.targetField();
            if (!target.empty()) out.insert(target);
        }
        StrSet2StrArray(in,  input);
        StrSet2StrArray(out, output);
    }

    TransportedData ReadRule::readTypedFromField(GB_TYPES readAsType, GBDATA *gb_field) const {
        xf_assert(GB_read_type(gb_field) != GB_DB); // fails e.g. if rule defined for a name used by a container

        switch (readAsType) {
            case GB_INT: {
                int asInt = GB_read_int(gb_field);
                return TransportedData(asInt);
            }
            case GB_FLOAT: {
                float asFloat = GB_read_float(gb_field);
                return TransportedData(asFloat);
            }
            case GB_STRING: {
                char   *asStr = GB_read_as_string(gb_field);
                string  data(asStr);
                free(asStr);
                return TransportedData(data);
            }
            default: xf_assert(0); break; // invalid type
        }
        xf_assert(0); // should be never reached.
        return TransportedData::none();
    }

    TransportedData ReadRule::aciAppliedTo(const string& toStr, GBDATA *gb_main, GBDATA *gb_dest_item) const {
        // We can not generally provide a meaningful item for ACI here.
        // Currently it always uses the destination item, but this item may be some dummy item,
        // e.g. a species clone used only during transfer.

        GBL_env      env(gb_main, NULp);
        GBL_call_env callEnv(gb_dest_item, env);

        char *result = GB_command_interpreter_in_env(toStr.c_str(), aci.c_str(), callEnv);
        if (result) {
            string converted(result);
            free(result);
            return TransportedData(converted);
        }
        return TransportedData::makeError(GB_await_error());
    }

    inline TransportedData cannotReadContainer(const char *containerName) {
        return TransportedData::makeError(GBS_global_string("cannot read as data ('%s' is a container)", containerName));
    }

    TransportedData ReadRule::readFrom(GBDATA *gb_item, GBDATA *gb_dest_item) const {
        // 'gb_dest_item' only used for ACI

        if (!gb_item) { // got no item -> can't read
            return TransportedData::makeError("lacking item to readFrom");
        }

        if (fields.empty()) {
            return TransportedData::makeError("no source field(s) specified");
        }

        if (multiple_source_fields()) {
            ConstStrArray field;
            GBT_split_string(field, fields.c_str(), ';');

            string concat;
            bool   gotData = false; // at least one input field found?
            for (size_t f = 0; f<field.size(); ++f) {
                GBDATA *gb_field = GB_search(gb_item, field[f], GB_FIND);
                if (gb_field) {
                    GB_TYPES sourceType = GB_read_type(gb_field);
                    if (sourceType == GB_DB) {
                        return cannotReadContainer(field[f]);
                    }

                    TransportedData plain = readTypedFromField(GB_STRING, gb_field); // ignores sourceType
                    if (plain.failed()) return plain;

                    xf_assert(plain.exists());
                    if (!concat.empty()) concat += separator;
                    concat                      += plain.getString();

                    gotData = true;
                }
                else if (GB_have_error()) {
                    return TransportedData::makeError(GB_await_error());
                }
            }

            if (gotData) {
                if (!aci.empty()) {
                    return aciAppliedTo(concat, GB_get_root(gb_item), gb_dest_item);
                }
                return TransportedData(concat);
            }
            // otherwise: do not transport if all source fields are missing
        }
        else {
            GBDATA *gb_field = GB_search(gb_item, fields.c_str(), GB_FIND);
            if (gb_field) {
                GB_TYPES sourceType = GB_read_type(gb_field);
                if (sourceType == GB_DB) {
                    return cannotReadContainer(fields.c_str());
                }
                if (!aci.empty()) {
                    TransportedData plain = readTypedFromField(GB_STRING, gb_field); // ignores sourceType
                    // @@@ store sourceType if no dest-type deduced?
                    return aciAppliedTo(plain.getString(), GB_get_root(gb_item), gb_dest_item);
                }
                return readTypedFromField(sourceType, gb_field);
            }
            else if (GB_have_error()) {
                return TransportedData::makeError(GB_await_error());
            }
            // otherwise: do not transport if source field is missing
        }
        // if field does not exist -> report "no type"
        return TransportedData::none();
    }

    static GB_ERROR unconvertedWrite(const TransportedData& data, GBDATA *gb_field) {
        GB_ERROR error = NULp;
        switch (data.getType()) {
            case GB_STRING: {
                const char *str = data.getString().c_str();
                error           = GB_write_string(gb_field, str);
                break;
            }
            case GB_INT: {
                int num = data.getInt();
                error   = GB_write_int(gb_field, num);
                break;
            }
            case GB_FLOAT: {
                float fnum = data.getFloat();
                error      = GB_write_float(gb_field, fnum);
                break;
            }
            default: { // unhandled type
                xf_assert(0);
                break;
            }
        }
        return error;
    }
    static GB_ERROR convertAndWrite(const TransportedData& data, GBDATA *gb_field, GB_TYPES wantedTargetType, bool acceptLossyConversion) {
        // perform conversion to 'wantedTargetType' and write to 'gb_field'
        GB_ERROR error = NULp;

        switch (data.getType()) {
            case GB_INT:
                if (wantedTargetType == GB_FLOAT) {
                    // convert int -> float
                    float   f = data.getInt();
                    int32_t i = int(f+.5); // round (just in case some underflow happened, causing sth like 4711.9999999)

                    if (i != data.getInt() && !acceptLossyConversion) {
                        error = GBS_global_string("lossy int->float type conversion (%i->%i)", data.getInt(), i);
                    }
                    else {
                        error = GB_write_float(gb_field, f);
                    }
                }
                else {
                    error = GB_write_lossless_int(gb_field, data.getInt());
                }
                break;

            case GB_FLOAT:
                if (wantedTargetType == GB_INT) {
                    // convert float -> int
                    double d  = data.getFloat();
                    int    i  = d>0 ? (int)(d+0.5) : (int)(d-0.5);
                    // @@@ increment a round-counter in RuleSet?
                    double d2 = i;

                    if (d != d2 && !acceptLossyConversion) { // precision loss
                        error = GBS_global_string("lossy float->int type conversion (%e->%e)", d, d2);
                    }
                    else {
                        error = GB_write_int(gb_field, i);
                    }
                }
                else {
                    error = GB_write_lossless_float(gb_field, data.getFloat());
                }
                break;

            case GB_STRING:
                error = GB_write_autoconv_string(gb_field, data.getString().c_str()); // @@@ avoid silent data loss
                // @@@ use GBT_write_float_converted / GBT_write_int_converted here!
                break;

            default:
                xf_assert(0); // unhandled type
                break;
        }

        return error;
    }

    GB_ERROR WriteRule::writeTo(const TransportedData& data, GBDATA *gb_item, bool acceptLossyConversion) const {
        if (!gb_item) return "lacking item to writeTo";

        // @@@ overwrite existing target field? should it be allowed or denied? optional?
        // @@@ try GBT_searchOrCreate_itemfield_according_to_changekey to create a field
        xf_assert(data.exists());

        GB_TYPES usedTargetType = forcesType() ? getTargetType() : data.getType();

        GB_ERROR error = check_hkey();
        if (!error) {
            GBDATA *gb_field = GB_search(gb_item, name.c_str(), usedTargetType); // Note: works with hierarchical keys
            if (!gb_field) {
                error = GB_await_error(); // field not created -> report why
            }
            else {
                if (data.getType() == usedTargetType) { // data and target have same type -> no conversion needed
                    error = unconvertedWrite(data, gb_field);
                }
                else { // type differs -> perform conversion (act like field converter reachable from info-window)
                    error = convertAndWrite(data, gb_field, usedTargetType, acceptLossyConversion);
                }
            }
        }
        return error;
    }

    GB_ERROR Rule::transferBy(GBDATA *gb_source, GBDATA *gb_dest) const {
        // @@@ detect target field type. has to be done before starting transfer (do only once for each rule!)
        // @@@ pass target field type to reader (to select best read method)!

        GB_ERROR        error = NULp;
        TransportedData tdata = readFrom(gb_source, gb_dest);
        if (tdata.failed()) {
            error = tdata.getError();
        }
        else if (tdata.exists()) {
            error = writeTo(tdata, gb_dest, precisionLossPermitted());
        }
        // else source missing -> do nothing.
        // Note: if target field exists and source field is missing -> target field remains intact.

        xf_assert(!GB_have_error()); // invalid to export an error (should get returned)
        return error;
    }

    GB_ERROR RuleSet::transferBy(GBDATA *gb_source, GBDATA *gb_dest) const {
        GB_ERROR error = NULp;
        size_t r;
        for (r = 0; r<size() && !error; ++r) {
            const Rule& rule = get(r);
            error            = rule.transferBy(gb_source, gb_dest);
        }
        if (error) {
            error = GBS_global_string("%s (in rule #%zu)", error, r);
        }

        xf_assert(!GB_have_error()); // invalid to export an error (should get returned)
        return error;
    }

    GB_ERROR RuleSet::saveTo(const char *filename) const {
        GB_ERROR  error = NULp;
        FILE     *out   = fopen(filename, "wt");
        if (!out) {
            error = GB_IO_error("saving", filename);
        }
        else {
            // print header:
            fputs("# arb field transfer set; version 1.0\n", out);
            fputc('\n', out);

            // print global RuleSet data:
            {
                ConstStrArray clines;
                GBT_split_string(clines, comment.c_str(), '\n');

                for (int c = 0; clines[c]; ++c) {
                    fprintf(out, "desc:%s\n", clines[c]);
                }
                fputc('\n', out);
            }
            fprintf(out, "transferUndef:%i\n", int(transferUndefFields));

            // print rules:
            for (size_t r = 0; r<size(); ++r) {
                const Rule& rule = get(r);
                string      cfg  = rule.getConfig();
                fprintf(out, "rule:%s\n", cfg.c_str());
            }
            fputc('\n', out);

            fclose(out);
        }
        return error;
    }

    inline bool isCommentLine(const string& line) {
        size_t leadingSpaces        = line.find_first_not_of(" \t");
        return line[leadingSpaces] == '#';
    }
    inline bool shallIgnore(const string& line) {
        // decide whether to ignore a line loaded from .fts file.
        return line.empty() || isCommentLine(line);
    }

    ErrorOrRuleSetPtr RuleSet::loadFrom(const char *filename) {
        ARB_ERROR  error;
        RuleSetPtr ruleset;

        FILE *in = fopen(filename, "rt");
        if (!in) {
            error = GB_IO_error("loading", filename);
        }
        else {
            ruleset = new RuleSet();
            BufferedFileReader reader(filename, in);

            string line;
            while (!error && reader.getLine(line)) {
                if (shallIgnore(line)) continue;

                size_t pos = line.find(':');
                if (pos == string::npos) {
                    error = GBS_global_string("expected ':' while parsing line '%s'", line.c_str());
                }
                else {
                    string tag     = line.substr(0, pos);
                    string content = line.substr(pos+1);

                    if (tag == "rule") {
                        ErrorOrRulePtr rule = Rule::makeFromConfig(content.c_str());
                        if (rule.hasError()) {
                            error = GBS_global_string("while reading rule from '%s': %s",
                                                      content.c_str(),
                                                      rule.getError().deliver());
                        }
                        else {
                            ruleset->add(rule.getValue());
                        }
                    }
                    else if (tag == "desc") {
                        const string& existing = ruleset->getComment();
                        ruleset->setComment(existing.empty() ? content : existing+'\n'+content);
                    }
                    else if (tag == "transferUndef") {
                        ruleset->set_transferUndefFields(bool(atoi(content.c_str())));
                    }
                    else {
                        error = GBS_global_string("unknown tag '%s' while parsing line '%s'",
                                                  tag.c_str(),
                                                  line.c_str());
                    }
                }
            }

            if (error) ruleset.setNull();
        }

        return ErrorOrRuleSetPtr(error, ruleset);
    }

    // --------------------------------
    //      configuration of rules

#define SOURCE "source"
#define ACI    "aci"
#define TARGET "target"
#define SEP    "sep"
#define TYPE   "type"
#define LOSS   "loss"

#define PERMITTED "permitted"

    inline const char *type2str(GB_TYPES type) {
        const char *str = NULp;
        switch (type) {
            case GB_STRING: str = "text";  break;
            case GB_INT:    str = "int";   break;
            case GB_FLOAT:  str = "float"; break;
            case GB_BITS:   str = "bits";  break;
            case GB_NONE:   str = "auto";  break;
            default: break;
        }
        return str;
    }
    inline GB_TYPES str2type(const char *str) {
        GB_TYPES type = GB_TYPE_MAX; // invalid
        switch (str[0]) {
            case 't': if (strcmp(str, "text")  == 0) type = GB_STRING; break;
            case 'i': if (strcmp(str, "int")   == 0) type = GB_INT;    break;
            case 'f': if (strcmp(str, "float") == 0) type = GB_FLOAT;  break;
            case 'b': if (strcmp(str, "bits")  == 0) type = GB_BITS;   break;
            case 'a': if (strcmp(str, "auto")  == 0) type = GB_NONE;   break;
        }
        return type;
    }

    void ReadRule::saveReadConfig(ConfigMapping& cfgmap) const {
        cfgmap.set_entry(SOURCE, fields);
        if (separator != NOSEP) cfgmap.set_entry(SEP, separator);
        if (!aci.empty()) cfgmap.set_entry(ACI, aci);
    }

    void WriteRule::saveWriteConfig(ConfigMapping& cfgmap) const {
        cfgmap.set_entry(TARGET, name);
        if (forcesType()) {
            cfgmap.set_entry(TYPE, type2str(getTargetType()));
        }
    }
    string Rule::getConfig() const {
        ConfigMapping cfgmap;

        saveReadConfig(cfgmap);
        saveWriteConfig(cfgmap);

        if (precisionLossPermitted()) {
            cfgmap.set_entry(LOSS, PERMITTED);
        }

        return cfgmap.config_string();
    }

    ErrorOrRulePtr Rule::makeFromConfig(const char *config) {
        RulePtr       rule;
        ConfigMapping cfgmap;
        GB_ERROR      error = cfgmap.parseFrom(config);

        if (!error) {
            const char *source = cfgmap.get_entry(SOURCE);
            const char *target = cfgmap.get_entry(TARGET);
            const char *sep    = cfgmap.get_entry(SEP);

            if (!source) error = "missing " SOURCE " entry";
            if (!target) error = "missing " TARGET " entry";

            if (!sep) sep = NOSEP; // default to 'no separator'

            if (!error) {
                const char *aci = cfgmap.get_entry(ACI);
                if (aci) {
                    rule = makeAciConverter(source, sep, aci, target);
                }
                else {
                    rule = makeSimple(source, sep, target);
                }

                const char *typeID = cfgmap.get_entry(TYPE);
                if (typeID) {
                    GB_TYPES type = str2type(typeID);
                    if (type == GB_TYPE_MAX) { // = unknown type ID
                        error = GBS_global_string("invalid type id '%s'", typeID);
                        rule.setNull();
                    }
                    else {
                        xf_assert(GB_TYPE_readable_as_string(type));
                        rule->setTargetType(type);
                    }
                }

                if (!error) {
                    const char *loss = cfgmap.get_entry(LOSS);
                    if (loss && strcmp(loss, PERMITTED) == 0) {
                        rule->permitPrecisionLoss();
                    }
                }
            }
        }

        return ErrorOrRulePtr(error, rule);
    }


    // ------------------------
    //      describe rules
    string ReadRule::describe() const {
        if (aci.empty()) return fields;
        return fields+"|ACI";
    }
    string WriteRule::describe() const {
        return name;
    }
    string Rule::getShortDescription() const {
        return ReadRule::describe() + " -> " + WriteRule::describe();
    }

    // -----------------------------
    //      ItemClonedByRuleSet
    string ItemClonedByRuleSet::lastReportedError;

    GB_ERROR ItemClonedByRuleSet::overlayOrCloneSub(const char *subName, GBDATA *gb_sub) {
        GBDATA   *gb_existing = GB_entry(gb_clone, subName);
        GB_ERROR  error;
        if (gb_existing) {                                // if target entry exists ..
            error = GB_copy_overlay(gb_existing, gb_sub); // .. overwrite its content.
        }
        else {                                                      // otherwise ..
            error = GB_incur_error_if(!GB_clone(gb_clone, gb_sub)); // .. clone source entry
        }
        return error;
    }

    GB_ERROR ItemClonedByRuleSet::cloneMissingSub(const char *subName, GBDATA *gb_sub) {
        GBDATA   *gb_existing = GB_entry(gb_clone, subName);
        GB_ERROR  error;
        if (gb_existing) { // if target entry exists ..
            error = NULp;  // .. keep it
        }
        else {                                                      // otherwise ..
            error = GB_incur_error_if(!GB_clone(gb_clone, gb_sub)); // .. clone source entry
        }
        return error;
    }

    GB_ERROR ItemClonedByRuleSet::copySubIfMissing(const char *subName) {
        // copy sub-field (or -container) if it doesn't exist in target
        GB_ERROR  error  = NULp;
        GBDATA   *gb_sub = GB_entry(gb_source, subName);
        if (!gb_sub) {
            error = GBS_global_string("no such entry '%s' (in source)", subName);
            UNCOVERED();
        }
        else {
            error = cloneMissingSub(subName, gb_sub); // sub = passed field or container
        }
        return error;
    }

    GB_ERROR ItemClonedByRuleSet::copyAlignments() {
        GB_ERROR error = NULp;
        for (GBDATA *gb_ali = GB_child(gb_source); gb_ali; gb_ali = GB_nextChild(gb_ali)) {
            if (GB_is_container(gb_ali)) {
                const char *aliname = GB_read_key_pntr(gb_ali);
                if (ARB_strBeginsWith(aliname, "ali_")) {
                    GBDATA *gb_data = GB_entry(gb_ali, "data");
                    if (gb_data) {
                        bool dataIsSTRING = GB_read_type(gb_data) == GB_STRING;
                        xf_assert(dataIsSTRING);
                        if (dataIsSTRING) {
                            error = overlayOrCloneSub(aliname, gb_ali); // sub = whole alignment container
                        }
                    }
                    else {
                        error = GB_incur_error();
                        UNCOVERED();
                    }
                }
            }
        }
        return error;
    }

    const char *ItemClonedByRuleSet::get_id_field() const {
        const char *field = NULp;
        switch (itemtype) {
            case CLONE_ITEM_SPECIES: field = "name"; break;
            default: xf_assert(0); break;
        }
        return field;
    }

    ItemClonedByRuleSet::ItemClonedByRuleSet(GBDATA*& gb_item, ClonableItemType itemtype_, RuleSetPtr ruleset, ItemCloneType type_, GBDATA *gb_refItem, const AlignmentTransporter *aliTransporter) :
        itemtype(itemtype_),
        gb_source(gb_item),
        type(type_)
    {
        // @@@ method is far too long -> split

        GB_ERROR       error = NULp;
        GB_transaction ta(gb_source);

#if defined(ASSERTION_USED)
        checked4error    = false;
        userCallbackUsed = false;
#endif

        if (type == CLONE_INTO_EXISTING) {
            if (gb_refItem) {
                gb_clone = gb_refItem; // use passed clone as target
            }
            else {
                error = "no target species specified (logic error)";
                UNCOVERED();
            }
        }
        else {
            GBDATA *gb_item_container;
            {
                GBDATA *gb_src_item_container = GB_get_father(gb_source);
                if (type == REAL_CLONE) {
                    gb_item_container = gb_refItem;
                    if (!gb_item_container) {
                        error = "no target item container specified (logic error)";
                    }
                    else if (gb_item_container == gb_src_item_container) {
                        error = "source and target item containers need to differ (logic error)";
                    }
                }
                else {
                    xf_assert(!gb_refItem); // passed value is ignored (please pass NULp)
                    gb_item_container = gb_src_item_container;
                }
            }

            if (!error) {
                xf_assert(itemtype_ == CLONE_ITEM_SPECIES);                   // next command only works for species
                gb_clone = GB_create_container(gb_item_container, "species"); // create separate species
                if (!gb_clone) {
                    error = GB_await_error();
                    UNCOVERED();
                }
            }
        }

        if (!error) {
            // apply ruleset:
            error = ruleset->transferBy(gb_source, gb_clone);

            // perform some standard transfers:
            const char *IDFIELD = get_id_field();
            if (!error) error   = copySubIfMissing(IDFIELD); // transfer IDFIELD for any itemtype

            switch (itemtype) {
                case CLONE_ITEM_SPECIES:
                    if (!error) error = copySubIfMissing("acc");
                    if (!error) {
                        if (aliTransporter) { // use user callback if given
                            if (aliTransporter->shallCopyBefore()) {
                                error = copyAlignments();
                            }
                            if (!error) {
                                error = aliTransporter->transport(gb_source, gb_clone); // e.g. used to adapt alignment in mergetool
                            }
#if defined(ASSERTION_USED)
                            userCallbackUsed = true;
#endif
                        }
                        else {
                            error = copyAlignments();
                        }
                    }
                    break;
                default: xf_assert(0); break;
            }

            if (!error && ruleset->shallTransferUndefFields()) {

                StrSet defined;
                // extract used fields:
                {
                    StrArray in, out;
                    ruleset->extractUsedFields(in, out);
                    // @@@ do extraction only once (not for each item transfer)

                    StrArray2StrSet(in, defined);
                    StrArray2StrSet(out, defined);
                }
                {
                    // exclude parent containers:
                    StrSet parents;
                    for (StrSet::const_iterator field = defined.begin(); field != defined.end(); ++field) {
                        size_t slashpos = field->find_first_of('/');
                        if (slashpos != string::npos) { // fieldname contains a slash
                            string parentname = field->substr(0, slashpos); // name of top-level parent container inside species
                            parents.insert(parentname);
                        }
                    }
                    defined.insert(parents.begin(), parents.end());
                }

                // transfer rest of fields (i.e. those neighter used by ruleset nor as standard field):
                for (GBDATA *gb_field = GB_child(gb_source); gb_field && !error;  gb_field = GB_nextChild(gb_field)) {
                    const char *key     = GB_read_key_pntr(gb_field);
                    bool        keyUsed = defined.find(key) != defined.end(); // key was read or written by ruleset

                    if (!keyUsed) {
                        error = copySubIfMissing(key);
                    }
                }
            }

            // @@@ do we need to preserve security etc of cloned species? (security of sub-fields is preserved; e.g. see r17967)

            if (!error) {
                xf_assert(correlated(aliTransporter, userCallbackUsed)); // custom transporter was not used (logic error?)

                switch (type) {
                    case REPLACE_ITEM_BY_CLONE:
                        error = GB_delete(gb_source);   // will be replaced by clone
                        if (!error) {
                            gb_item   = NULp;
                            gb_source = NULp;
                        }
                        break;

                    case RENAME_ITEM_WHILE_TEMP_CLONE_EXISTS: {
                        GBDATA *gb_id = GB_entry(gb_source, IDFIELD);
                        if (!gb_id) {
                            error = GBS_global_string("expected field '%s' not found", IDFIELD);
                        }
                        else {
                            const char *name = GB_read_char_pntr(gb_id);
                            xf_assert(name);
                            orgName = name; // store name (written back in dtor)

                            error = GB_write_string(gb_id, "fake"); // change name // @@@ use different name
                        }
                        break;
                    }
                    case CLONE_INTO_EXISTING:
                    case REAL_CLONE:
                        // nothing to do here
                        break;
                }
            }
        }

        error = ta.close(error);
        if (error) {
            errorCopy = error; // store copy of error in string
            gb_clone  = NULp;
        }
    }

    ItemClonedByRuleSet::~ItemClonedByRuleSet() {
        if (!has_error()) { // if error occurred during construction -> TA was aborted -> nothing to undo
            if (type == RENAME_ITEM_WHILE_TEMP_CLONE_EXISTS) {
                GB_transaction ta(gb_source);
                GB_ERROR       error = NULp;

                GBDATA *gb_id = GB_entry(gb_source, get_id_field());
                xf_assert(gb_id);

                if (gb_id) {
                    xf_assert(!orgName.empty());
                    error = GB_write_string(gb_id, orgName.c_str());
                    if (error) {
                        fprintf(stderr, "Failed to rename original item after temp. clone (Reason: %s)", error);
                        xf_assert(0); // should not happen
                    }
                }

                // delete temp clone:
                if (!error) {
                    error = GB_delete(gb_clone);
                    if (error) {
                        fprintf(stderr, "Failed to delete temp. clone (Reason: %s)", error);
                        xf_assert(0); // should not happen
                    }
                }
            }
        }
    }

};

// --------------------------------------------------------------------------------

#ifdef UNIT_TESTS

#include <arb_diff.h>
#include <arb_file.h>
#include <arb_defs.h>

#ifndef TEST_UNIT_H
#include <test_unit.h>
#endif

void TEST_type2id() {
    using namespace FieldTransfer;

    for (GB_TYPES t = GB_NONE; t<=GB_TYPE_MAX; t = GB_TYPES(t+1)) {
        const char *id = type2str(t);
        if (id) {
            TEST_ANNOTATE(id);
            TEST_EXPECT_EQUAL(str2type(id), t);
        }
    }
}
void TEST_transportedData() {
    using namespace FieldTransfer;

    GB_ERROR error;
    {
        TransportedData noData    = TransportedData::none();
        TransportedData errorData = TransportedData::makeError("the error msg");

        TEST_REJECT(noData.failed());
        TEST_REJECT(noData.exists());

        TEST_EXPECT(errorData.failed());
        error = errorData.getError();
        TEST_EXPECT_EQUAL(error, "the error msg");
    }
    TEST_EXPECT_EQUAL(error, "the error msg"); // error has to survive destruction of TransportedData

    TransportedData greet("hello");
    TransportedData num(4711);
    TransportedData fnum(0.815f);

    TEST_REJECT(greet.failed());
    TEST_EXPECT(greet.exists());
    TEST_EXPECT_EQUAL(greet.getString(), "hello");

    TEST_REJECT(num.failed());
    TEST_EXPECT(num.exists());
    TEST_EXPECT_EQUAL(num.getInt(), 4711);

    TEST_REJECT(fnum.failed());
    TEST_EXPECT(fnum.exists());
    TEST_EXPECT_SIMILAR(fnum.getFloat(), 0.815, 0.000001);
}

void TEST_xferset() {
    FieldTransfer::RuleSet fts;

    TEST_EXPECT_ZERO(fts.size());
    TEST_EXPECT(fts.empty());
    TEST_EXPECT_EQUAL(fts.size(), 0);

    // --------------------
    //      add rules:
    fts.add(new FieldTransfer::Rule(FieldTransfer::ReadRule("location", NOSEP), // add a simple rule (one source, no ACI)
                                    FieldTransfer::WriteRule("geolocation")));
    TEST_EXPECT(!fts.empty());
    TEST_EXPECT_EQUAL(fts.size(), 1);

    fts.add(FieldTransfer::Rule::permitPrecisionLoss(new FieldTransfer::Rule(FieldTransfer::ReadRule("isolation", NOSEP, "upper"), // add an ACI rule (one source)
                                                                             FieldTransfer::WriteRule("isolation_source", GB_INT)))); // force int type
    TEST_EXPECT_EQUAL(fts.size(), 2);

    // @@@ add multisource rules (with and w/o ACI)!

    // ---------------------
    //      query rules
    for (size_t r = 0; r<fts.size(); ++r) {
        TEST_ANNOTATE(GBS_global_string("r=%zu", r));

        const FieldTransfer::Rule& rule = fts.get(r);
        switch (r) {
            // @@@ add tests for source field(s)

            case 0: // simple rule

                TEST_EXPECT_EQUAL(rule.targetField(), "geolocation");
                TEST_REJECT(rule.forcesType());
                TEST_REJECT(rule.precisionLossPermitted());
                break;

            case 1: // basic ACI rule

                TEST_EXPECT_EQUAL(rule.targetField(), "isolation_source");
                TEST_EXPECT(rule.forcesType());                  // type is forced ..
                TEST_EXPECT_EQUAL(rule.getTargetType(), GB_INT); // .. to int
                TEST_EXPECT(rule.precisionLossPermitted());
                break;

            default:
                xf_assert(0); // untested rule
                break;
        }
    }
    TEST_ANNOTATE(NULp);
    TEST_EXPECT_EQUAL(fts.size(), 2);

    // -------------------------------------------
    //      test rule replacement and removal
    {
        // order=01
        string cfg0 = fts.get(0).getConfig();
        string cfg1 = fts.get(1).getConfig();

        TEST_EXPECT_DIFFERENT(cfg0, cfg1); // otherwise test below does not test replacement

        {
            // swap 2 rules
            FieldTransfer::RulePtr tmp = fts.getPtr(0);
            fts.replace(0, fts.getPtr(1));
            fts.replace(1, tmp);
            // order=10
        }

        string newcfg0 = fts.get(0).getConfig();
        string newcfg1 = fts.get(1).getConfig();

        TEST_EXPECT_EQUAL(newcfg0, cfg1);
        TEST_EXPECT_EQUAL(newcfg1, cfg0);

        {
            int insertedAt;

            insertedAt = fts.insertBefore(0, fts.getPtr(1)); // insert before first -> order = 010
            TEST_EXPECT_EQUAL(fts.size(), 3);
            TEST_EXPECT_EQUAL(insertedAt, 0);
            TEST_EXPECT_EQUAL(fts.get(insertedAt).getConfig(), cfg0);

            insertedAt = fts.insertBefore(2, fts.getPtr(1)); // insert before last -> order = 0110
            TEST_EXPECT_EQUAL(fts.size(), 4);
            TEST_EXPECT_EQUAL(insertedAt, 2);
            TEST_EXPECT_EQUAL(fts.get(insertedAt).getConfig(), cfg1);

            insertedAt = fts.insertBefore(7, fts.getPtr(1)); // insert before invalid position = append -> order = 01101
            TEST_EXPECT_EQUAL(fts.size(), 5);
            TEST_EXPECT_EQUAL(insertedAt, 4);
            TEST_EXPECT_EQUAL(fts.get(insertedAt).getConfig(), cfg1);

            // "undo" inserts
            fts.erase(1); // -> order = 0101
            fts.erase(3); // erase at end -> order = 010
            fts.erase(0); // -> order = 10
        }

        fts.erase(0); // erase 1st rule -> order = 0
        TEST_EXPECT_EQUAL(fts.size(), 1);
        string finalcfg = fts.get(0).getConfig();
        TEST_EXPECT_EQUAL(finalcfg, cfg0);
    }
}

class FailingRule: public FieldTransfer::Rule {
    string partOfFailReason;
public:
    FailingRule(const Rule& failing, string part) : Rule(failing), partOfFailReason(part) {}
    FailingRule(FieldTransfer::RulePtr failing, string part) : Rule(*failing), partOfFailReason(part) {}
    const char *expectedPartOfFailure() const { return partOfFailReason.c_str(); }
};


struct XferEnv : virtual Noncopyable { // provides test environment for transfer tests
    GB_shell shell;

    const char *target_ascii;

    GBDATA *gb_src;
    GBDATA *gb_dest;

    XferEnv() :
        target_ascii("TEST_fields_xferred.arb")
    {
        gb_src  = GB_open("TEST_fields_ascii.arb", "r"); // ../../UNIT_TESTER/run/TEST_fields_ascii.arb
        gb_dest = GB_open(target_ascii, "wc");
    }
    ~XferEnv() {
        TEST_EXPECT_ZERO_OR_SHOW_ERRNO(GB_unlink(target_ascii));

        GB_close(gb_dest);
        GB_close(gb_src);
    }

    void transferAllSpeciesBy(const FieldTransfer::RuleSet& ruleset) { // transfer all species according to Ruleset
        // @@@ transferAllSpeciesBy is quite similar to what has to happen in merge-tool
        GB_transaction tas(gb_src);
        GB_transaction tad(gb_dest);

        for (GBDATA *gb_src_species = GBT_first_species(gb_src);
             gb_src_species;
             gb_src_species = GBT_next_species(gb_src_species))
        {
            const char *name = GBT_get_name(gb_src_species);
            TEST_REJECT_NULL(name);

            GBDATA *gb_dest_species = GBT_find_or_create_species(gb_dest, name, false);
            TEST_REJECT_NULL(gb_dest_species);

            // @@@ transferAllSpeciesBy could allow overwrites (keep existing fields; allow overwrite of fields):
            //     -> try to use ItemClonedByRuleSet with CLONE_INTO_EXISTING instead of direct call to transferBy below!

            TEST_EXPECT_ZERO(GB_read_flag(gb_dest_species)); // (was previously done by GBT_find_or_create_species)
            TEST_EXPECT_NO_ERROR(ruleset.transferBy(gb_src_species, gb_dest_species));
        }
    }

    // ----------------------------------------------------------------------------

    void copyAllSpecies() {
        GB_transaction tas(gb_src);
        GB_transaction tad(gb_dest);

        GBDATA *gb_dest_species_data = GBT_get_species_data(gb_dest);
        TEST_REJECT_NULL(gb_dest_species_data);

        for (GBDATA *gb_src_species = GBT_first_species(gb_src);
             gb_src_species;
             gb_src_species = GBT_next_species(gb_src_species))
        {
            const char *name = GBT_get_name(gb_src_species);
            TEST_REJECT_NULL(name);

            GBDATA *gb_dest_exists = GBT_find_species(gb_dest, name);
            TEST_EXPECT_NULL(gb_dest_exists); // this method cannot handle overwrites

            GBDATA *gb_dest_species = GB_create_container(gb_dest_species_data, "species");
            TEST_REJECT_NULL(gb_dest_species);

            TEST_EXPECT_NO_ERROR(GB_copy_dropProtectMarksAndTempstate(gb_dest_species, gb_src_species));
            TEST_EXPECT_ZERO(GB_read_flag(gb_dest_species));
        }
    }

    // ----------------------------------------------------------------------------
    //      write transferred data to ascii db + compare with expected result:
    void save() {
        TEST_EXPECT_NO_ERROR(GB_save_as(gb_dest, target_ascii, "a"));
    }
    void saveAndCompare(const char *expected_ascii, bool allowAutoUpdate) {
        save();
        if (allowAutoUpdate) {
// #define TEST_AUTO_UPDATE // uncomment to update expected result
#if defined(TEST_AUTO_UPDATE)
            TEST_COPY_FILE(target_ascii, expected_ascii);
#endif
        }
        TEST_EXPECT_TEXTFILE_DIFFLINES(target_ascii, expected_ascii, 0);
    }
};

static const char *expRuleConfig[] = {
    "source='lat_lon';target='geolocation'",
    "source='seq_quality_slv';target='seq/slv_quality'",
    "source='homop_slv';target='slv_homop'",

    "source='no1';target='notTransferred'",

    "source='pubmed_id';target='str2int';type='int'",
    "source='pubmed_id';target='str2flt';type='float'",
    "source='stop';target='int2flt';type='float'",
    "source='stop';target='int2str';type='text'",
    "source='align_ident_slv';target='flt2str';type='text'",
    "loss='permitted';source='align_ident_slv';target='flt2int';type='int'",

    "aci='|lower|contains(partial)|isAbove(0)';source='description';target='describedAsPartial'",

    "aci='|fdiv(2.0)';source='align_bp_score_slv';target='halfBPscoreStr'",
    "aci='|fdiv(2.0)';source='align_bp_score_slv';target='halfBPscore';type='int'",
    "aci='|fdiv(2.0)';source='align_bp_score_slv';target='halfBPscoreFlt';type='float'",

    "aci='|fmult(3.5)';source='homop_slv';target='multiHomopStr'",
    "aci='|fmult(3.5)';source='homop_slv';target='multiHomopInt';type='int'",
    "aci='|fmult(3.5)';source='homop_slv';target='multiHomop';type='float'",

    "sep='/';source='embl_class;embl_division';target='embl_class_division'",
    "sep='-';source='align_startpos_slv;align_stoppos_slv';target='align_range_slv'",
    "sep='\\'';source='no1;align_bp_score_slv;no2;rel_ltp;no3';target='missing'",

    "sep=';';source='NO1;no2;no3';target='skipped'",

    "aci='|upper';sep=':';source='embl_class;embl_division';target='emblClassDivision'",
    "aci='|\"<\";dd;\">\"';sep=';';source='no1;no2;no3';target='skipped2'",
};

static const char *EXPECTED_ASCII        = "TEST_fields_xferred_expected.arb"; // ../../UNIT_TESTER/run/TEST_fields_xferred_expected.arb
static const char *EXPECTED_ASCII_CLONED = "TEST_fields_cloned_expected.arb"; // ../../UNIT_TESTER/run/TEST_fields_cloned_expected.arb

void TEST_xferBySet() {
    // tests data transfer between items using RuleSet|s
    using namespace FieldTransfer;
    XferEnv env;

    // --------------------------------------------------------------
    //      create rules and transfer item data using RuleSet|s:

    typedef std::vector<FailingRule> FailingRuleCont;

    RuleSet         ruleset;
    FailingRuleCont failing; // failing rules should go here

#define FAILING_add(rule,msg) failing.push_back(FailingRule(rule, msg))

    ruleset.add(Rule::makeSimple("lat_lon",         NOSEP, "geolocation"));     // STRING->STRING
    ruleset.add(Rule::makeSimple("seq_quality_slv", NOSEP, "seq/slv_quality")); // INT   ->INT     (generate hierarchical target key)
    ruleset.add(Rule::makeSimple("homop_slv",       NOSEP, "slv_homop"));       // FLOAT ->FLOAT

    ruleset.add(Rule::makeSimple("no1", NOSEP, "notTransferred")); // missing fields are skipped

    // force target types
    ruleset.add(Rule::forceTargetType(GB_INT,    Rule::makeSimple("pubmed_id",       NOSEP, "str2int"))); // STRING->INT
    ruleset.add(Rule::forceTargetType(GB_FLOAT,  Rule::makeSimple("pubmed_id",       NOSEP, "str2flt"))); // STRING->FLOAT
    ruleset.add(Rule::forceTargetType(GB_FLOAT,  Rule::makeSimple("stop",            NOSEP, "int2flt"))); // INT->FLOAT
    ruleset.add(Rule::forceTargetType(GB_STRING, Rule::makeSimple("stop",            NOSEP, "int2str"))); // INT->STRING
    ruleset.add(Rule::forceTargetType(GB_STRING, Rule::makeSimple("align_ident_slv", NOSEP, "flt2str"))); // FLOAT->STRING
    FAILING_add(Rule::forceTargetType(GB_INT,    Rule::makeSimple("align_ident_slv", NOSEP, "dummy")), "lossy float->int type conversion (9.484605e+01->9.500000e+01)"); // FLOAT->INT
    ruleset.add(Rule::permitPrecisionLoss(Rule::forceTargetType(GB_INT, Rule::makeSimple("align_ident_slv", NOSEP, "flt2int"))));    // FLOAT->INT
    // @@@ test forcedTargetType(GB_BITS)

    // @@@ test transfer with existing target keys (and mismatching i.e. not default types)
    // -> shall force type conversion (e.g. int->float, float->string, string->int)

    ruleset.add(Rule::makeAciConverter("description", NOSEP, "|lower|contains(partial)|isAbove(0)", "describedAsPartial")); // transports STRING through ACI to STRING

    // INT | ACI -> STRING|INT|FLOAT:
    ruleset.add(Rule::makeAciConverter("align_bp_score_slv", NOSEP, "|fdiv(2.0)", "halfBPscoreStr"));                                  // transports INT through ACI to STRING
    ruleset.add(Rule::forceTargetType(GB_INT, Rule::makeAciConverter("align_bp_score_slv", NOSEP, "|fdiv(2.0)", "halfBPscore")));      // transports INT through ACI to INT // @@@ why does this not complain about conversion loss? (e.g. PurGergo 58.5 -> 58). examine!!!
    ruleset.add(Rule::forceTargetType(GB_FLOAT, Rule::makeAciConverter("align_bp_score_slv", NOSEP, "|fdiv(2.0)", "halfBPscoreFlt"))); // transports INT through ACI to FLOAT

    // FLOAT | ACI -> STRING:
    ruleset.add(Rule::makeAciConverter("homop_slv", NOSEP, "|fmult(3.5)", "multiHomopStr"));                                // transports FLOAT through ACI to STRING
    ruleset.add(Rule::forceTargetType(GB_INT, Rule::makeAciConverter("homop_slv", NOSEP, "|fmult(3.5)", "multiHomopInt"))); // transports FLOAT through ACI to INT // @@@ conversion loss happens (for all species?) // @@@ examine!
    ruleset.add(Rule::forceTargetType(GB_FLOAT, Rule::makeAciConverter("homop_slv", NOSEP, "|fmult(3.5)", "multiHomop")));  // transports FLOAT through ACI to FLOAT

    // @@@ test ACIs containing the following chars: ['"\\ = ]

    // test concatenating rules:
    ruleset.add(Rule::makeSimple("embl_class;embl_division",               "/", "embl_class_division")); // concat 2 STRINGs
    ruleset.add(Rule::makeSimple("align_startpos_slv;align_stoppos_slv",   "-", "align_range_slv"));     // concat 2 INTs
    ruleset.add(Rule::makeSimple("no1;align_bp_score_slv;no2;rel_ltp;no3", "'", "missing"));             // concat INT + STRING (plus 3 non-existing fields)

    ruleset.add(Rule::makeSimple("NO1;no2;no3", ";", "skipped")); // concat 3 non-existing fields -> field 'skipped' is NOT written to result DB

    // test concatenation + ACI:
    ruleset.add(Rule::makeAciConverter("embl_class;embl_division", ":", "|upper",          "emblClassDivision")); // concat 2 STRINGs
    ruleset.add(Rule::makeAciConverter("no1;no2;no3",              ";", "|\"<\";dd;\">\"", "skipped2"));          // concat 3 non-existing fields and apply ACI -> field 'skipped2' is NOT written to result DB + ACI not applied

    // ----------------------------------------------------------------------------
    // please do not change 'ruleset' below this point
    // ----------------------------------------------------------------------------

    // test input/output field extraction
    {
        StrArray input;
        StrArray output;

        ruleset.extractUsedFields(input, output);

        TEST_EXPECT_STRARRAY_CONTAINS(input,  ';', "align_bp_score_slv;align_ident_slv;align_startpos_slv;align_stoppos_slv;description;embl_class;embl_division;homop_slv;lat_lon;no1;no2;no3;pubmed_id;rel_ltp;seq_quality_slv;stop");
        TEST_EXPECT_STRARRAY_CONTAINS(output, ';', "align_range_slv;describedAsPartial;emblClassDivision;embl_class_division;flt2int;flt2str;geolocation;halfBPscore;halfBPscoreFlt;halfBPscoreStr;int2flt;int2str;missing;multiHomop;multiHomopInt;multiHomopStr;notTransferred;seq/slv_quality;skipped;skipped2;slv_homop;str2flt;str2int");
    }

    // convert all rules in 'ruleset' into string and test versus expRuleConfig:
    const size_t cfgs = ARRAY_ELEMS(expRuleConfig);
    const size_t rulz = ruleset.size();
    {
        const size_t testableRepr = min(cfgs, rulz);
        for (size_t r = 0; r<testableRepr; ++r) {
            TEST_ANNOTATE(GBS_global_string("r=%zu", r));
            const Rule& rule = ruleset.get(r);
            string      rep  = rule.getConfig();
            TEST_EXPECT_EQUAL(expRuleConfig[r], rep.c_str());
        }
    }

    TEST_EXPECT_EQUAL(cfgs, rulz);

    // test no 2 rules have equal config:
    for (size_t r1 = 0; r1<rulz; ++r1) {
        for (size_t r2 = r1+1; r2<rulz; ++r2) {
            TEST_ANNOTATE(GBS_global_string("r1/r2=%zu/%zu", r1, r2));
            TEST_EXPECT_DIFFERENT(expRuleConfig[r1], expRuleConfig[r2]);
        }
    }
    TEST_ANNOTATE(NULp);

    env.transferAllSpeciesBy(ruleset);

    // -------------------------------------------
    //      test missing source-/target-item:
    {
        GB_transaction tas(env.gb_src);
        GB_transaction tad(env.gb_dest);

        GBDATA *gb_src_species = GBT_first_species(env.gb_src);
        TEST_REJECT_NULL(gb_src_species);

        const char *name = GBT_get_name(gb_src_species);
        TEST_REJECT_NULL(name);

        GBDATA *gb_dest_species = GBT_find_species(env.gb_dest, name);     // already has been created by 'transferAllSpeciesBy' above
        TEST_REJECT_NULL(gb_dest_species);

        TEST_EXPECT_ERROR_CONTAINS(ruleset.transferBy(NULp,           gb_dest_species), "lacking item to readFrom");
        TEST_EXPECT_ERROR_CONTAINS(ruleset.transferBy(gb_src_species, NULp),            "lacking item to writeTo");
    }

    // ---------------------------------------------
    //      test rules failing during transfer:
    FAILING_add(Rule::forceTargetType(GB_FLOAT, Rule::makeSimple("nuc_region", NOSEP, "str2flt")), "cannot convert '1..1494' to float"); // test conversion errors (e.g. non-numeric string -> int or float)
    FAILING_add(Rule::makeAciConverter("homop_slv", NOSEP, "|fmult(3.5, ooo)", "dummy"), "Unknown command '3.5'");
    FAILING_add(Rule::makeSimple("stop", NOSEP, "xx*xx"), "Invalid character '*' in 'xx*xx'");
    FAILING_add(Rule::makeSimple("ali_16s", NOSEP, "whatever"), "cannot read as data ('ali_16s' is a container)");

    for (FailingRuleCont::const_iterator failRule = failing.begin(); failRule != failing.end(); ++failRule) {
        const FailingRule& testableRule = *failRule;
        RuleSet            separated;
        separated.add(new Rule(testableRule));

        // apply rule:
        {
            GB_transaction tas(env.gb_src);
            GB_transaction tad(env.gb_dest);

            GB_ERROR error = NULp;

            for (GBDATA *gb_src_species = GBT_first_species(env.gb_src);
                 gb_src_species && !error;
                 gb_src_species = GBT_next_species(gb_src_species))
            {
                const char *name = GBT_get_name(gb_src_species);
                if (!name) {
                    error = "cannot search for unnamed species";
                }
                else {
                    GBDATA *gb_dest_species = GBT_find_species(env.gb_dest, name);     // already has been created by 'transferBy' above
                    error = separated.transferBy(gb_src_species, gb_dest_species);
                }
            }
            tad.close(error); // aborts transaction (if error occurs, which is expected here)
            TEST_EXPECT_ERROR_CONTAINS(error, testableRule.expectedPartOfFailure());
        }
    }

    // ----------------------------------------------------------------
    //      test type of each field is same across all items of DB
    {
        GB_transaction tad(env.gb_dest);
        GBDATA *gb_fake_species_data = GB_create_container(env.gb_dest, "tmp"); // necessary because GBT_scan_db never scans DIRECT childs

        typedef map<string,GB_TYPES> TypedField;
        TypedField seen;

        GB_ERROR error = NULp;
        for (GBDATA *gb_dest_species = GBT_first_species(env.gb_dest);
             gb_dest_species && !error;
             gb_dest_species = GBT_next_species(gb_dest_species))
        {
            TEST_ANNOTATE(GBS_global_string("name=%s", GBT_get_name_or_description(gb_dest_species)));

            GBDATA *gb_specCopy = GB_create_container(gb_fake_species_data, "tmp");
            error               = GB_copy_dropProtectMarksAndTempstate(gb_specCopy, gb_dest_species);

            if (error) break;

            StrArray curr;
            GBT_scan_db(curr, gb_fake_species_data, NULp);
            TEST_REJECT_ZERO(curr.size()); // expect fields

            for (int i = 0; curr[i]; ++i) {
                const char *scanned = curr[i]; // 1st char is type
                const char *field   = scanned+1;
                GB_TYPES    type    = GB_TYPES(scanned[0]);

                TypedField::iterator found = seen.find(field);
                if (found != seen.end()) {
                    if (type != found->second) {
                        TEST_ANNOTATE(field);
                        TEST_EXPECT_EQUAL(type, found->second); // existing field has to have same type (in all species)
                    }
                }
                else {
                    fprintf(stderr, "field='%s' type='%i'\n", field, type);
                    seen[field] = type; // insert new field
                }
            }

            if (!error) error = GB_delete(gb_specCopy);
        }
        if (!error) error = GB_delete(gb_fake_species_data);
        TEST_EXPECT_NO_ERROR(error);
    }

    // ----------------------------------------------------------------------------
    xf_assert(rulz == ruleset.size()); // please do not change 'ruleset' after 'rulz' has been set!

    env.saveAndCompare(EXPECTED_ASCII, true);
}

void TEST_LATE_ruleConfigsReadable() {
    // run this test later than TEST_xferBySet

    using namespace FieldTransfer;

    {
        // test failing Rule configs:
        struct InvalidConfig {
            const char *config;
            GB_ERROR    failure;
        };
        InvalidConfig invalidCfg[] = {
            { TARGET "='xxx'",                                 "missing source entry" },
            { SOURCE "='xxx'",                                 "missing target entry" },
            { "tag='halfquot;",                                "could not find matching quote" },
            { TARGET "='xxx';" SOURCE "='xxx';type='bizarre'", "invalid type id 'bizarre'" },
        };

        for (size_t i = 0; i<ARRAY_ELEMS(invalidCfg); ++i) {
            InvalidConfig& CFG = invalidCfg[i];
            TEST_ANNOTATE(GBS_global_string("invalidCfg='%s'", CFG.config));

            ErrorOrRulePtr result = Rule::makeFromConfig(CFG.config);
            TEST_EXPECT(result.hasError());
            TEST_EXPECT_ERROR_CONTAINS(result.getError(), CFG.failure);
        }
        TEST_ANNOTATE(NULp);
    }

    const size_t cfgs = ARRAY_ELEMS(expRuleConfig);
    RuleSet      ruleset;

    // convert config->Rule + Rule->config + compare configs:
    for (size_t r = 0; r<cfgs; ++r) {
        const char *config = expRuleConfig[r];

        ErrorOrRulePtr result = Rule::makeFromConfig(config);
        if (result.hasError()) {
            TEST_EXPECT_NO_ERROR(result.getError());
        }
        else {
            RulePtr rule           = result.getValue();
            string  reloadedConfig = rule->getConfig();
            TEST_EXPECT_EQUAL(reloadedConfig, config);

            ruleset.add(rule);
        }
    }

    // test RuleSet comment:
    const char *COMMENT = "A multi-\nline-\ntest-\ncomment.";
    ruleset.setComment(COMMENT);
    TEST_EXPECT_EQUAL(ruleset.getComment(), COMMENT);

    ruleset.set_transferUndefFields(true);
    TEST_EXPECT(ruleset.shallTransferUndefFields());

    // save RuleSet + reload it + compare:
    RuleSet reloaded_ruleset;
    {
        const char *rulesetSaved    = "impexp/rulesetCurr.fts";
        const char *rulesetExpected = "impexp/ruleset.fts";

        TEST_EXPECT_NO_ERROR(ruleset.saveTo(rulesetSaved));
// #define TEST_AUTO_UPDATE_RS // uncomment to update expected result
#if defined(TEST_AUTO_UPDATE_RS)
        TEST_COPY_FILE(rulesetSaved, rulesetExpected);
#endif
        TEST_EXPECT_TEXTFILE_DIFFLINES(rulesetSaved, rulesetExpected, 0);
        TEST_EXPECT_ZERO_OR_SHOW_ERRNO(GB_unlink(rulesetSaved));

        // reload RuleSet:
        {
            ErrorOrRuleSetPtr loaded = RuleSet::loadFrom(rulesetExpected);
            if (loaded.hasError()) TEST_EXPECT_NO_ERROR(loaded.getError()); // if error -> dump+fail

            const RuleSet& loadedSet = *loaded.getValue();
            TEST_EXPECT_EQUAL(loadedSet.size(), ruleset.size());

            // compare reloaded rules configs vs. array of expected configs.
            // tests:
            // - save+load is correct and complete
            // - Rule order is stable
            for (size_t r = 0; r<loadedSet.size(); ++r) {
                const Rule& rule = loadedSet.get(r);
                string cfg = rule.getConfig();
                TEST_EXPECT_EQUAL(cfg.c_str(), expRuleConfig[r]);
            }

            // test comment survives reload:
            TEST_EXPECT_EQUAL(loadedSet.getComment(), COMMENT);

            // test transferUndefFields survives save/load:
            TEST_EXPECT(loadedSet.shallTransferUndefFields());

            // use reloaded Ruleset for tests below:
            reloaded_ruleset = loadedSet; // also tests RuleSet-copy-ctor works.

            // test comment gets copied:
            TEST_EXPECT_EQUAL(reloaded_ruleset.getComment(), loadedSet.getComment());
        }
    }

    // test RuleSet load/save errors:
    {
        const char        *noSuchFile = "nosuch.fts";
        ErrorOrRuleSetPtr  loaded     = RuleSet::loadFrom(noSuchFile);

        TEST_EXPECT(loaded.hasError());
        TEST_EXPECT_ERROR_CONTAINS(loaded.getError(), "No such file or directory");

        const char *unsavable = "noSuchDir/whatever.fts";
        TEST_EXPECT_ERROR_CONTAINS(ruleset.saveTo(unsavable), "No such file or directory");
    }

    // load empty file -> empty RuleSet
    {
        const char *emptyFile = "general/empty.input";

        ErrorOrRuleSetPtr empty = RuleSet::loadFrom(emptyFile);
        TEST_REJECT(empty.hasError());

        const RuleSet& emptySet = *empty.getValue();
        TEST_EXPECT_ZERO(emptySet.size());            // test emptySet has no rules
        TEST_EXPECT_EQUAL(emptySet.getComment(), ""); // test emptySet has no comment
    }

    // use 'reloaded_ruleset' to modify same DB (as above in TEST_xferBySet):
    {
        XferEnv env;
        env.transferAllSpeciesBy(reloaded_ruleset);
        env.saveAndCompare(EXPECTED_ASCII, false); // if this fails -> saving/reloading config looses Rule information
    }
}

#define CUSTOM_ALI_TRANSPORT_ERROR "custom ali transport error"

struct TestAlignmentTransporter FINAL_TYPE : public FieldTransfer::AlignmentTransporter {
    int mode;
    TestAlignmentTransporter(int mode_) : mode(mode_) {}
    bool shallCopyBefore() const OVERRIDE {
        return false; // do not call copyAlignments() b4 calling transport()
    }
    GB_ERROR transport(GBDATA*gb_src_item, GBDATA *gb_dst_item) const OVERRIDE {
        GB_ERROR error = NULp;
        switch (mode) {
            case 1: // custom error
                error = CUSTOM_ALI_TRANSPORT_ERROR;
                break;

            case 2: // do nothing -> sequence still has old value (or is not copied)
                break;

            case 3: { // write reverse sequence data
                GBDATA *gb_src_data = GBT_find_sequence(gb_src_item, "ali_16s");
                GBDATA *gb_dst_data = GBT_find_sequence(gb_dst_item, "ali_16s");

                if (!gb_dst_data) { // destination has no 'ali_16s' -> clone whole container
                    GBDATA *gb_src_ali = GB_get_father(gb_src_data);

                    error = GB_incur_error_if(!GB_clone(gb_dst_item, gb_src_ali));
                    if (!error) {
                        gb_dst_data = GBT_find_sequence(gb_dst_item, "ali_16s");
                        xf_assert(gb_dst_data);
                    }
                }

                if (!error) {
                    const char *seq = GB_read_char_pntr(gb_src_data);
                    char       *rev = GBT_reverseNucSequence(seq, strlen(seq));

                    error = GB_write_string(gb_dst_data, rev);
                    free(rev);
                }
                break;
            }
            default: xf_assert(0); break; // unsupported mode
        }
        return error;
    }
};

void TEST_clone_by_ruleset() {
    using namespace FieldTransfer;

    RuleSetPtr ruleset;
    {
        const char *rulesetExpected = "impexp/ruleset.fts"; // same ruleset as used in tests above

        ErrorOrRuleSetPtr loaded = RuleSet::loadFrom(rulesetExpected);
        if (loaded.hasError()) TEST_EXPECT_NO_ERROR(loaded.getError()); // if RuleSet load error -> dump+fail. see .@loadFrom

        ruleset = loaded.getValue();
    }

    // use 'ruleset' to modify same DB (but use ItemClonedByRuleSet here)
    {
        XferEnv env;
        env.copyAllSpecies(); // copy species of input DB -> output DB

        GBDATA *gb_overwritten_species = NULp;
        char   *overwrittenName        = NULp;

        // clone some species (inside output db):
        {
            GB_transaction ta(env.gb_dest);

            GBDATA *gb_next_species = NULp;
            GBDATA *gb_first_clone  = NULp;
            int     count           = 0;

            for (GBDATA *gb_species = GBT_first_species(env.gb_dest);
                 gb_species && gb_species != gb_first_clone;
                 gb_species                = gb_next_species, ++count)
            {
                gb_next_species = GBT_next_species(gb_species);

                TEST_EXPECT_EQUAL(GB_countEntries(gb_species, "name"), 1); // safety-belt (had problems with duplicate name-entries)

                char *orgName = nulldup(GBT_get_name(gb_species));
                TEST_REJECT_NULL(orgName);

                GBDATA        *gb_clone = NULp;
                ItemCloneType  cloneHow = (count == 3 || count == 7) ? RENAME_ITEM_WHILE_TEMP_CLONE_EXISTS : REPLACE_ITEM_BY_CLONE;

                ruleset->set_transferUndefFields(count == 4); // test transfer of undefined fields for species #4

                {
                    ItemClonedByRuleSet clone(gb_species, CLONE_ITEM_SPECIES, ruleset, cloneHow, NULp, NULp);

                    if (clone.has_error()) TEST_EXPECT_NO_ERROR(clone.get_error());
                    gb_clone = clone.get_clone();
                    if (!gb_first_clone) {
                        xf_assert(cloneHow == REPLACE_ITEM_BY_CLONE); // limit will not work otherwise
                        gb_first_clone = gb_clone;
                    }

                    switch (cloneHow) {
                        case REPLACE_ITEM_BY_CLONE:
                            TEST_EXPECT_NULL(gb_species);
                            break;
                        case RENAME_ITEM_WHILE_TEMP_CLONE_EXISTS:
                            TEST_EXPECT_EQUAL(GB_countEntries(gb_species, "name"), 1);
                            TEST_EXPECT_EQUAL(GBT_get_name(gb_species), "fake"); // @@@ need a temporary name which cannot clash with existing names
                            break;
                        default:
                            xf_assert(0); // not tested here
                            break;
                    }

                    TEST_EXPECT_EQUAL(GB_countEntries(gb_clone, "name"), 1);
                    TEST_EXPECT_EQUAL(GBT_get_name(gb_clone), orgName);
                }
                // 'clone' has been destroyed now!

                if (cloneHow == RENAME_ITEM_WHILE_TEMP_CLONE_EXISTS) {
                    TEST_EXPECT_EQUAL(GBT_get_name(gb_species), orgName); // rename back worked (RENAME_ITEM_WHILE_TEMP_CLONE_EXISTS)
                }
                int orgNameCount = 0;
                for (GBDATA *gb_peek = GBT_first_species(env.gb_dest); gb_peek; gb_peek = GBT_next_species(gb_peek)) {
                    bool hasOrgName  = strcmp(GBT_get_name(gb_peek), orgName) == 0;
                    orgNameCount    += hasOrgName;
                    switch (cloneHow) {
                        case REPLACE_ITEM_BY_CLONE:
                            if (hasOrgName) TEST_EXPECT(gb_peek == gb_clone);   // orgName only used in persisting clone
                            TEST_EXPECT(gb_peek != gb_species);                 // species has been removed
                            break;
                        case RENAME_ITEM_WHILE_TEMP_CLONE_EXISTS:
                            if (hasOrgName) TEST_EXPECT(gb_peek == gb_species); // orgName only used in original species (i.e. temp. clone did vanish)
                            TEST_EXPECT(gb_peek != gb_clone);                   // clone has been removed
                            break;
                        default:
                            xf_assert(0); // not tested here
                            break;
                    }
                    // @@@ also test against "fake" names?
                }
                TEST_EXPECT_EQUAL(orgNameCount, 1); // species with duplicate names unwanted

                if (count == 3) {
                    gb_overwritten_species = gb_species; // = copy of original
                    overwrittenName        = ARB_strdup(orgName);
                }
                free(orgName);
            }
        }

        // test merging one species from source DB onto existing (cloned) species in dest DB:
        {
            GBDATA *gb_source_species;

            {
                GB_transaction ta1(env.gb_src);
                GB_transaction ta2(env.gb_dest);

                gb_source_species = GBT_find_species(env.gb_src, overwrittenName);
                TEST_REJECT_NULL(gb_source_species);      // (in gb_src)
                TEST_REJECT_NULL(gb_overwritten_species); // (in gb_dest)

                {
                    GBDATA *gb_name = GB_entry(gb_overwritten_species, "name");
                    TEST_EXPECT_NO_ERROR(GB_write_string(gb_name, "notOverwritten")); // prepare to test overwrite by data.

                    // modify "data" in "ali_16s" to prove overwrite later:
                    GBDATA *gb_seq = GBT_find_sequence(gb_overwritten_species, "ali_16s");
                    TEST_REJECT_NULL(gb_seq);

                    const char *seq    = GB_read_char_pntr(gb_seq);
                    char       *seqMod = GBS_string_eval(seq, ":U=T");

                    TEST_EXPECT_NO_ERROR(GB_write_string(gb_seq, seqMod));
                    free(seqMod);
                }
            }

            SmartPtr<TestAlignmentTransporter> reverseAliTransporter;

            // overwrite with result of ruleset (=> mix original and clone)
            for (int pass = 1; pass<=4; ++pass) {
                TEST_ANNOTATE(GBS_global_string("pass %i", pass));

                GB_transaction ta1(env.gb_src);
                GB_transaction ta2(env.gb_dest);

                SmartPtr<TestAlignmentTransporter> aliTransporter;
                if (pass<4) {
                    aliTransporter = new TestAlignmentTransporter(pass);
                    if (pass == 3) reverseAliTransporter = aliTransporter; // keep for later
                }

                ItemClonedByRuleSet overclone(gb_source_species, CLONE_ITEM_SPECIES, ruleset, CLONE_INTO_EXISTING, gb_overwritten_species, aliTransporter.content());

                if (pass == 1) {
                    TEST_EXPECT(overclone.has_error());
                    TEST_EXPECT_ERROR_CONTAINS(overclone.get_error(), CUSTOM_ALI_TRANSPORT_ERROR);
                    ta2.close(overclone.get_error());
                    ta1.close(overclone.get_error());
                }
                else {
                    if (overclone.has_error()) TEST_EXPECT_NO_ERROR(overclone.get_error()); // expect no error, but show message if expectation fails
                    TEST_EXPECT(overclone.get_clone() == gb_overwritten_species);           // test get_clone reports gb_overwritten_species here
                    TEST_EXPECT_EQUAL(GBT_get_name(gb_overwritten_species), "notOverwritten"); // test name of clone does not get overwritten
                    TEST_EXPECT_EQUAL(GB_countEntries(gb_overwritten_species, "name"), 1);

                    {
                        GBDATA *gb_seq = GBT_find_sequence(gb_overwritten_species, "ali_16s");
                        TEST_REJECT_NULL(gb_seq);

                        const char *seq = GB_read_char_pntr(gb_seq);

                        switch (pass) {
                            case 2: TEST_EXPECT_CONTAINS(seq, "GAAGTAGCTTGCTACTTTGCCGGCGAGCGGCGGAC"); break; // custom transporter: do nothing
                            case 3: TEST_EXPECT_CONTAINS(seq, "CAGGCGGCGAGCGGCCGUUUCAUCGUUCGAUGAAG"); break; // custom transporter: writes reversed sequence data
                            case 4: TEST_EXPECT_CONTAINS(seq, "GAAGUAGCUUGCUACUUUGCCGGCGAGCGGCGGAC"); break; // default behavior (=copy sequence over)
                            default: xf_assert(0); break; // unexpected 'pass'
                        }


                        GBDATA *gb_ali = GB_get_father(gb_seq);
                        TEST_EXPECT_EQUAL(GB_countEntries(gb_ali, "data"), 1);
                    }

                    TEST_EXPECT_EQUAL(GB_countEntries(gb_overwritten_species, "ali_16s"), 1);
                }
            }

            // "test" REAL_CLONE mode

            {
                GB_transaction ta1(env.gb_src);
                GB_transaction ta2(env.gb_dest);

                ItemClonedByRuleSet realClone(gb_source_species, CLONE_ITEM_SPECIES, ruleset, REAL_CLONE, GBT_get_species_data(env.gb_dest), &*reverseAliTransporter);

                if (realClone.has_error()) TEST_EXPECT_NO_ERROR(realClone.get_error()); // expect no error, but show message if expectation fails

                GBDATA *gb_clone = realClone.get_clone();

                TEST_REJECT_NULL(gb_clone);
                TEST_REJECT_NULL(gb_source_species);
                TEST_EXPECT(gb_clone != gb_source_species);
                TEST_EXPECT_EQUAL(GBT_get_name(gb_clone), GBT_get_name(gb_source_species));

                TEST_REJECT(GB_get_father(gb_clone) == GB_get_father(gb_source_species));

                {
                    GBDATA *gb_seq = GBT_find_sequence(gb_clone, "ali_16s");
                    TEST_REJECT_NULL(gb_seq);

                    const char *seq = GB_read_char_pntr(gb_seq);

                    // TEST_EXPECT_CONTAINS(seq, "GAAGUAGCUUGCUACUUUGCCGGCGAGCGGCGGAC"); // default behavior (=copy sequence over)
                    TEST_EXPECT_CONTAINS(seq, "CAGGCGGCGAGCGGCCGUUUCAUCGUUCGAUGAAG"); // custom transporter: writes reversed sequence data

                    GBDATA *gb_ali = GB_get_father(gb_seq);
                    TEST_EXPECT_EQUAL(GB_countEntries(gb_ali, "data"), 1);
                }
            }
        }

        env.saveAndCompare(EXPECTED_ASCII_CLONED, true);
        free(overwrittenName);
    }
}

#endif // UNIT_TESTS

// --------------------------------------------------------------------------------