AW_inotify.cxx

Go to the documentation of this file.

// =============================================================== //
//                                                                 //
//   File      : AW_inotify.cxx                                    //
//   Purpose   : watch file/directory changes                      //
//                                                                 //
//   Coded by Ralf Westram (coder@reallysoft.de) in October 2017   //
//   http://www.arb-home.de/                                       //
//                                                                 //
// =============================================================== //

#include "aw_inotify.hxx"

#include "aw_root.hxx"
#include <arbdbt.h>
#include <arb_file.h>

#include <unistd.h>

#include <string>
#include <list>
#include <set>

using namespace std;

#if defined(LINUX)
# define USE_INOTIFY
#endif

#if !defined(USE_INOTIFY)
# define USE_STATPOLL
#endif

#if defined(USE_STATPOLL)
#include <sys/stat.h>
#endif

#if defined(USE_INOTIFY)
#include <sys/inotify.h>
#endif

#if defined(DEBUG)
// #define TRACE_INOTIFY
// #define TRACE_INOTIFY_BASIC
#endif

#if defined(TRACE_INOTIFY)
#define IF_TRACE_INOTIFY(x) x
#else // !TRACE_INOTIFY
#define IF_TRACE_INOTIFY(x)
#endif


typedef set<FileChangedCallback> CallbackList;
class TrackedFiles;

class TrackedFile : public Noncopyable {
    string       file;
    CallbackList callbacks;

#if defined(USE_INOTIFY)
    int watch_descriptor;
#else // USE_STATPOLL

    mutable int lastModtime; // last known modification time of 'file'
    int getModtime() const {
        struct stat st;
        if (stat(file.c_str(), &st) == 0) return st.st_mtime;
        return 0;
    }
    bool was_changed() const {
        int  currModtime = getModtime();
        bool changed     = currModtime>lastModtime;
        lastModtime      = currModtime;
        return changed;
    }
#endif

public:
    TrackedFile(const string& filename) :
        file(filename)
    {
#if defined(USE_INOTIFY)
        watch_descriptor = -1;
#else // USE_STATPOLL
        lastModtime      = 0;
#endif
    }
#if defined(USE_INOTIFY)
    ~TrackedFile() {
        aw_assert(watch_descriptor == -1); // watch has to be removed before destruction
    }
#endif

    const string& get_name() const { return file; }

    void add_callback   (const FileChangedCallback& cb) { callbacks.insert(cb); }
    void remove_callback(const FileChangedCallback& cb) { callbacks.erase(cb); }

    bool empty() const { return callbacks.empty(); }

    void callAll(ChangeReason reason) const {
        bool deleted_still_exists = reason == CR_DELETED && GB_is_regularfile(file.c_str());
        if (!deleted_still_exists) {
            CallbackList copy = callbacks;
            for (CallbackList::const_iterator cb = copy.begin(); cb != copy.end(); ++cb) {
                if (callbacks.find(*cb) != callbacks.end()) {
                    (*cb)(file.c_str(), reason);
                }
            }
        }
    }

#if defined(USE_INOTIFY)
    int get_watch_descriptor() const {
        return watch_descriptor;
    }
    GB_ERROR add_watch(int inotifier) {
        aw_assert(watch_descriptor == -1);
        watch_descriptor = inotify_add_watch(inotifier, file.c_str(), IN_DELETE_SELF|IN_MOVE_SELF|IN_CLOSE_WRITE|IN_DELETE);
        return watch_descriptor<0 ? GB_IO_error("watching", file.c_str()) : NULp;
    }
    GB_ERROR remove_watch(int inotifier) {
        GB_ERROR err = NULp;
        if (watch_descriptor != -1) {
            if (inotify_rm_watch(inotifier, watch_descriptor)<0) {
                err = GB_IO_error("un-watching", file.c_str());
            }
            watch_descriptor = -1;
        }
        return err;
    }
    void mark_as_disabled() {
        watch_descriptor = -1;
    }
    void track_creation(TrackedFiles *tracked);

#else // USE_STATPOLL
    void callback_if_changed() const { if (was_changed()) callAll(CR_MODIFIED); }
#endif
};

typedef SmartPtr<TrackedFile> TrackedFilePtr;

class TrackedFiles : virtual Noncopyable {
    typedef list<TrackedFilePtr> FileList;

    FileList files;

#if defined(USE_INOTIFY)
    int inotifier;

    RefPtr<const char> error;

    SmartCharPtr ievent_buffer;
    int          oversize;

    bool reactivate_stale;

    size_t get_ievent_buffer_size() {
        return sizeof(inotify_event)+oversize;
    }
    void realloc_buffer(int new_oversize) {
        aw_assert(new_oversize>oversize);
        oversize      = new_oversize;
        ievent_buffer = ARB_alloc<char>(get_ievent_buffer_size());
    }

    inotify_event *get_ievent_buffer() {
        if (oversize<0) realloc_buffer(9*sizeof(inotify_event)); // reserve memory for 10 events
        return reinterpret_cast<inotify_event*>(&*ievent_buffer);
    }
    void increase_buffer_oversize() {
        aw_assert(oversize>0);
        realloc_buffer(oversize*3/2);
    }


#endif

    FileList::iterator find(const string& file) {
        for (FileList::iterator f = files.begin(); f != files.end(); ++f) {
            if (file == (*f)->get_name()) return f;
        }
        return files.end();
    }
#if defined(USE_INOTIFY)
    FileList::iterator find_watch_descriptor(int wd) {
        for (FileList::iterator f = files.begin(); f != files.end(); ++f) {
            if (wd == (*f)->get_watch_descriptor()) {
                return f;
            }
        }
        return files.end();
    }

    void check_for_created_files() {
        aw_assert(reactivate_stale);
        for (FileList::iterator fi = files.begin(); fi != files.end(); ++fi) {
            TrackedFilePtr f = *fi;
            if (f->get_watch_descriptor() == -1) {
                GB_ERROR err = f->add_watch(inotifier);
                if (!err) { // successfully reactivated
                    f->callAll(CR_CREATED);
                }
#if defined(TRACE_INOTIFY)
                fprintf(stderr, "check_for_created_files: add_watch for '%s': %s wd=%i\n", f->get_name().c_str(), err, f->get_watch_descriptor());
#endif
            }
        }
        // @@@ if no "stale" watch descriptor left => uninstall all watches of parent_modified_cb
    }
#endif

public:
#if defined(USE_INOTIFY)
    TrackedFiles() :
        error(NULp),
        oversize(-1),
        reactivate_stale(false)
    {
        inotifier = inotify_init();
        if (inotifier<0) {
            error = GB_IO_error("creating", "<inotify-instance>");
        }
    }
    ~TrackedFiles() {
        for (FileList::iterator fi = files.begin(); fi != files.end(); ++fi) {
            TrackedFilePtr f = *fi;
            if (f->get_watch_descriptor() >= 0) {
                GB_ERROR  ferr  = f->remove_watch(inotifier);
                if (ferr) fprintf(stderr, "Error in ~TrackedFiles: %s\n", ferr);
            }
            aw_assert(f->get_watch_descriptor() == -1);
        }
        close(inotifier);
    }
#endif

    GB_ERROR get_error() {
#if defined(USE_INOTIFY)
        GB_ERROR err = error;
        error        = NULp;
        return err;
#else // USE_STATPOLL
        return NULp;
#endif
    }

    bool empty() const {
        return files.empty();
    }
    void insert(string file, const FileChangedCallback& fccb) {
        // @@@ use canonical path of file!

#if defined(TRACE_INOTIFY_BASIC)
        fprintf(stderr, "[inotifier] + %s\n", file.c_str());
#endif

#if defined(USE_INOTIFY)
        aw_assert(!error);
#endif
        FileList::iterator foundIter = find(file);
        TrackedFilePtr     found;

        if (foundIter != files.end()) {
            found = *foundIter;
        }
        else {
            found = new TrackedFile(file);
            files.push_back(found);
#if defined(USE_INOTIFY)
            GB_ERROR werr = found->add_watch(inotifier);
            if (werr) found->track_creation(this);
#if defined(TRACE_INOTIFY)
            fprintf(stderr, "insert: add_watch for '%s': %s wd=%i\n", found->get_name().c_str(), werr, found->get_watch_descriptor());
#endif
#endif
        }
        found->add_callback(fccb);
    }
    void erase(string file, const FileChangedCallback& fccb) {
        // @@@ use canonical path of file!

#if defined(TRACE_INOTIFY_BASIC)
        fprintf(stderr, "[inotifier] - %s\n", file.c_str());
#endif

#if defined(USE_INOTIFY)
        aw_assert(!error);
#endif

        FileList::iterator foundIter = find(file);
        if (foundIter != files.end()) {
            TrackedFilePtr found = *foundIter;
            found->remove_callback(fccb);
            if (found->empty()) {
#if defined(USE_INOTIFY)
                if (found->get_watch_descriptor() != -1) {
                    error = found->remove_watch(inotifier);
                }
#endif
                files.erase(foundIter);
            }
        }
    }

    void check_changes();
#if defined(USE_INOTIFY)
    void request_reactivate_stale() { reactivate_stale = true; }
#endif
};

#if defined(USE_INOTIFY)
static void parent_modified_cb(const char *IF_TRACE_INOTIFY(parent_dir), ChangeReason IF_TRACE_INOTIFY(reason), TrackedFiles *tracked) {
#if defined(TRACE_INOTIFY)
    fprintf(stderr, "parent_modified_cb(dir='%s', reason=%i) called\n", parent_dir, int(reason));
#endif
    tracked->request_reactivate_stale();
}

void TrackedFile::track_creation(TrackedFiles *tracked) { // @@@ creation not tracked if !USE_INOTIFY
    aw_assert(get_watch_descriptor() == -1); // otherwise it exists

#if defined(TRACE_INOTIFY)
    fprintf(stderr, "TrackedFile::track_creation: file='%s'\n", file.c_str());
#endif

    char *parent_dir;
    GB_split_full_path(file.c_str(), &parent_dir, NULp, NULp, NULp);

#if defined(TRACE_INOTIFY)
    fprintf(stderr, "track_creation: parent_dir='%s'\n", parent_dir);
#endif

    aw_assert(parent_dir);
    if (parent_dir) {
        AW_add_inotification(parent_dir, makeFileChangedCallback(parent_modified_cb, tracked));
        free(parent_dir);
    }
}
#endif

void TrackedFiles::check_changes() {
    // called manually from unittest below
    // called via timer callback otherwise

#if defined(USE_INOTIFY)
    aw_assert(!error);

    timespec timeout = { 0, 0 }; // don't wait
    fd_set readfds;
    FD_ZERO(&readfds);
    FD_SET(inotifier, &readfds);

    int sel;
    while ((sel = pselect(inotifier+1, &readfds, NULp, NULp, &timeout, NULp))) { // only poll if events are waiting
        aw_assert(sel>0);

        inotify_event *ievent = get_ievent_buffer();
        ssize_t        got    = read(inotifier, ievent, get_ievent_buffer_size());

        if (got >= ssize_t(sizeof(inotify_event))) {
            while (got>0) {
                FileList::iterator foundIter = find_watch_descriptor(ievent->wd);
                if (foundIter != files.end()) {
                    TrackedFilePtr found         = *foundIter;
                    bool           watch_removed = (ievent->mask & IN_IGNORED);
#if defined(TRACE_INOTIFY)
                    const char *name = (ievent->len>1) ? ievent->name : "";

                    fprintf(stderr,
                            "got inotify event for '%s' (wd=%i, mask=%X, watch_removed=%i) '%s'\n",
                            found->get_name().c_str(), ievent->wd, ievent->mask, int(watch_removed), name);
#endif
                    if (watch_removed) {
                        found->mark_as_disabled();
                        found->track_creation(this);
                    }
                    else if (ievent->mask & IN_DELETE_SELF) {
                        found->callAll(CR_DELETED);
                    }
                    else if (ievent->mask & IN_MOVE_SELF) {
                        found->callAll(CR_DELETED);
                        found->remove_watch(inotifier); // invalidate watch descriptor of found
                        found->track_creation(this);    // ensure re-creation gets detected
                        reactivate_stale = true;        // causes check whether new target has a "stale" watch
                    }
                    else if (ievent->mask & (IN_DELETE|IN_CLOSE_WRITE)) { // item deleted from directory or file/dir modified
                        found->callAll(CR_MODIFIED);
                    }
                    else {
                        aw_assert(0); // unhandled event from inotify
                    }
                }
                else {
#if defined(TRACE_INOTIFY)
                    fprintf(stderr, "ignoring event for unknown watch_descriptor=%i\n", ievent->wd);
#endif
                }

                int event_len  = sizeof(inotify_event)+ievent->len;
                got           -= event_len;
                ievent         = reinterpret_cast<inotify_event*>(reinterpret_cast<char*>(ievent)+event_len);
            }
        }
        else {
            bool     buffer_too_small = false;
            GB_ERROR read_err         = NULp;

            if (got == -1) {
                if (errno == EINVAL) buffer_too_small = true;
                else read_err = GB_IO_error("reading", "<inotify-descriptor>");
            }
            else if (got == 0) buffer_too_small = true;
            else {
                aw_assert(0);
            }

            if (read_err) {
                fprintf(stderr, "inotifier broken: %s\n", read_err);
                aw_assert(0);
                break; // while loop
            }

            if (!buffer_too_small) {
                aw_assert(buffer_too_small);
                GBK_terminate("inotify event queue broken? (neither buffer_too_small nor read_err)");
            }
            increase_buffer_oversize();
        }

        aw_assert(FD_ISSET(inotifier, &readfds));
    }

    if (reactivate_stale) {
        check_for_created_files();
        reactivate_stale = false; // do once if any dir-content changes
    }

#else // USE_STATPOLL
    for (FileList::iterator fi = files.begin(); fi != files.end(); ++fi) {
        TrackedFilePtr f = *fi;
        f->callback_if_changed();
    }
#endif
}

// --------------------------------------------------------------------------------

static SmartPtr<TrackedFiles> allTrackers(new TrackedFiles);

static bool    maintain_timer_callback = true;
const unsigned AW_INOTIFY_TIMER        = 700; // ms

static unsigned timed_inotifications_check_cb() {
    allTrackers->check_changes();
    if (allTrackers->empty()) {
#if defined(TRACE_INOTIFY_BASIC)
        fputs("[inotifier] - timer callback\n", stderr);
#endif
        return 0;
    }
    return AW_INOTIFY_TIMER;
}

static void show_tracked_error() {
    GB_ERROR err = allTrackers->get_error();
    if (err) {
        fprintf(stderr, "Error in TrackedFiles: %s\n", err);
    }
}

void AW_add_inotification(const char *file, const FileChangedCallback& fccb) {
    show_tracked_error();
    if (allTrackers->empty() & maintain_timer_callback) {
#if defined(TRACE_INOTIFY_BASIC)
        fputs("[inotifier] + timer callback\n", stderr);
#endif
        AW_root::SINGLETON->add_timed_callback(AW_INOTIFY_TIMER, makeTimedCallback(timed_inotifications_check_cb));
    }
    allTrackers->insert(file, fccb);
    show_tracked_error();
}

void AW_remove_inotification(const char *file, const FileChangedCallback& fccb) {
    allTrackers->erase(file, fccb);
    show_tracked_error();
}

// --------------------------------------------------------------------------------

#ifdef UNIT_TESTS
#ifndef TEST_UNIT_H
#include <test_unit.h>
#endif
#include <arb_cs.h>
#include <arb_str.h>

const int FLAGS = 4;
static int change_flag[FLAGS][CHANGE_REASONS];

static void trace_file_changed_cb(const char *IF_TRACE_INOTIFY(file), ChangeReason reason, int flag) {
    aw_assert(flag<FLAGS);
    change_flag[flag][reason]++;

#if defined(TRACE_INOTIFY)
    fprintf(stderr, "trace_file_changed_cb: flag=%i reason=%i file='%s'\n", flag, int(reason), file);
#endif
}

// buf layout is "0123-0123-0123" (digits here are flag indices)
#define BUFIDX(f,r) ((r)*(FLAGS+1)+(f))

static const char *update_change_counts() {
    for (int f = 0; f<FLAGS; ++f) {
        for (int r = 0; r<CHANGE_REASONS; ++r) {
            change_flag[f][r] = 0;
        }
    }

    allTrackers->check_changes();

    static char buf[BUFIDX(0,CHANGE_REASONS)];
    for (int r = 0; r<CHANGE_REASONS; ++r) {
        for (int f = 0; f<FLAGS; ++f) {
            buf[BUFIDX(f,r)] = '0'+change_flag[f][r];
        }
        buf[BUFIDX(FLAGS,r)] = '-';
    }
    buf[BUFIDX(0,CHANGE_REASONS)-1] = 0;
    return buf;
}

static arb_test::match_expectation change_counts_are(const char *expected_counts) {
    using namespace   arb_test;
    expectation_group expected;

    const char *detected_counts = update_change_counts();

    expected.add(that(detected_counts).is_equal_to(expected_counts));

    return all().ofgroup(expected);
}

#define TEST_EXPECT_CHANGE_COUNTS(expected) TEST_EXPECTATION(change_counts_are(expected))

static void update_file(const char *filename) {
    FILE *out = fopen(filename, "wt");
    aw_assert(out);
    fputs("hi", out);
    fclose(out);
}

inline void touch_files(const char *f1, const char *f2 = NULp) {
#if defined(USE_STATPOLL)
    static time_t last = 0;
    {
        time_t now;
        do time(&now); while (now == last); // wait for new second (to ensure timestamps differ)
    }
#endif

    if (f1) update_file(f1);
    if (f2) update_file(f2);

#if defined(USE_STATPOLL)
    time(&last);
#endif
}

inline void erase_files(const char *f1, const char *f2 = NULp) {
    if (f1) TEST_EXPECT_MORE_EQUAL(GB_unlink(f1), 0);
    if (f2) TEST_EXPECT_MORE_EQUAL(GB_unlink(f2), 0);
}

inline void move_file(const char *src, const char *dst) {
    TEST_EXPECT_NO_ERROR(GB_move_file(src, dst));
}
inline void double_move_file(const char *src, const char *tmp, const char *dst) {
    move_file(src, tmp);
    move_file(tmp, dst);
}

#define INOTIFY_TESTDIR "inotify"

// #define RETRY_INOTIFICATIONS_TEST

void TEST_inotifications() {
#if !defined(USE_INOTIFY)
# warning inotifications broken in poll mode
    MISSING_TEST(TEST_inotifications);
    // @@@ fix behavior in poll mode (tests below fail)
    // functionality needed in arb seems to work nevertheless (maybe just test less if !USE_INOTIFY?)
    return;
#endif

    // for some unknown reason this test randomly fails on one build host
    //
    // Update: the previous changes (log:trunk@16775:16780) do not fix the problem :(
    // * if the test fails initially, repeated calls always fail!
    // * but initial failure only happens with 5-10% probability (of calls of test-executable)
    //
    static bool inotify_tests_failing_randomly = ARB_stricmp(arb_gethostname(), "build-jessie") == 0;
    if (inotify_tests_failing_randomly) {
#if defined(RETRY_INOTIFICATIONS_TEST)
        TEST_ALLOW_RETRY(5); // tell test-suite to try this test up to 5 times
#else
        MISSING_TEST("TEST_inotifications fails randomly on build-jessie -> skipped!");
        return; // disable test on this host
#endif
    }

    const char *inotify_testdir = INOTIFY_TESTDIR;

    const char *testfile1 = INOTIFY_TESTDIR "/inotify1.testfile";
    const char *testfile2 = INOTIFY_TESTDIR "/inotify2.testfile";
    const char *nofile    = INOTIFY_TESTDIR "/no.testfile";

    TEST_EXPECT_NO_ERROR(GB_create_directory(INOTIFY_TESTDIR));

#if defined(RETRY_INOTIFICATIONS_TEST)
    static bool retrying = false;
    if (retrying) {
        // remove any leftovers from last try
        erase_files(testfile1,testfile2);
        erase_files(nofile);
        TEST_EXPECT_ZERO(rmdir(inotify_testdir));
        allTrackers = new TrackedFiles;
        TEST_EXPECT_NO_ERROR(GB_create_directory(INOTIFY_TESTDIR));
    }
    retrying = true;
#endif

    FileChangedCallback fccb1 = makeFileChangedCallback(trace_file_changed_cb, 0);
    FileChangedCallback fccb2 = makeFileChangedCallback(trace_file_changed_cb, 1);
    FileChangedCallback fccb3 = makeFileChangedCallback(trace_file_changed_cb, 2);
    FileChangedCallback dccb4 = makeFileChangedCallback(trace_file_changed_cb, 3); // tracks directory

    maintain_timer_callback = false;
    erase_files(nofile); // make sure file does not exist

    // event order  (= order of ChangeReason) is "MODIFIED-DELETED-UNKNOWN"
    TEST_EXPECT_CHANGE_COUNTS("0000-0000-0000");

    AW_add_inotification(inotify_testdir, dccb4);

    touch_files(testfile1, testfile2); TEST_EXPECT_CHANGE_COUNTS("0002-0000-0000"); // two cbs for two changes in directory

    AW_add_inotification(testfile1, fccb1);

    touch_files(testfile1, testfile2); TEST_EXPECT_CHANGE_COUNTS("1002-0000-0000"); // two cbs for dir-changes; one cb for testfile1

    AW_add_inotification(testfile2, fccb2);

    touch_files(testfile1, testfile2); TEST_EXPECT_CHANGE_COUNTS("1102-0000-0000");

    erase_files(testfile1); TEST_EXPECT_CHANGE_COUNTS("0001-1000-0000"); // dir changed + testfile1 deleted
    touch_files(testfile1); TEST_EXPECT_CHANGE_COUNTS("0001-0000-1000"); // dir changed + testfile1 (re-)created
    touch_files(testfile1); TEST_EXPECT_CHANGE_COUNTS("1001-0000-0000"); // dir changed + testfile1 modified

    erase_files(testfile2); TEST_EXPECT_CHANGE_COUNTS("0001-0100-0000"); // dir changed + testfile2 deleted

    AW_add_inotification(nofile, fccb3); // add a tracker on non-existing file

    // touch non-yet-existing 'nofile' -> should detect creation
    touch_files(nofile);                      TEST_EXPECT_CHANGE_COUNTS("0001-0000-0010"); // detects creation of non-existing file
    erase_files(nofile); touch_files(nofile); TEST_EXPECT_CHANGE_COUNTS("0002-0000-0010"); // does detect quick delete+recreate as "create" (delete gets suppressed)
    erase_files(nofile);                      TEST_EXPECT_CHANGE_COUNTS("0001-0010-0000"); // detects erase
    touch_files(nofile); erase_files(nofile); TEST_EXPECT_CHANGE_COUNTS("0002-0000-0000"); // does NOT detect quick create+delete (just 2 directory changes)

    // test moving files
    move_file(testfile1, nofile);                   TEST_EXPECT_CHANGE_COUNTS("0000-1000-0010"); // detects delete 'testfile1' and create 'nofile'
    touch_files(nofile);                            TEST_EXPECT_CHANGE_COUNTS("0011-0000-0000"); // detects modify 'nofile'
    touch_files(testfile1);                         TEST_EXPECT_CHANGE_COUNTS("0001-0000-1000"); // detects create 'testfile1'
    move_file(nofile, testfile1);                   TEST_EXPECT_CHANGE_COUNTS("0000-0010-1000"); // detects delete 'nofile' and create 'testfile1' (delete of overwritten 'testfile1' gets suppressed)
    double_move_file(testfile1, nofile, testfile1); TEST_EXPECT_CHANGE_COUNTS("0000-0000-1000"); // detects creation of 'testfile1' ('nofile' does not trigger)
    double_move_file(testfile1, nofile, testfile2); TEST_EXPECT_CHANGE_COUNTS("0000-1000-0100"); // delete 'testfile1' + create 'testfile2' ('nofile' does not trigger)
    touch_files(testfile1, nofile);                 TEST_EXPECT_CHANGE_COUNTS("0002-0000-1010"); // create 'testfile1'+'nofile'
    double_move_file(testfile2, nofile, testfile1); TEST_EXPECT_CHANGE_COUNTS("0000-0110-1000"); // delete 'testfile2' and 'nofile' + create 'testfile1'

    AW_add_inotification(testfile1, fccb3);
    AW_add_inotification(testfile2, fccb3);

    touch_files(testfile1, testfile2); TEST_EXPECT_CHANGE_COUNTS("1012-0000-0110"); // dir changed + testfile1 modified + testfile2 created

    touch_files(testfile1);            TEST_EXPECT_CHANGE_COUNTS("1011-0000-0000"); // 2 cbs for file + 1 cb for dir triggered by one touch
    touch_files(testfile2);            TEST_EXPECT_CHANGE_COUNTS("0111-0000-0000"); // same for other testfile

    touch_files(testfile1, testfile2); TEST_EXPECT_CHANGE_COUNTS("1122-0000-0000"); // 6 callbacks triggered by 2 touches
    AW_add_inotification(testfile2, fccb3); // (try to) add a tracker twice
    touch_files(testfile1, testfile2); TEST_EXPECT_CHANGE_COUNTS("1122-0000-0000"); // still 6 callbacks triggered by 2 touches

    AW_remove_inotification(testfile1, fccb1);

    touch_files(testfile1, testfile2); TEST_EXPECT_CHANGE_COUNTS("0122-0000-0000");

    // changing cd is ok for triggers @@@ might fail for delete+re-create
    touch_files(testfile1, testfile2);
    {
        TEST_EXPECT_ZERO(chdir(inotify_testdir));
        TEST_EXPECT_CHANGE_COUNTS("0122-0000-0000");
        TEST_EXPECT_ZERO(chdir(".."));
    }

    AW_remove_inotification(testfile2, fccb2);

    touch_files(testfile1, testfile2); TEST_EXPECT_CHANGE_COUNTS("0022-0000-0000");

    AW_remove_inotification(testfile1, fccb3);

    touch_files(testfile1, testfile2); TEST_EXPECT_CHANGE_COUNTS("0012-0000-0000");

    AW_remove_inotification(testfile2, fccb3);

    touch_files(testfile1, testfile2); TEST_EXPECT_CHANGE_COUNTS("0002-0000-0000");

    AW_remove_inotification(inotify_testdir, dccb4);

    touch_files(testfile1, testfile2); TEST_EXPECT_CHANGE_COUNTS("0000-0000-0000");

    TEST_EXPECT_ZERO(GB_unlink(testfile1));
    TEST_EXPECT_ZERO(GB_unlink(testfile2));
    TEST_EXPECT_ZERO(rmdir(inotify_testdir));

    allTrackers = new TrackedFiles; // test cleanup (normally occurs at program exit)
}

#endif // UNIT_TESTS

// --------------------------------------------------------------------------------