io-gncxml-v2.cpp

/********************************************************************\
 * Copyright (C) 2000,2001 Gnumatic Inc.                            *
 *                                                                  *
 * This program is free software; you can redistribute it and/or    *
 * modify it under the terms of the GNU General Public License as   *
 * published by the Free Software Foundation; either version 2 of   *
 * the License, or (at your option) any later version.              *
 *                                                                  *
 * This program is distributed in the hope that it will be useful,  *
 * but WITHOUT ANY WARRANTY; without even the implied warranty of   *
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the    *
 * GNU General Public License for more details.                     *
 *                                                                  *
 * You should have received a copy of the GNU General Public License*
 * along with this program; if not, contact:                        *
 *                                                                  *
 * Free Software Foundation           Voice:  +1-617-542-5942       *
 * 51 Franklin Street, Fifth Floor    Fax:    +1-617-542-2652       *
 * Boston, MA  02110-1301,  USA       gnu@gnu.org                   *
\********************************************************************/
#include <glib.h>
#include <glib/gstdio.h>

#include <config.h>

#include <platform.h>
#if PLATFORM(WINDOWS)
#ifdef __STRICT_ANSI__
#undef __STRICT_ANSI__
#define __STRICT_ANSI_UNSET__ 1
#endif
#ifdef _NO_OLDNAMES
#undef _NO_OLDNAMES
#endif
#ifdef _UWIN
#undef _UWIN
#endif
#include <windows.h>
#endif
#include <fcntl.h>
#include <string.h>
#ifdef HAVE_UNISTD_H
# include <unistd.h>
#endif
#include <zlib.h>
#include <errno.h>
#include <cstdint>

#include "gnc-engine.h"
#include "gnc-pricedb-p.h"
#include "Scrub.h"
#include "SX-book.h"
#include "SX-book-p.h"
#include "Transaction.h"
#include "TransactionP.hpp"
#include "TransLog.h"
#if PLATFORM(WINDOWS)
#ifdef __STRICT_ANSI_UNSET__
#undef __STRICT_ANSI_UNSET__
#define __STRICT_ANSI__ 1
#endif
#endif
#if COMPILER(MSVC)
# define g_fopen fopen
# define g_open _open
#endif

#include "gnc-xml-backend.hpp"
#include "sixtp-parsers.h"
#include "sixtp-utils.h"
#include "gnc-xml.h"
#include "io-utils.h"
#include "sixtp-dom-parsers.h"
#include "io-gncxml-v2.h"
#include "io-gncxml-gen.h"

static QofLogModule log_module = GNC_MOD_IO;

typedef struct
{
    gint fd;
    gchar* filename;
    gchar* perms;
    gboolean write;
} gz_thread_params_t;

/* Callback structure */
struct file_backend
{
    gboolean        ok;
    gpointer        data;
    sixtp_gdv2*     gd;
    const char*     tag;
    sixtp*          parser;
    FILE*           out;
    QofBook*        book;
};

static std::vector<GncXmlDataType_t> backend_registry;
void
gnc_xml_register_backend(GncXmlDataType_t& xmlbe)
{
    backend_registry.push_back(xmlbe);
}

#define GNC_V2_STRING "gnc-v2"
/* non-static because they are used in sixtp.c */
const gchar* gnc_v2_xml_version_string = GNC_V2_STRING;
extern const gchar*
gnc_v2_book_version_string;        /* see gnc-book-xml-v2 */

/* Forward declarations */
static std::pair<FILE*, GThread*> try_gz_open (const char* filename,
                                               const char* perms,
                                               gboolean compress,
                                               gboolean write);
static bool is_gzipped_file (const gchar* name);

static void
clear_up_account_commodity (
    gnc_commodity_table* tbl, Account* act,
    gnc_commodity * (*getter) (const Account* account),
    void (*setter) (Account* account, gnc_commodity* comm),
    int (*scu_getter) (const Account* account),
    void (*scu_setter) (Account* account, int scu))
{
    gnc_commodity* gcom;
    gnc_commodity* com = getter (act);
    int old_scu;

    if (scu_getter)
        old_scu = scu_getter (act);
    else
        old_scu = 0;

    if (!com)
    {
        return;
    }

    gcom = gnc_commodity_table_lookup (tbl, gnc_commodity_get_namespace (com),
                                       gnc_commodity_get_mnemonic (com));

    if (gcom == com)
    {
        return;
    }
    else if (!gcom)
    {
        PWARN ("unable to find global commodity for %s adding new",
               gnc_commodity_get_unique_name (com));
        gnc_commodity_table_insert (tbl, com);
    }
    else
    {
        setter (act, gcom);
        if (old_scu != 0 && scu_setter)
            scu_setter (act, old_scu);
        gnc_commodity_destroy (com);
    }
}

static void
clear_up_transaction_commodity (
    gnc_commodity_table* tbl, Transaction* trans,
    gnc_commodity * (*getter) (const Transaction* trans),
    void (*setter) (Transaction* trans, gnc_commodity* comm))
{
    gnc_commodity* gcom;
    gnc_commodity* com = getter (trans);

    if (!com)
    {
        return;
    }

    gcom = gnc_commodity_table_lookup (tbl, gnc_commodity_get_namespace (com),
                                       gnc_commodity_get_mnemonic (com));

    if (gcom == com)
    {
        return;
    }
    else if (!gcom)
    {
        PWARN ("unable to find global commodity for %s adding new",
               gnc_commodity_get_unique_name (com));
        gnc_commodity_table_insert (tbl, com);
    }
    else
    {
        xaccTransBeginEdit (trans);
        setter (trans, gcom);
        xaccTransCommitEdit (trans);
        gnc_commodity_destroy (com);
    }
}

static gboolean
add_account_local (sixtp_gdv2* data, Account* act)
{
    gnc_commodity_table* table;
    Account* parent, *root;
    int type;

    table = gnc_commodity_table_get_table (data->book);

    clear_up_account_commodity (table, act,
                                DxaccAccountGetCurrency,
                                DxaccAccountSetCurrency,
                                NULL, NULL);

    clear_up_account_commodity (table, act,
                                xaccAccountGetCommodity,
                                xaccAccountSetCommodity,
                                xaccAccountGetCommoditySCUi,
                                xaccAccountSetCommoditySCU);

    xaccAccountScrubCommodity (act);
    xaccAccountScrubKvp (act);

    /* Backwards compatibility.  If there's no parent, see if this
     * account is of type ROOT.  If not, find or create a ROOT
     * account and make that the parent. */
    type = xaccAccountGetType (act);
    if (type == ACCT_TYPE_ROOT)
    {
        gnc_book_set_root_account (data->book, act);
    }
    else
    {
        parent = gnc_account_get_parent (act);
        if (parent == NULL)
        {
            root = gnc_book_get_root_account (data->book);
            gnc_account_append_child (root, act);
        }
    }

    data->counter.accounts_loaded++;
    sixtp_run_callback (data, "account");

    return FALSE;
}

static gboolean
add_book_local (sixtp_gdv2* data, QofBook* book)
{
    data->counter.books_loaded++;
    sixtp_run_callback (data, "book");

    return FALSE;
}

static gboolean
add_commodity_local (sixtp_gdv2* data, gnc_commodity* com)
{
    gnc_commodity_table* table;

    table = gnc_commodity_table_get_table (data->book);

    gnc_commodity_table_insert (table, com);

    data->counter.commodities_loaded++;
    sixtp_run_callback (data, "commodities");

    return TRUE;
}

static gboolean
add_transaction_local (sixtp_gdv2* data, Transaction* trn)
{
    gnc_commodity_table* table;

    table = gnc_commodity_table_get_table (data->book);

    xaccTransBeginEdit (trn);
    clear_up_transaction_commodity (table, trn,
                                    xaccTransGetCurrency,
                                    xaccTransSetCurrency);

    xaccTransScrubCurrency (trn);
    xaccTransScrubPostedDate (trn);
    xaccTransCommitEdit (trn);

    data->counter.transactions_loaded++;
    sixtp_run_callback (data, "transaction");
    return TRUE;
}

static gboolean
add_schedXaction_local (sixtp_gdv2* data, SchedXaction* sx)
{
    SchedXactions* sxes;
    sxes = gnc_book_get_schedxactions (data->book);
    gnc_sxes_add_sx (sxes, sx);
    data->counter.schedXactions_loaded++;
    sixtp_run_callback (data, "schedXactions");
    return TRUE;
}

static gboolean
add_template_transaction_local (sixtp_gdv2* data,
                                gnc_template_xaction_data* txd)
{
    GList* n;
    Account* acctRoot = NULL;
    QofBook* book;

    book = data->book;

    /* expect a struct of: */
    /* . template accounts. */
    /* . transactions in those accounts. */
    for (n = txd->accts; n; n = n->next)
    {
        if (gnc_account_get_parent ((Account*)n->data) == NULL)
        {
            if (xaccAccountGetType ((Account*)n->data) == ACCT_TYPE_ROOT)
            {
                /* replace the gnc_book_init-created root account */
                gnc_book_set_template_root (book, (Account*)n->data);
            }
            else
            {
                /* This is an old data file that doesn't have a template root
                   account and this is a top level account.  Make it a child
                   of the template root account. */
                acctRoot = gnc_book_get_template_root (book);
                gnc_account_append_child (acctRoot, (Account*)n->data);
            }
        }

    }

    for (n = txd->transactions; n; n = n->next)
    {
        /* insert transactions into accounts */
        add_transaction_local (data, (Transaction*)n->data);
    }

    return TRUE;
}

static gboolean
add_pricedb_local (sixtp_gdv2* data, GNCPriceDB* db)
{
    /* gnc_pricedb_print_contents(db, stdout); */
    return TRUE;
}

static void
counter (const GncXmlDataType_t& data, file_backend* be_data)
{
    g_return_if_fail (data.version == GNC_FILE_BACKEND_VERS);

    if (be_data->ok == TRUE)
        return;

    if (!g_strcmp0 (be_data->tag, data.type_name))
        be_data->ok = TRUE;

    /* XXX: should we do anything with this counter? */
}

static gboolean
gnc_counter_end_handler (gpointer data_for_children,
                         GSList* data_from_children, GSList* sibling_data,
                         gpointer parent_data, gpointer global_data,
                         gpointer* result, const gchar* tag)
{
    char* strval;
    gint64 val;
    char* type;
    xmlNodePtr tree = (xmlNodePtr)data_for_children;
    gxpf_data* gdata = (gxpf_data*)global_data;
    sixtp_gdv2* sixdata = (sixtp_gdv2*)gdata->parsedata;
    gboolean ret = TRUE;

    if (parent_data)
        return TRUE;

    /* OK.  For some messed up reason this is getting called again with a
       NULL tag.  So we ignore those cases */
    if (!tag)
        return TRUE;

    g_return_val_if_fail (tree, FALSE);

    /* Note: BADXML.
     *
     * This is invalid xml because the namespace isn't declared in the
     * tag itself. This should be changed to 'type' at some point. */
    type = (char*)xmlGetProp (tree, BAD_CAST "cd:type");
    strval = dom_tree_to_text (tree);
    if (!string_to_gint64 (strval, &val))
    {
        PERR ("string_to_gint64 failed with input: %s",
              strval ? strval : "(null)");
        ret = FALSE;
    }
    else if (g_strcmp0 (type, "transaction") == 0)
    {
        sixdata->counter.transactions_total = val;
    }
    else if (g_strcmp0 (type, "account") == 0)
    {
        sixdata->counter.accounts_total = val;
    }
    else if (g_strcmp0 (type, "book") == 0)
    {
        sixdata->counter.books_total = val;
    }
    else if (g_strcmp0 (type, "commodity") == 0)
    {
        sixdata->counter.commodities_total = val;
    }
    else if (g_strcmp0 (type, "schedxaction") == 0)
    {
        sixdata->counter.schedXactions_total = val;
    }
    else if (g_strcmp0 (type, "budget") == 0)
    {
        sixdata->counter.budgets_total = val;
    }
    else if (g_strcmp0 (type, "price") == 0)
    {
        sixdata->counter.prices_total = val;
    }
    else
    {
        struct file_backend be_data;

        be_data.ok = FALSE;
        be_data.tag = type;
        for(auto data : backend_registry)
            counter(data, &be_data);

        if (be_data.ok == FALSE)
        {
            PERR ("Unknown type: %s", type ? type : "(null)");
            /* Do *NOT* flag this as an error. Gnucash 1.8 writes invalid
             * xml by writing the 'cd:type' attribute without providing
             * the namespace in the gnc:count-data tag.  The parser is
             * entirely within its rights to refuse to read this bad
             * attribute. Gnucash will function correctly without the data
             * in this tag, so just let the error pass. */
            ret = TRUE;
        }
    }

    g_free (strval);
    xmlFree (type);
    xmlFreeNode (tree);
    return ret;
}

static sixtp*
gnc_counter_sixtp_parser_create (void)
{
    return sixtp_dom_parser_new (gnc_counter_end_handler, NULL, NULL);
}

static void
debug_print_counter_data (load_counter* data)
{
    DEBUG ("Transactions: Total: %d, Loaded: %d",
           data->transactions_total, data->transactions_loaded);
    DEBUG ("Accounts: Total: %d, Loaded: %d",
           data->accounts_total, data->accounts_loaded);
    DEBUG ("Books: Total: %d, Loaded: %d",
           data->books_total, data->books_loaded);
    DEBUG ("Commodities: Total: %d, Loaded: %d",
           data->commodities_total, data->commodities_loaded);
    DEBUG ("Scheduled Transactions: Total: %d, Loaded: %d",
           data->schedXactions_total, data->schedXactions_loaded);
    DEBUG ("Budgets: Total: %d, Loaded: %d",
           data->budgets_total, data->budgets_loaded);
}

static void
file_rw_feedback (sixtp_gdv2* gd, const char* type)
{
    load_counter* counter;
    int loaded, total, percentage;

    g_assert (gd != NULL);
    if (!gd->gui_display_fn)
        return;

    counter = &gd->counter;
    loaded = counter->transactions_loaded + counter->accounts_loaded +
             counter->books_loaded + counter->commodities_loaded +
             counter->schedXactions_loaded + counter->budgets_loaded +
             counter->prices_loaded;
    total = counter->transactions_total + counter->accounts_total +
            counter->books_total + counter->commodities_total +
            counter->schedXactions_total + counter->budgets_total +
            counter->prices_total;
    if (total == 0)
        total = 1;

    percentage = (loaded * 100) / total;
    if (percentage > 100)
    {
        /* FIXME: Perhaps the below should be replaced by:
        print_counter_data(counter); */
//      printf("Transactions: Total: %d, Loaded: %d\n",
//             counter->transactions_total, counter->transactions_loaded);
//      printf("Accounts: Total: %d, Loaded: %d\n",
//             counter->accounts_total, counter->accounts_loaded);
//      printf("Books: Total: %d, Loaded: %d\n",
//             counter->books_total, counter->books_loaded);
//      printf("Commodities: Total: %d, Loaded: %d\n",
//             counter->commodities_total, counter->commodities_loaded);
//      printf("Scheduled Transactions: Total: %d, Loaded: %d\n",
//             counter->schedXactions_total, counter->schedXactions_loaded);
//      printf("Budgets: Total: %d, Loaded: %d\n",
//       counter->budgets_total, counter->budgets_loaded);
    }
    gd->gui_display_fn (NULL, percentage);
}

static const char* BOOK_TAG = "gnc:book";
static const char* BOOK_ID_TAG = "book:id";
static const char* BOOK_SLOTS_TAG = "book:slots";
static const char* ACCOUNT_TAG = "gnc:account";
static const char* PRICEDB_TAG = "gnc:pricedb";
static const char* COMMODITY_TAG = "gnc:commodity";
static const char* COUNT_DATA_TAG = "gnc:count-data";
static const char* TRANSACTION_TAG = "gnc:transaction";
static const char* SCHEDXACTION_TAG = "gnc:schedxaction";
static const char* TEMPLATE_TRANSACTION_TAG = "gnc:template-transactions";
static const char* BUDGET_TAG = "gnc:budget";

static void
add_item (const GncXmlDataType_t& data, struct file_backend* be_data)
{
    g_return_if_fail (data.version == GNC_FILE_BACKEND_VERS);

    if (be_data->ok)
        return;

    if (!g_strcmp0 (be_data->tag, data.type_name))
    {
        if (data.add_item)
            (data.add_item)(be_data->gd, be_data->data);

        be_data->ok = TRUE;
    }
}

static gboolean
book_callback (const char* tag, gpointer globaldata, gpointer data)
{
    sixtp_gdv2* gd = (sixtp_gdv2*)globaldata;

    if (g_strcmp0 (tag, ACCOUNT_TAG) == 0)
    {
        add_account_local (gd, (Account*)data);
    }
    else if (g_strcmp0 (tag, PRICEDB_TAG) == 0)
    {
        add_pricedb_local (gd, (GNCPriceDB*)data);
    }
    else if (g_strcmp0 (tag, COMMODITY_TAG) == 0)
    {
        add_commodity_local (gd, (gnc_commodity*)data);
    }
    else if (g_strcmp0 (tag, TRANSACTION_TAG) == 0)
    {
        add_transaction_local (gd, (Transaction*)data);
    }
    else if (g_strcmp0 (tag, SCHEDXACTION_TAG) == 0)
    {
        add_schedXaction_local (gd, (SchedXaction*)data);
    }
    else if (g_strcmp0 (tag, TEMPLATE_TRANSACTION_TAG) == 0)
    {
        add_template_transaction_local (gd, (gnc_template_xaction_data*)data);
    }
    else if (g_strcmp0 (tag, BUDGET_TAG) == 0)
    {
        // Nothing needed here.
    }
    else
    {
        struct file_backend be_data;

        be_data.ok = FALSE;
        be_data.tag = tag;
        be_data.gd = gd;
        be_data.data = data;

        for (auto data : backend_registry)
            add_item(data, &be_data);

        if (be_data.ok == FALSE)
        {
            PWARN ("unexpected tag %s", tag);
        }
    }
    return TRUE;
}

static gboolean
generic_callback (const char* tag, gpointer globaldata, gpointer data)
{
    sixtp_gdv2* gd = (sixtp_gdv2*)globaldata;

    if (g_strcmp0 (tag, BOOK_TAG) == 0)
    {
        add_book_local (gd, (QofBook*)data);
        book_callback (tag, globaldata, data);
    }
    else
    {
        // PWARN ("importing pre-book-style XML data file");
        book_callback (tag, globaldata, data);
    }
    return TRUE;
}

static void
add_parser(const GncXmlDataType_t& data, struct file_backend* be_data)
{
    g_return_if_fail (data.version == GNC_FILE_BACKEND_VERS);

    if (be_data->ok == FALSE)
        return;

    if (data.create_parser)
        if (!sixtp_add_some_sub_parsers(
                    be_data->parser, TRUE,
                    data.type_name, (data.create_parser)(),
                    NULL, NULL))
            be_data->ok = FALSE;
}

static void
scrub (const GncXmlDataType_t& data, struct file_backend* be_data)
{
    g_return_if_fail (data.version == GNC_FILE_BACKEND_VERS);

    if (data.scrub)
        (data.scrub)(be_data->book);
}

static sixtp_gdv2*
gnc_sixtp_gdv2_new (
    QofBook* book,
    gboolean exporting,
    countCallbackFn countcallback,
    QofBePercentageFunc gui_display_fn)
{
    sixtp_gdv2* gd = g_new0 (sixtp_gdv2, 1);

    if (gd == NULL) return NULL;

    gd->book = book;
    gd->counter.accounts_loaded = 0;
    gd->counter.accounts_total = 0;
    gd->counter.books_loaded = 0;
    gd->counter.books_total = 0;
    gd->counter.commodities_loaded = 0;
    gd->counter.commodities_total = 0;
    gd->counter.transactions_loaded = 0;
    gd->counter.transactions_total = 0;
    gd->counter.prices_loaded = 0;
    gd->counter.prices_total = 0;
    gd->counter.schedXactions_loaded = 0;
    gd->counter.schedXactions_total = 0;
    gd->counter.budgets_loaded = 0;
    gd->counter.budgets_total = 0;
    gd->exporting = exporting;
    gd->countCallback = countcallback;
    gd->gui_display_fn = gui_display_fn;
    return gd;
}

static gboolean
qof_session_load_from_xml_file_v2_full (
    GncXmlBackend* xml_be, QofBook* book,
    sixtp_push_handler push_handler, gpointer push_user_data,
    QofBookFileType type)
{
    Account* root;
    Account* template_root;
    sixtp_gdv2* gd;
    sixtp* top_parser;
    sixtp* main_parser;
    sixtp* book_parser;
    struct file_backend be_data;
    gboolean retval;
    char* v2type = NULL;

    gd = gnc_sixtp_gdv2_new (book, FALSE, file_rw_feedback,
                             xml_be->get_percentage());

    top_parser = sixtp_new ();
    main_parser = sixtp_new ();
    book_parser = sixtp_new ();

    if (type == GNC_BOOK_XML2_FILE)
        v2type = g_strdup (GNC_V2_STRING);

    if (!sixtp_add_some_sub_parsers (
            top_parser, TRUE,
            v2type, main_parser,
            NULL, NULL))
    {
        g_free (v2type);
        goto bail;
    }

    g_free (v2type);

    if (!sixtp_add_some_sub_parsers (
            main_parser, TRUE,
            COUNT_DATA_TAG, gnc_counter_sixtp_parser_create (),
            BOOK_TAG, book_parser,

            /* the following are present here only to support
             * the older, pre-book format.  Basically, the top-level
             * book is implicit. */
            PRICEDB_TAG, gnc_pricedb_sixtp_parser_create (),
            COMMODITY_TAG, gnc_commodity_sixtp_parser_create (),
            ACCOUNT_TAG, gnc_account_sixtp_parser_create (),
            TRANSACTION_TAG, gnc_transaction_sixtp_parser_create (),
            SCHEDXACTION_TAG, gnc_schedXaction_sixtp_parser_create (),
            TEMPLATE_TRANSACTION_TAG, gnc_template_transaction_sixtp_parser_create (),
            NULL, NULL))
    {
        goto bail;
    }

    if (!sixtp_add_some_sub_parsers (
            book_parser, TRUE,
            BOOK_ID_TAG, gnc_book_id_sixtp_parser_create (),
            BOOK_SLOTS_TAG, gnc_book_slots_sixtp_parser_create (),
            COUNT_DATA_TAG, gnc_counter_sixtp_parser_create (),
            PRICEDB_TAG, gnc_pricedb_sixtp_parser_create (),
            COMMODITY_TAG, gnc_commodity_sixtp_parser_create (),
            ACCOUNT_TAG, gnc_account_sixtp_parser_create (),
            BUDGET_TAG, gnc_budget_sixtp_parser_create (),
            TRANSACTION_TAG, gnc_transaction_sixtp_parser_create (),
            SCHEDXACTION_TAG, gnc_schedXaction_sixtp_parser_create (),
            TEMPLATE_TRANSACTION_TAG, gnc_template_transaction_sixtp_parser_create (),
            NULL, NULL))
    {
        goto bail;
    }

    be_data.ok = TRUE;
    be_data.parser = book_parser;
    for (auto data : backend_registry)
        add_parser(data, &be_data);
    if (be_data.ok == FALSE)
        goto bail;

    /* stop logging while we load */
    xaccLogDisable ();
    xaccDisableDataScrubbing ();

    if (push_handler)
    {
        gpointer parse_result = NULL;
        gxpf_data gpdata;

        gpdata.cb = generic_callback;
        gpdata.parsedata = gd;
        gpdata.bookdata = book;

        retval = sixtp_parse_push (top_parser, push_handler, push_user_data,
                                   NULL, &gpdata, &parse_result);
    }
    else
    {
        /* Even though libxml2 knows how to decompress zipped files, we
         * do it ourself since as of version 2.9.1 it has a bug that
         * causes it to fail to decompress certain files. See
         * https://bugs.gnucash.org/show_bug.cgi?id=712528 for more
         * info.
         */
        auto filename = xml_be->get_filename();
        auto [file, thread] = try_gz_open (filename, "r",
                                           is_gzipped_file (filename), FALSE);
        if (!file)
        {
            PWARN ("Unable to open file %s", filename);
            retval = false;
        }
        else
        {
            retval = gnc_xml_parse_fd (top_parser, file,
                                       generic_callback, gd, book);
            fclose (file);
            if (thread)
                g_thread_join (thread);
        }
    }

    if (!retval)
    {
        sixtp_destroy (top_parser);
        xaccLogEnable ();
        xaccEnableDataScrubbing ();
        goto bail;
    }
    debug_print_counter_data (&gd->counter);

    /* destroy the parser */
    sixtp_destroy (top_parser);
    g_free (gd);

    xaccEnableDataScrubbing ();

    /* Mark the session as saved */
    qof_book_mark_session_saved (book);

    /* Call individual scrub functions */
    memset (&be_data, 0, sizeof (be_data));
    be_data.book = book;
    for (auto data : backend_registry)
        scrub(data, &be_data);

    /* fix price quote sources */
    root = gnc_book_get_root_account (book);
    xaccAccountTreeScrubQuoteSources (root, gnc_commodity_table_get_table (book));

    /* Fix account and transaction commodities */
    xaccAccountTreeScrubCommodities (root);

    /* Fix split amount/value */
    xaccAccountTreeScrubSplits (root);

    /* commit all groups, this completes the BeginEdit started when the
     * account_end_handler finished reading the account.
     */
    template_root = gnc_book_get_template_root (book);
    gnc_account_foreach_descendant (root,
                                    (AccountCb) xaccAccountCommitEdit,
                                    NULL);
    gnc_account_foreach_descendant (template_root,
                                    (AccountCb) xaccAccountCommitEdit,
                                    NULL);
    /* if these exist in the XML file then they will be uncommitted */
    if (qof_instance_get_editlevel(root) != 0)
        xaccAccountCommitEdit(root);
    if (qof_instance_get_editlevel(template_root) != 0)
        xaccAccountCommitEdit(template_root);

    /* start logging again */
    xaccLogEnable ();

    return TRUE;

bail:
    g_free (gd);
    return FALSE;
}

gboolean
qof_session_load_from_xml_file_v2 (GncXmlBackend* xml_be, QofBook* book,
                                   QofBookFileType type)
{
    return qof_session_load_from_xml_file_v2_full (xml_be, book, NULL, NULL, type);
}

/***********************************************************************/

static gboolean
write_counts (FILE* out, ...)
{
    va_list ap;
    const char* type;
    gboolean success = TRUE;

    va_start (ap, out);
    type = va_arg (ap, char*);

    while (success && type)
    {
        int amount = va_arg (ap, int);

        if (amount != 0)
        {
#if GNUCASH_REALLY_BUILD_AN_XML_TREE_ON_OUTPUT
            char *type_dup = g_strdup (type);
            char* val;
            xmlNodePtr node;

            val = g_strdup_printf ("%d", amount);

            node = xmlNewNode (NULL, BAD_CAST COUNT_DATA_TAG);
            /* Note: BADXML.
             *
             * This is invalid xml because the namespace isn't
             * declared in the tag itself. This should be changed to
             * 'type' at some point. */
            xmlSetProp (node, BAD_CAST "cd:type", checked_char_cast (type_dup));
            xmlNodeAddContent (node, checked_char_cast (val));
            g_free (val);
            g_free (type_dup);

            xmlElemDump (out, NULL, node);
            xmlFreeNode (node);

            if (ferror (out) || fprintf (out, "\n") < 0)
            {
                success = FALSE;
                break;
            }
#else
            if (fprintf (out, "<%s %s=\"%s\">%d</%s>\n",
                         COUNT_DATA_TAG, "cd:type", type, amount, COUNT_DATA_TAG) < 0)
            {
                success = FALSE;
                break;
            }
#endif

        }

        type = va_arg (ap, char*);
    }

    va_end (ap);
    return success;
}

static gint
compare_namespaces (gconstpointer a, gconstpointer b)
{
    const gchar* sa = (const gchar*) a;
    const gchar* sb = (const gchar*) b;
    return (g_strcmp0 (sa, sb));
}

static gint
compare_commodity_ids (gconstpointer a, gconstpointer b)
{
    const gnc_commodity* ca = (const gnc_commodity*) a;
    const gnc_commodity* cb = (const gnc_commodity*) b;
    return (g_strcmp0 (gnc_commodity_get_mnemonic (ca),
                       gnc_commodity_get_mnemonic (cb)));
}

static gboolean write_pricedb (FILE* out, QofBook* book, sixtp_gdv2* gd);
static gboolean write_transactions (FILE* out, QofBook* book, sixtp_gdv2* gd);
static gboolean write_template_transaction_data (FILE* out, QofBook* book,
                                                 sixtp_gdv2* gd);
static gboolean write_schedXactions (FILE* out, QofBook* book, sixtp_gdv2* gd);
static void write_budget (QofInstance* ent, gpointer data);

static void
write_counts(const GncXmlDataType_t& data, struct file_backend* be_data)
{
    g_return_if_fail (data.version == GNC_FILE_BACKEND_VERS);

    if (data.get_count)
        write_counts (be_data->out, data.type_name,
                      (data.get_count) (be_data->book),
                      NULL);
}

static void
write_data(const GncXmlDataType_t& data, struct file_backend* be_data)
{
    g_return_if_fail (data.version == GNC_FILE_BACKEND_VERS);

    if (data.write && !ferror(be_data->out))
        (data.write)(be_data->out, be_data->book);
}

static gboolean
write_book (FILE* out, QofBook* book, sixtp_gdv2* gd)
{
    struct file_backend be_data;

    be_data.out = out;
    be_data.book = book;
    be_data.gd = gd;
    if (fprintf (out, "<%s version=\"%s\">\n", BOOK_TAG,
                 gnc_v2_book_version_string) < 0)
        return FALSE;
    if (!write_book_parts (out, book))
        return FALSE;

    /* gd->counter.{foo}_total fields should have all these totals
       already collected.  I don't know why we're re-calling all these
       functions.  */
    if (!write_counts (out,
                       "commodity",
                       gnc_commodity_table_get_size (
                           gnc_commodity_table_get_table (book)),
                       "account",
                       1 + gnc_account_n_descendants (gnc_book_get_root_account (book)),
                       "transaction",
                       gnc_book_count_transactions (book),
                       "schedxaction",
                       g_list_length (gnc_book_get_schedxactions (book)->sx_list),
                       "budget", qof_collection_count (
                           qof_book_get_collection (book, GNC_ID_BUDGET)),
                       "price", gnc_pricedb_get_num_prices (gnc_pricedb_get_db (book)),
                       NULL))
        return FALSE;

    for (auto data : backend_registry)
        write_counts(data, &be_data);

    if (ferror (out)
        || !write_commodities (out, book, gd)
        || !write_pricedb (out, book, gd)
        || !write_accounts (out, book, gd)
        || !write_transactions (out, book, gd)
        || !write_template_transaction_data (out, book, gd)
        || !write_schedXactions (out, book, gd))

        return FALSE;

    qof_collection_foreach (qof_book_get_collection (book, GNC_ID_BUDGET),
                            write_budget, &be_data);
    if (ferror (out))
        return FALSE;

    for (auto data : backend_registry)
        write_data(data, &be_data);
    if (ferror(out))
        return FALSE;

    if (fprintf (out, "</%s>\n", BOOK_TAG) < 0)
        return FALSE;

    return TRUE;
}

gboolean
write_commodities (FILE* out, QofBook* book, sixtp_gdv2* gd)
{
    gnc_commodity_table* tbl;
    GList* namespaces;
    GList* lp;
    gboolean success = TRUE;

    tbl = gnc_commodity_table_get_table (book);

    namespaces = gnc_commodity_table_get_namespaces (tbl);
    if (namespaces)
    {
        namespaces = g_list_sort (namespaces, compare_namespaces);
    }

    for (lp = namespaces; success && lp; lp = lp->next)
    {
        GList* comms, *lp2;
        xmlNodePtr comnode;

        comms = gnc_commodity_table_get_commodities (tbl,
                                                     static_cast<const char*> (lp->data));
        comms = g_list_sort (comms, compare_commodity_ids);

        for (lp2 = comms; lp2; lp2 = lp2->next)
        {
            comnode = gnc_commodity_dom_tree_create (static_cast<const gnc_commodity*>
                                                     (lp2->data));
            if (comnode == NULL)
                continue;

            xmlElemDump (out, NULL, comnode);
            if (ferror (out) || fprintf (out, "\n") < 0)
            {
                success = FALSE;
                break;
            }

            xmlFreeNode (comnode);
            gd->counter.commodities_loaded++;
            sixtp_run_callback (gd, "commodities");
        }

        g_list_free (comms);
    }

    if (namespaces) g_list_free (namespaces);

    return success;
}

static gboolean
write_pricedb (FILE* out, QofBook* book, sixtp_gdv2* gd)
{
    xmlNodePtr node;
    xmlNodePtr parent;
    xmlOutputBufferPtr outbuf;

    parent = gnc_pricedb_dom_tree_create (gnc_pricedb_get_db (book));

    if (!parent)
    {
        return TRUE;
    }

    /* Write out the parent pricedb tag then loop to write out each price.
       We do it this way instead of just calling xmlElemDump so that we can
       increment the progress bar as we go. */

    if (fprintf (out, "<%s version=\"%s\">\n", parent->name,
                 xmlGetProp (parent, BAD_CAST "version")) < 0)
        return FALSE;

    /* We create our own output buffer so we can call xmlNodeDumpOutput to get
       the indentation correct. */
    outbuf = xmlOutputBufferCreateFile (out, NULL);
    if (outbuf == NULL)
    {
        xmlFreeNode (parent);
        return FALSE;
    }

    for (node = parent->children; node; node = node->next)
    {
        /* Write two spaces since xmlNodeDumpOutput doesn't indent the first line */
        xmlOutputBufferWrite (outbuf, 2, "  ");
        xmlNodeDumpOutput (outbuf, NULL, node, 1, 1, NULL);
        /* It also doesn't terminate the last line */
        xmlOutputBufferWrite (outbuf, 1, "\n");
        if (ferror (out))
            break;
        gd->counter.prices_loaded += 1;
        sixtp_run_callback (gd, "prices");
    }

    xmlOutputBufferClose (outbuf);

    if (ferror (out) || fprintf (out, "</%s>\n", parent->name) < 0)
    {
        xmlFreeNode (parent);
        return FALSE;
    }

    xmlFreeNode (parent);
    return TRUE;
}

static int
xml_add_trn_data (Transaction* t, gpointer data)
{
    struct file_backend* be_data = static_cast<decltype (be_data)> (data);
    xmlNodePtr node;

    node = gnc_transaction_dom_tree_create (t);

    xmlElemDump (be_data->out, NULL, node);
    xmlFreeNode (node);

    if (ferror (be_data->out) || fprintf (be_data->out, "\n") < 0)
        return -1;

    be_data->gd->counter.transactions_loaded++;
    sixtp_run_callback (be_data->gd, "transaction");
    return 0;
}

static gboolean
write_transactions (FILE* out, QofBook* book, sixtp_gdv2* gd)
{
    struct file_backend be_data;

    be_data.out = out;
    be_data.gd = gd;
    return 0 ==
           xaccAccountTreeForEachTransaction (gnc_book_get_root_account (book),
                                              xml_add_trn_data,
                                              (gpointer) &be_data);
}

static gboolean
write_template_transaction_data (FILE* out, QofBook* book, sixtp_gdv2* gd)
{
    Account* ra;
    struct file_backend be_data;

    be_data.out = out;
    be_data.gd = gd;

    ra = gnc_book_get_template_root (book);
    if (gnc_account_n_descendants (ra) > 0)
    {
        if (fprintf (out, "<%s>\n", TEMPLATE_TRANSACTION_TAG) < 0
            || !write_account_tree (out, ra, gd)
            || xaccAccountTreeForEachTransaction (ra, xml_add_trn_data, (gpointer)&be_data)
            || fprintf (out, "</%s>\n", TEMPLATE_TRANSACTION_TAG) < 0)

            return FALSE;
    }

    return TRUE;
}

static gboolean
write_schedXactions (FILE* out, QofBook* book, sixtp_gdv2* gd)
{
    GList* schedXactions;
    SchedXaction* tmpSX;
    xmlNodePtr node;

    schedXactions = gnc_book_get_schedxactions (book)->sx_list;

    if (schedXactions == NULL)
        return TRUE;

    do
    {
        tmpSX = static_cast<decltype (tmpSX)> (schedXactions->data);
        node = gnc_schedXaction_dom_tree_create (tmpSX);
        xmlElemDump (out, NULL, node);
        xmlFreeNode (node);
        if (ferror (out) || fprintf (out, "\n") < 0)
            return FALSE;
        gd->counter.schedXactions_loaded++;
        sixtp_run_callback (gd, "schedXactions");
    }
    while ((schedXactions = schedXactions->next));

    return TRUE;
}

static void
write_budget (QofInstance* ent, gpointer data)
{
    xmlNodePtr node;
    struct file_backend* file_be = static_cast<decltype (file_be)> (data);

    GncBudget* bgt = GNC_BUDGET (ent);

    if (ferror (file_be->out))
        return;

    node = gnc_budget_dom_tree_create (bgt);
    xmlElemDump (file_be->out, NULL, node);
    xmlFreeNode (node);
    if (ferror (file_be->out) || fprintf (file_be->out, "\n") < 0)
        return;

    file_be->gd->counter.budgets_loaded++;
    sixtp_run_callback (file_be->gd, "budgets");
}

gboolean
gnc_xml2_write_namespace_decl (FILE* out, const char* name_space)
{
    g_return_val_if_fail (name_space, FALSE);
    return fprintf (out, "\n     xmlns:%s=\"http://www.gnucash.org/XML/%s\"",
                    name_space, name_space) >= 0;
}

static void
write_namespace (const GncXmlDataType_t& data, FILE* out)
{
    g_return_if_fail (data.version == GNC_FILE_BACKEND_VERS);

    if (data.ns && !ferror(out))
        (data.ns)(out);
}

static gboolean
write_v2_header (FILE* out)
{
    if (fprintf (out, "<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n") < 0
        || fprintf (out, "<" GNC_V2_STRING) < 0

        || !gnc_xml2_write_namespace_decl (out, "gnc")
        || !gnc_xml2_write_namespace_decl (out, "act")
        || !gnc_xml2_write_namespace_decl (out, "book")
        || !gnc_xml2_write_namespace_decl (out, "cd")
        || !gnc_xml2_write_namespace_decl (out, "cmdty")
        || !gnc_xml2_write_namespace_decl (out, "price")
        || !gnc_xml2_write_namespace_decl (out, "slot")
        || !gnc_xml2_write_namespace_decl (out, "split")
        || !gnc_xml2_write_namespace_decl (out, "sx")
        || !gnc_xml2_write_namespace_decl (out, "trn")
        || !gnc_xml2_write_namespace_decl (out, "ts")
        || !gnc_xml2_write_namespace_decl (out, "fs")
        || !gnc_xml2_write_namespace_decl (out, "bgt")
        || !gnc_xml2_write_namespace_decl (out, "recurrence")
        || !gnc_xml2_write_namespace_decl (out, "lot"))

        return FALSE;

    /* now cope with the plugins */
    for (auto data : backend_registry)
        write_namespace(data, out);

    if (ferror (out) || fprintf (out, ">\n") < 0)
        return FALSE;

    return TRUE;
}

gboolean
gnc_book_write_to_xml_filehandle_v2 (QofBook* book, FILE* out)
{
    QofBackend* qof_be;
    sixtp_gdv2* gd;
    gboolean success = TRUE;

    if (!out) return FALSE;

    if (!write_v2_header (out)
        || !write_counts (out, "book", 1, NULL))
        return FALSE;

    qof_be = qof_book_get_backend (book);
    gd = gnc_sixtp_gdv2_new (book, FALSE, file_rw_feedback,
                             qof_be->get_percentage());
    gd->counter.commodities_total =
        gnc_commodity_table_get_size (gnc_commodity_table_get_table (book));
    gd->counter.accounts_total = 1 +
                                 gnc_account_n_descendants (gnc_book_get_root_account (book));
    gd->counter.transactions_total = gnc_book_count_transactions (book);
    gd->counter.schedXactions_total =
        g_list_length (gnc_book_get_schedxactions (book)->sx_list);
    gd->counter.budgets_total = qof_collection_count (
                                    qof_book_get_collection (book, GNC_ID_BUDGET));
    gd->counter.prices_total = gnc_pricedb_get_num_prices (gnc_pricedb_get_db (
                                                               book));

    if (!write_book (out, book, gd)
        || fprintf (out, "</" GNC_V2_STRING ">\n\n") < 0)
        success = FALSE;

    g_free (gd);
    return success;
}

/*
 * This function is called by the "export" code.
 */
gboolean
gnc_book_write_accounts_to_xml_filehandle_v2 (QofBackend* qof_be, QofBook* book,
                                              FILE* out)
{
    gnc_commodity_table* table;
    Account* root;
    int ncom, nacc;
    sixtp_gdv2* gd;
    gboolean success = TRUE;

    if (!out) return FALSE;

    root = gnc_book_get_root_account (book);
    nacc = 1 + gnc_account_n_descendants (root);

    table = gnc_commodity_table_get_table (book);
    ncom = gnc_commodity_table_get_size (table);

    if (!write_v2_header (out)
        || !write_counts (out, "commodity", ncom, "account", nacc, NULL))
        return FALSE;

    gd = gnc_sixtp_gdv2_new (book, TRUE, file_rw_feedback,
                             qof_be->get_percentage());
    gd->counter.commodities_total = ncom;
    gd->counter.accounts_total = nacc;

    if (!write_commodities (out, book, gd)
        || !write_accounts (out, book, gd)
        || fprintf (out, "</" GNC_V2_STRING ">\n\n") < 0)
        success = FALSE;

    g_free (gd);
    return success;
}

static inline gzFile
do_gzopen (const char* filename, const char* perms)
{
#ifdef G_OS_WIN32
    gzFile file;
    char* new_perms = nullptr;
    char* conv_name = g_win32_locale_filename_from_utf8 (filename);

    if (!conv_name)
    {
        g_warning ("Could not convert '%s' to system codepage",
                   filename);
        return nullptr;
    }

    if (strchr (perms, 'b'))
        new_perms = g_strdup (perms);
    else
        new_perms = g_strdup_printf ("%cb%s", *perms, perms + 1);

    file = gzopen (conv_name, new_perms);
    g_free (new_perms);
    g_free (conv_name);
    return file;
#else
    return gzopen (filename, perms);
#endif
}

constexpr uint32_t BUFLEN{4096};

static inline bool
gz_thread_write (gzFile file, gz_thread_params_t* params)
{
    bool success = true;
    gchar buffer[BUFLEN];

    while (success)
    {
        auto bytes = read (params->fd, buffer, BUFLEN);
        if (bytes > 0)
        {
            if (gzwrite (file, buffer, bytes) <= 0)
            {
                gint errnum;
                auto error = gzerror (file, &errnum);
                g_warning ("Could not write the compressed file '%s'. The error is: '%s' (%d)",
                           params->filename, error, errnum);
                success = false;
            }
        }
        else if (bytes == 0)
        {
            break;
        }
        else
        {
            g_warning ("Could not read from pipe. The error is '%s' (errno %d)",
                       g_strerror (errno) ? g_strerror (errno) : "", errno);
            success = false;
        }
    }
    return success;
}

#if COMPILER(MSVC)
#define WRITE_FN _write
#else
#define WRITE_FN write
#endif

static inline bool
gz_thread_read (gzFile file, gz_thread_params_t* params)
{
    bool success = true;
    gchar buffer[BUFLEN];

    while (success)
    {
        auto gzval = gzread (file, buffer, BUFLEN);
        if (gzval > 0)
        {
            if (WRITE_FN (params->fd, buffer, gzval) < 0)
            {
                g_warning ("Could not write to pipe. The error is '%s' (%d)",
                           g_strerror (errno) ? g_strerror (errno) : "", errno);
                success = false;
            }
        }
        else if (gzval == 0)
        {
            break;
        }
        else
        {
            gint errnum;
            const gchar* error = gzerror (file, &errnum);
            g_warning ("Could not read from compressed file '%s'. The error is: '%s' (%d)",
                       params->filename, error, errnum);
            success = false;
        }
    }
    return success;
}

/* Compress or decompress function that is to be run in a separate thread.
 * Returns 1 on success or 0 otherwise, stuffed into a pointer type. */
static gpointer
gz_thread_func (gz_thread_params_t* params)
{
    gint gzval;
    bool success = true;

    auto file = do_gzopen (params->filename, params->perms);

    if (!file)
    {
        g_warning ("Child threads gzopen failed");
        success = 0;
        goto cleanup_gz_thread_func;
    }

    if (params->write)
    {
        success = gz_thread_write (file, params);
    }
    else
    {
        success = gz_thread_read (file, params);
    }

    if ((gzval = gzclose (file)) != Z_OK)
    {
        g_warning ("Could not close the compressed file '%s' (errnum %d)",
                   params->filename, gzval);
        success = false;
    }

cleanup_gz_thread_func:
    close (params->fd);
    g_free (params->filename);
    g_free (params->perms);
    g_free (params);

    return GINT_TO_POINTER (success);
}

static std::pair<FILE*, GThread*>
try_gz_open (const char* filename, const char* perms, gboolean compress,
             gboolean write)
{
    if (strstr (filename, ".gz.") != NULL) /* its got a temp extension */
        compress = TRUE;

    if (!compress)
        return std::pair<FILE*, GThread*>(g_fopen (filename, perms),
                                          nullptr);

    {
        int filedes[2]{};

#ifdef G_OS_WIN32
        if (_pipe (filedes, 4096, _O_BINARY) < 0)
        {
#else
        /* Set CLOEXEC on the pipe FDs so that if the user runs a
         * report while saving WebKit's fork won't get an open copy
         * and keep the pipe from closing. See
         * https://bugs.gnucash.org/show_bug.cgi?id=798250. Win32
         * doesn't fork nor does it support CLOEXEC.
         */
        if (pipe (filedes) < 0 ||
            fcntl(filedes[0], F_SETFD, FD_CLOEXEC) == -1 ||
        fcntl(filedes[1], F_SETFD, FD_CLOEXEC) == -1)
        {
#endif
            g_warning ("Pipe setup failed with errno %d. Opening uncompressed file.", errno);
            if (filedes[0])
            {
                close(filedes[0]);
                close(filedes[1]);
            }

            return std::pair<FILE*, GThread*>(g_fopen (filename, perms),
                                              nullptr);
        }

        gz_thread_params_t* params = g_new (gz_thread_params_t, 1);
        params->fd = filedes[write ? 0 : 1];
        params->filename = g_strdup (filename);
        params->perms = g_strdup (perms);
        params->write = write;

        auto thread = g_thread_new ("xml_thread", (GThreadFunc) gz_thread_func,
                                    params);

        FILE* file = nullptr;

        if (!thread)
        {
            g_warning ("Could not create thread for (de)compression.");
            g_free (params->filename);
            g_free (params->perms);
            g_free (params);
            close (filedes[0]);
            close (filedes[1]);
            file = g_fopen (filename, perms);
        }
        else
        {
            if (write)
                file = fdopen (filedes[1], "w");
            else
                file = fdopen (filedes[0], "r");
        }

        return std::pair<FILE*, GThread*>(file, thread);
    }
}

gboolean
gnc_book_write_to_xml_file_v2 (QofBook* book, const char* filename,
                               gboolean compress)
{
    bool success = true;

    auto [file, thread] = try_gz_open (filename, "w", compress, TRUE);
    if (!file)
        return false;

    /* Try to write as much as possible */
    if (!gnc_book_write_to_xml_filehandle_v2 (book, file))
        success = false;

    /* Close the output stream */
    if (fclose (file))
        success = false;

    /* Optionally wait for parallel compression threads */
    if (thread)
    {
        if (g_thread_join (thread) == nullptr)
            success = false;
    }

    return success;
}

/*
 * Have to pass in the backend as this routine needs the temporary
 * backend for file export, not the real backend which could be
 * postgresql or anything else.
 */
gboolean
gnc_book_write_accounts_to_xml_file_v2 (QofBackend* qof_be, QofBook* book,
                                        const char* filename)
{
    FILE* out;
    gboolean success = TRUE;

    out = g_fopen (filename, "w");

    /* Try to write as much as possible */
    if (!out
        || !gnc_book_write_accounts_to_xml_filehandle_v2 (qof_be, book, out))
        success = FALSE;

    /* Close the output stream */
    if (out && fclose (out))
        success = FALSE;

    if (!success && !qof_be->check_error())
    {

        /* Use a generic write error code */
        qof_backend_set_error (qof_be, ERR_FILEIO_WRITE_ERROR);
    }

    return success;
}

/***********************************************************************/
static bool
is_gzipped_file (const gchar* name)
{
    unsigned char buf[2];
    int fd = g_open (name, O_RDONLY, 0);

    if (fd == -1)
    {
        return false;
    }

    if (read (fd, buf, 2) != 2)
    {
        close (fd);
        return false;
    }
    close (fd);

    /* 037 0213 are the header id bytes for a gzipped file. */
    if (buf[0] == 037 && buf[1] == 0213)
    {
        return true;
    }

    return false;
}

QofBookFileType
gnc_is_xml_data_file_v2 (const gchar* name, gboolean* with_encoding)
{
    if (is_gzipped_file (name))
    {
        gzFile file = NULL;
        char first_chunk[256];
        int num_read;

        file = do_gzopen (name, "r");

        if (file == NULL)
            return GNC_BOOK_NOT_OURS;

        num_read = gzread (file, first_chunk, sizeof (first_chunk) - 1);
        gzclose (file);

        if (num_read < 1)
            return GNC_BOOK_NOT_OURS;

        return gnc_is_our_first_xml_chunk (first_chunk, with_encoding);
    }

    return (gnc_is_our_xml_file (name, with_encoding));
}


static void
replace_character_references (gchar* string)
{
    gchar* cursor, *semicolon, *tail;
    glong number;

    for (cursor = strstr (string, "&#");
         cursor && *cursor;
         cursor = strstr (cursor, "&#"))
    {
        semicolon = strchr (cursor, ';');
        if (semicolon && *semicolon)
        {

            /* parse number */
            errno = 0;
            if (* (cursor + 2) == 'x')
            {
                number = strtol (cursor + 3, &tail, 16);
            }
            else
            {
                number = strtol (cursor + 2, &tail, 10);
            }
            if (errno || tail != semicolon || number < 0 || number > 255)
            {
                PWARN ("Illegal character reference");
                return;
            }

            /* overwrite '&' with the specified character */
            *cursor = (gchar) number;
            cursor++;
            if (* (semicolon + 1))
            {
                /* move text after semicolon the the left */
                tail = g_strdup (semicolon + 1);
                strcpy (cursor, tail);
                g_free (tail);
            }
            else
            {
                /* cut here */
                *cursor = '\0';
            }

        }
        else
        {
            PWARN ("Unclosed character reference");
            return;
        }
    }
}

static void
conv_free (conv_type* conv)
{
    if (conv)
    {
        g_free (conv->utf8_string);
        g_free (conv);
    }
}

static void
conv_list_free (GList* conv_list)
{
    g_list_foreach (conv_list, (GFunc) conv_free, NULL);
    g_list_free (conv_list);
}

typedef struct
{
    GQuark encoding;
    GIConv iconv;
} iconv_item_type;

gint
gnc_xml2_find_ambiguous (const gchar* filename, GList* encodings,
                         GHashTable** unique, GHashTable** ambiguous,
                         GList** impossible)
{
    GList* iconv_list = NULL, *conv_list = NULL, *iter;
    iconv_item_type* iconv_item = NULL, *ascii = NULL;
    const gchar* enc;
    GHashTable* processed = NULL;
    gint n_impossible = 0;
    GError* error = NULL;
    gboolean clean_return = FALSE;

    auto [file, thread] = try_gz_open (filename, "r",
                                       is_gzipped_file (filename), FALSE);
    if (file == NULL)
    {
        PWARN ("Unable to open file %s", filename);
        goto cleanup_find_ambs;
    }

    /* we need ascii */
    ascii = g_new (iconv_item_type, 1);
    ascii->encoding = g_quark_from_string ("ASCII");
    ascii->iconv = g_iconv_open ("UTF-8", "ASCII");
    if (ascii->iconv == (GIConv) - 1)
    {
        PWARN ("Unable to open ASCII ICONV conversion descriptor");
        goto cleanup_find_ambs;
    }

    /* call iconv_open on encodings */
    for (iter = encodings; iter; iter = iter->next)
    {
        iconv_item = g_new (iconv_item_type, 1);
        iconv_item->encoding = GPOINTER_TO_UINT (iter->data);
        if (iconv_item->encoding == ascii->encoding)
        {
            continue;
        }

        enc = g_quark_to_string (iconv_item->encoding);
        iconv_item->iconv = g_iconv_open ("UTF-8", enc);
        if (iconv_item->iconv == (GIConv) - 1)
        {
            PWARN ("Unable to open IConv conversion descriptor for '%s'", enc);
            g_free (iconv_item);
            goto cleanup_find_ambs;
        }
        else
        {
            iconv_list = g_list_prepend (iconv_list, iconv_item);
        }
    }

    /* prepare data containers */
    if (unique)
        *unique = g_hash_table_new_full (g_str_hash, g_str_equal, g_free,
                                         (GDestroyNotify) conv_free);
    if (ambiguous)
        *ambiguous = g_hash_table_new_full (g_str_hash, g_str_equal, g_free,
                                            (GDestroyNotify) conv_list_free);
    if (impossible)
        *impossible = NULL;
    processed = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL);

    /* loop through lines */
    while (1)
    {
        gchar line[256], *word, *utf8;
        gchar** word_array, **word_cursor;
        conv_type* conv = NULL;

        if (!fgets (line, sizeof (line) - 1, file))
        {
            if (feof (file))
            {
                break;
            }
            else
            {
                goto cleanup_find_ambs;
            }
        }

        g_strchomp (line);
        replace_character_references (line);
        word_array = g_strsplit_set (line, "> <", 0);

        /* loop through words */
        for (word_cursor = word_array; *word_cursor; word_cursor++)
        {
            word = *word_cursor;
            if (!word)
                continue;

            utf8 = g_convert_with_iconv (word, -1, ascii->iconv,
                                         NULL, NULL, &error);
            if (utf8)
            {
                /* pure ascii */
                g_free (utf8);
                continue;
            }
            g_error_free (error);
            error = NULL;

            if (g_hash_table_lookup_extended (processed, word, NULL, NULL))
            {
                /* already processed */
                continue;
            }

            /* loop through encodings */
            conv_list = NULL;
            for (iter = iconv_list; iter; iter = iter->next)
            {
                iconv_item = static_cast<decltype (iconv_item)> (iter->data);
                utf8 = g_convert_with_iconv (word, -1, iconv_item->iconv,
                                             NULL, NULL, &error);
                if (utf8)
                {
                    conv = g_new (conv_type, 1);
                    conv->encoding = iconv_item->encoding;
                    conv->utf8_string = utf8;
                    conv_list = g_list_prepend (conv_list, conv);
                }
                else
                {
                    g_error_free (error);
                    error = NULL;
                }
            }

            /* no successful conversion */
            if (!conv_list)
            {
                if (impossible)
                    *impossible = g_list_append (*impossible, g_strdup (word));
                n_impossible++;
            }

            /* more than one successful conversion */
            else if (conv_list->next)
            {
                if (ambiguous)
                {
                    g_hash_table_insert (*ambiguous, g_strdup (word), conv_list);
                }
                else
                {
                    conv_list_free (conv_list);
                }
            }

            /* only one successful conversion */
            else
            {
                if (unique)
                {
                    g_hash_table_insert (*unique, g_strdup (word), conv);
                }
                else
                {
                    conv_free (conv);
                }
                g_list_free (conv_list);
            }

            g_hash_table_insert (processed, g_strdup (word), NULL);
        }
        g_strfreev (word_array);
    }

    clean_return = TRUE;

cleanup_find_ambs:

    if (iconv_list)
    {
        for (iter = iconv_list; iter; iter = iter->next)
        {
            if (iter->data)
            {
                g_iconv_close (((iconv_item_type*) iter->data)->iconv);
                g_free (iter->data);
            }
        }
        g_list_free (iconv_list);
    }
    if (processed)
        g_hash_table_destroy (processed);
    if (ascii)
        g_free (ascii);
    if (file)
    {
        fclose (file);
        if (thread)
            g_thread_join (thread);
    }

    return (clean_return) ? n_impossible : -1;
}

typedef struct
{
    const char* filename;
    GHashTable* subst;
} push_data_type;

static void
parse_with_subst_push_handler (xmlParserCtxtPtr xml_context,
                               push_data_type* push_data)
{
    GIConv ascii = (GIConv) - 1;
    GString* output = NULL;
    GError* error = NULL;

    auto filename = push_data->filename;
    auto [file, thread] = try_gz_open (filename, "r",
                                       is_gzipped_file (filename), FALSE);
    if (!file)
    {
        PWARN ("Unable to open file %s", filename);
        goto cleanup_push_handler;
    }

    ascii = g_iconv_open ("UTF-8", "ASCII");
    if (ascii == (GIConv) - 1)
    {
        PWARN ("Unable to open ASCII ICONV conversion descriptor");
        goto cleanup_push_handler;
    }

    /* loop through lines */
    while (1)
    {
        gchar line[256], *word, *repl, *utf8;
        gint pos, len;
        gchar* start, *cursor;

        if (!fgets (line, sizeof (line) - 1, file))
        {
            if (feof (file))
            {
                break;
            }
            else
            {
                goto cleanup_push_handler;
            }
        }

        replace_character_references (line);
        output = g_string_new (line);

        /* loop through words */
        cursor = output->str;
        pos = 0;
        while (1)
        {
            /* ignore delimiters */
            while (*cursor == '>' || *cursor == ' ' || *cursor == '<' ||
                   *cursor == '\n')
            {
                cursor++;
                pos += 1;
            }

            if (!*cursor)
                /* welcome to EOL */
                break;

            /* search for a delimiter */
            start = cursor;
            len = 0;
            while (*cursor && *cursor != '>' && *cursor != ' ' && *cursor != '<' &&
                   *cursor != '\n')
            {
                cursor++;
                len++;
            }

            utf8 = g_convert_with_iconv (start, len, ascii, NULL, NULL, &error);

            if (utf8)
            {
                /* pure ascii */
                g_free (utf8);
                pos += len;
            }
            else
            {
                g_error_free (error);
                error = NULL;

                word = g_strndup (start, len);
                repl = static_cast<decltype (repl)> (g_hash_table_lookup (push_data->subst,
                                                                          word));
                g_free (word);
                if (repl)
                {
                    /* there is a replacement */
                    output = g_string_insert (g_string_erase (output, pos, len),
                                              pos, repl);
                    pos += strlen (repl);
                    cursor = output->str + pos;
                }
                else
                {
                    /* there is no replacement, return immediately */
                    goto cleanup_push_handler;
                }
            }
        }

        if (xmlParseChunk (xml_context, output->str, output->len, 0) != 0)
        {
            goto cleanup_push_handler;
        }
    }

    /* last chunk */
    xmlParseChunk (xml_context, "", 0, 1);

cleanup_push_handler:

    if (output)
        g_string_free (output, TRUE);
    if (ascii != (GIConv) - 1)
        g_iconv_close (ascii);
    if (file)
    {
        fclose (file);
        if (thread)
            g_thread_join (thread);
    }
}

gboolean
gnc_xml2_parse_with_subst (GncXmlBackend* xml_be, QofBook* book, GHashTable* subst)
{
    push_data_type* push_data;
    gboolean success;

    push_data = g_new (push_data_type, 1);
    push_data->filename = xml_be->get_filename();
    push_data->subst = subst;

    success = qof_session_load_from_xml_file_v2_full (
                  xml_be, book, (sixtp_push_handler) parse_with_subst_push_handler,
                  push_data, GNC_BOOK_XML2_FILE);
    g_free (push_data);

    if (success)
        qof_instance_set_dirty (QOF_INSTANCE (book));

    return success;
}