Bugzilla Administration/BugzillaPM

From GnuCash
Jump to: navigation, search
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
#
# This Source Code Form is "Incompatible With Secondary Licenses", as
# defined by the Mozilla Public License, v. 2.0.
#
# This is based off Gnats, but designed to pull from Gnome Bugzilla to
# migrate GnuCash entries.  This uses the Bugzilla JSON API to 
#
# Re-written by Derek Atkins <derek@ihtfp.com>
#
# Notes:
# * may need to set ulimit:   ulimit -n 16384
# * When running the migration script ensure you set TZ=America/New_York
#

package Bugzilla::Migrate::Bugzilla;
use strict;
use base qw(Bugzilla::Migrate);

use Bugzilla::Constants;
use Bugzilla::Install::Util qw(indicate_progress);
use Bugzilla::Util qw(format_time trim generate_random_password);

use File::Basename;
use IO::File;
use List::MoreUtils qw(firstidx);
use List::Util qw(first);
use JSON;
use MIME::Base64;
use Date::Manip::Date;

my %duplicates;
my %depends_on;
my %attach_status;

use constant REQUIRED_MODULES => [
#    {
#        package => 'Email-Simple-FromHandle',
#        module  => 'Email::Simple::FromHandle',
#        # This version added seekable handles.
#        version => 0.050,
#    },
];

use constant NON_COMMENT_FIELDS => (
    'alias',
    'attachments',
    'blocks',
    'cf_gnome_target',
    'cf_gnome_version',
    'comment',
    'comments',
    'creation_time',
    'creator',
    'depends_on',
    'dupe_of',
    'flags',
    'groups',
    'history',
    'id',
    'is_cc_accessible',
    'is_confirmed',
    'is_creator_accessible',
    'is_open',
    'last_change_time',
    'platform',
    'severity',
    'status',
    'summary',
    'url',
    'whiteboard',
    );

use constant FIELD_MAP => {
};

use constant VALUE_MAP => {
    bug_severity => {
    },
    bug_status => {
    },
    bug_status_resolution => {
    },
    priority => {
    },
};

use constant BZ_CONFIG_VARS => (
    {
        name    => 'data_path',
        default => '/root/Bugzilla/data',
        desc    => <<END,
# The path to the directory that contains the BZ database JSON data.
END
    },
    {
        name    => 'timezone',
        default => 'UTC',
        desc    => <<END,
# Default Time Zone.
END
    },
);

sub CONFIG_VARS {
    my $self = shift;
    my @vars = (BZ_CONFIG_VARS, $self->SUPER::CONFIG_VARS);
    my $field_map = first { $_->{name} eq 'translate_fields' } @vars;
    $field_map->{default} = FIELD_MAP;
    my $value_map = first { $_->{name} eq 'translate_values' } @vars;
    $value_map->{default} = VALUE_MAP;
    return @vars;
}

#########
# Hooks #
#########

#
# BEFORE_INSERT -- create Keywords!
#
sub before_insert {
    my $self = shift;
    my $dbh  = Bugzilla->dbh;

    my $path = $self->config('data_path');
    my $file =  "$path/bug_fields.json";
    $self->debug("Reading bug fields from $file");
    open(my $fields_fh, '<', $file) || die "$file: $!";
    my $json = decode_json <$fields_fh>;
    close($fields_fh);

    # Process all the fields
    foreach my $field (@{$json}) {
	#print "Testing field " . $field->{name} . "\n";
	if ($field->{name} eq "keywords") {
	    # Insert all the keywords
	    foreach my $keyword (@{$field->{values}}) {
		#print "Inserting keyword: " . $keyword->{name} . "\n";
		Bugzilla::Keyword->create($keyword);
	    }
	}
    }
    #die "Testing done.\n";

    # Create new bug_status fields:
    #    value   => "NEW",
    #    sortkey => "10",
    #    is_open => "1",  # "0" == closed, requires resolution
    my $statuses = [
	{ value => "NEW", sortkey => 0, is_open => 1 },
	{ value => "ASSIGNED", sortkey => 20, is_open => 1 },
	{ value => "NEEDINFO", sortkey => 40, is_open => 1 },
	{ value => "REOPENED", sortkey => 60, is_open => 1 },
	];
    foreach my $status (@$statuses) {
	Bugzilla::Field::Choice->type("bug_status")->create($status);
    }
    
    # Remove the IN_PROGRESS and CONFIRMED bug_status -- we don't use them
    foreach my $status ("IN_PROGRESS", "CONFIRMED") {
	my $value = Bugzilla::Field::Choice->type("bug_status")->check($status);
	$value->remove_from_db() if ($value);
    }

    # Create new resolution fields
    my $resolutions = [
	{ value => "NOTABUG", sortkey => 520 },
	{ value => "NOTGNUCASH", sortkey =>  540 },
	{ value => "INCOMPLETE", sortkey => 600 },
	{ value => "OBSOLETE", sortkey => 640 },
	];
    foreach my $resolution (@$resolutions) {
	Bugzilla::Field::Choice->type("resolution")->create($resolution);
    }

    # Remove the WORKSFORME resolution -- we don't use it
    foreach my $res ("WORKSFORME") {
	my $value = Bugzilla::Field::Choice->type("resolution")->check($res);
	$value->remove_from_db() if ($value);
    }

    #
    # Update the work flow matrix
    #
    print "Updating workflow...\n";
    my $workflow =  {
	"Start" => {"NEW" => 1},
	"ASSIGNED" => {"NEEDINFO" => 1, "NEW" => 1},
	"NEEDINFO" => {"ASSIGNED" => 1, "NEW" => 1},
	"NEW" => {"ASSIGNED" => 1, "NEEDINFO" => 1},
	"REOPENED" => {"ASSIGNED" => 1, "NEEDINFO" => 1},
	"RESOLVED" => {"REOPENED" => 1},
	"VERIFIED" => {"REOPENED" => 1}
    };

    # Get the list of bug_status IDs
    my $statuses;
    $statuses->{"Start"} = undef;
    foreach my $status (keys %{$workflow}) {
	next if ($status eq "Start");
	my $value = Bugzilla::Field::Choice->type("bug_status")->check($status);
	$statuses->{$status} = $value->id if ($value);
    }

    # Insert new workflow mappings
    my $sth_insert = $dbh->prepare('INSERT INTO status_workflow (old_status, new_status)                                                                       
				   VALUES (?, ?)');
    foreach my $status (keys %{$workflow}) {
	foreach my $dst (keys %{$workflow->{$status}}) {
	    print "Adding link from $status -> $dst\n";
	    $sth_insert->execute($statuses->{$status}, $statuses->{$dst});
	}
    }

    # Create GnuCash group
    my $gnc_devs = Bugzilla::Group->create({
        name        => "GnuCash_developers",
        description => "List of GnuCash Developers",
        isactive    => 1,
        isbuggroup  => 1,
			    });

    # Create Attachment Status Flag(s)
    foreach my $flag (
	{
	    name => "needs-work",
	    desc => "A GnuCash developer has reviewed the proposed patch and requested changes",
	    sort => "101",
	},
	{
	    name => "accepted-commit_now",
	    desc => "The attachment is a patch and has been approved for committing by a GnuCash developer",
	    sort => "102"
	},
	{
	    name => "accepted-commit_after_freeze",
	    desc => "The attachment is a patch and has been approved for committing by a GnuCash developer after the code freeze lifts",
	    sort => "103"
	},
	{
	    name => "committed",
	    desc => "The patch has been committed to the canonical repository",
	    sort => "104"
	},
	{
	    name => "rejected",
	    desc => "The proposed patch has been reviewed by a GnuCash developer and found irreparably unsuitable for incorporation into GnuCash.",
	    sort => "105"
	}) {

	Bugzilla::FlagType->create({
	    name        => $flag->{name},
	    description => $flag->{desc},
	    target_type => "attachment",
	    sortkey     => $flag->{sort},
	    is_active   => 1,
	    is_requestable   => 0,
	    is_requesteeble  => 0,
	    is_multiplicable => 0,
	    grant_group      => "GnuCash_developers",
	    inclusions       => ["0:0"],
	    exclusions       => []
				   });
    }
}

#
# AFTER INSERT -- Stuff to be done after we're done inserting data
#
sub after_insert {
    my $self = shift;
    my $dbh  = Bugzilla->dbh;

    # We need to populate the duplicate database entries afterwards
    # instead of during the import.  So let's do that now
    foreach my $key (keys %duplicates) {
	print "Adding duplicate: $key -> $duplicates{$key}\n";
	$dbh->do('INSERT INTO duplicates (dupe, dupe_of) VALUES (?,?)',
		 undef, $key, $duplicates{$key});
    }

    # Populate the dependcies database
    foreach my $key (keys %depends_on) {
	foreach my $depends_on_id (@{$depends_on{$key}}) {
	    print "Adding dependency: $key -> $depends_on_id\n";
	    $dbh->do('INSERT INTO dependencies (blocked, dependson) VALUES (?, ?)',
		     undef, $key, $depends_on_id);
	}
    }
    
    # Reset the Super User because we added new groups!
    Bugzilla->set_user(Bugzilla::User->super_user);

    # Give GnuCash_developers permissions on GnuCash product
    #  Entry Mem   Oth  Canedit editcomp canconf editbugs Bugs
    #   []  Shown Shown   []      [X]      [X]     [X]     1
    print "Adding Group Controls.\n";
    my $group = Bugzilla::Group->new({name => "GnuCash_developers"});
    foreach my $prodname ("GnuCash", "Website", "Documentation", "Packaging") {
	my $product = Bugzilla::Product->new({name => $prodname});
	$product->set_group_controls($group, {
	    entry          => 0,
	    membercontrol  => 1,  # Shown
	    othercontrol   => 1,  # Shown
	    canedit        => 0,
	    editcomponents => 1,
	    canconfirm     => 1,
	    editbugs       => 1
				     });
	$product->update;
    }

    # Insert users into GnuCash_developers group
    # Names acquired from:
    #   https://bugzilla.gnome.org/page.cgi?id=browse.html&product=GnuCash
    print "Adding users to GnuCash_developers Group.\n";
    #print "Group membership: " . GROUP_MEMBERSHIP . "\n";
    #print "Group bless: " . GROUP_BLESS . "\n";

    # These are users that have set flags on attachments but
    # technically do not have permission to do so.  So let's
    # give them permission and then take it away.
    my @tempusers = ("yasuakit\@gmail.com", "tim\@filmchicago.org", "fred+gnome\@resel.fr", "matt_graham2001\@hotmail.com");

    foreach my $username ("linas\@linas.org", "alex.aycinena\@gmail.com", "andi5.py\@gmx.net", "andrew\@swclan.homelinux.org", "benoitg\@coeus.ca", "cedayiv\@gmail.com", "chris\@wilddev.net", "chris.shoemaker\@cox.net", "stimming\@tuhh.de", "cri79\@ngi.it", "bugzilla\@love2code.net", "warlord\@MIT.EDU", "frank.h.ellenberger\@gmail.com", "info\@kobaltwit.be", "jralls\@ceridwen.fremont.ca.us", "jsled\@asynchronous.org", "joslwah\@gmail.com", "usselmann.m\@icg-online.de", "micha\@lenk.info", "mta\@umich.edu", "mikee\@saxicola.idps.co.uk", "phil.longstaff\@gmail.com", "robgowin\@gmail.com", "tim\@thewunders.org", "tbullock\@nd.edu", "yawar.amin\@gmail.com", @tempusers) {
	my $user = new Bugzilla::User ({name => $username});
	if ($user) {
	    # This is a hack because set_group() and set_bless_groups
	    #  throws an error when I update.  Specifically it does not
	    #  like is_bless != 1.  However, the table default is 0,
	    #  so if we just supply the 'string' here (yay perl!)
	    #  then it bypasses the User.pm logging logic.
	    $user->{_group_changes}{GROUP_MEMBERSHIP} = [[],[$group]];
	    my $is_bless = GROUP_BLESS;
	    $user->{_group_changes}{$is_bless} = [[],[$group]];
	    $user->update();
	}
    }

    # Add Admin Users
    $group = Bugzilla::Group->new({name => "admin"});
    foreach my $username ("jralls\@ceridwen.fremont.ca.us", "frank.h.ellenberger\@gmail.com", "info\@kobaltwit.be") {
	my $user = new Bugzilla::User ({name => $username});
	if ($user) {
	    $user->{_group_changes}{GROUP_MEMBERSHIP} = [[],[$group]];
	    $user->update();
	}
    }

    # (Re)set attachment status flags
    #     perl -e 'use lib qw(/usr/share/bugzilla); use Bugzilla; use Bugzilla::Attachment; Bugzilla->set_user(Bugzilla::User->super_user); my $at = Bugzilla::Attachment->new({id=> 36, cache=>1}); print join " ", sort keys %{$at->flag_types} . "\n"'
    foreach my $attach_id (sort keys %attach_status) {
	my $attach = new Bugzilla::Attachment({ id => $attach_id, cache => 1 });
	my $flags = $attach->flag_types;
	my $flag_id;
	foreach my $flag (@$flags) {
	    if ($flag->{name} eq $attach_status{$attach_id}->{status}) {
		$flag_id = $flag->{id};
		last;
	    }
	}

	if ($flag_id) {
	    my $who = Bugzilla::User->check($attach_status{$attach_id}->{who});
	    Bugzilla->set_user($who);
	    print "Setting Attachment #$attach_id (user: " . $attach_status{$attach_id}->{who} . ") flag: " . $attach_status{$attach_id}->{status} . "\n";
	    $attach->set_flags([{type_id => $flag_id, status => '+'}], []);
	    $attach->update($attach_status{$attach_id}->{when});
	    Bugzilla->set_user(Bugzilla::User->super_user);
	} else {
	    print "Did not find attachment status: " . $attach_status{$attach_id}->{status} . " for attachment #$attach_id\n";
	}
    }
    
    # Remove the temp-users from the GnuCash Users group!!
    foreach my $username (@tempusers) {
	my $user = new Bugzilla::User ({name => $username});
	if ($user) {
	    $user->{_group_changes}{GROUP_MEMBERSHIP} = [[$group],[]];
	    my $is_bless = GROUP_BLESS;
	    $user->{_group_changes}{$is_bless} = [[$group],[]];
	    $user->update();
	}
    }
    
    # Delete TestProduct
    my $test_product = Bugzilla::Product->new({name => "TestProduct"});
    $test_product->remove_from_db({delete_series => 1}) if ($test_product);
}

#########
# Users #
#########

sub _read_users {
    my $self = shift;
    my $path = $self->config('data_path');
    my $file =  "$path/users.json";
    $self->debug("Reading users from $file");
    open(my $users_fh, '<', $file) || die "$file: $!";
    my $json = decode_json <$users_fh>;
    close($users_fh);

    # Convert user data to required format
    # Change @gnome.bugs to @gnucash.bugs
    foreach my $user (@{$json}) {
	$user->{login_name} = delete $user->{name};
	$user->{login_name} =~ s/\@gnome.bugs/\@gnucash.bugs/;
	$user->{realname} = delete $user->{real_name};
	delete $user->{id};
    }
    
    return $json;
}

############
# Products #
############

sub _read_products {
    my $self = shift;
    my $path = $self->config('data_path');
    my $file =  "$path/products.json";
    $self->debug("Reading categories from $file");

    open(my $products_fh, '<', $file) || die "$file: $!";
    my $json = decode_json <$products_fh>;
    close($products_fh);

    # Convert product data to required format
    my $gnc_prod;
    foreach my $prod (@{$json}) {
	if ($prod->{name} eq "GnuCash") { $gnc_prod = $prod; }
	$prod->{isactive} = delete $prod->{is_active};
	$prod->{allows_unconfirmed} = delete $prod->{has_unconfirmed};
	$prod->{defaultmilestone} = delete $prod->{default_milestone};
	delete $prod->{versions};  # gets created from bug data
	delete $prod->{milestones};# gets created from bug data
	delete $prod->{id};
	$prod->{version} = "3.0";  # XXX

	foreach my $comp (@{$prod->{components}}) {
	    $comp->{initialowner} = delete $comp->{default_assigned_to};
	    $comp->{initialowner} =~ s/\@gnome.bugs/\@gnucash.bugs/;
	    $comp->{initialqacontact} = delete $comp->{default_qa_contact};
	    $comp->{initialqacontact} =~ s/\@gnome.bugs/\@gnucash.bugs/;
	    $comp->{isactive} = delete $comp->{is_active};
	    delete $comp->{sort_key};
	    delete $comp->{flag_types};
	    delete $comp->{id};
	}
    }

    # GnuCash-specific changes:
    #  Re-apply initial_cc to components
    my $initial_cc = {
	"Backend - SQL" => ["gnucash-core-maint\@gnucash.bugs", "jralls\@ceridwen.fremont.ca.us", "phil.longstaff\@gmail.com"],
	    "Backend - XML" => ["gnucash-core-maint\@gnucash.bugs", "stimming\@tuhh.de"],
	    "Budgets" => ["gnucash-core-maint\@gnucash.bugs"],
	    "Build system" => ["gnucash-core-maint\@gnucash.bugs", "jralls\@ceridwen.fremont.ca.us", "stimming\@tuhh.de", "warlord\@MIT.EDU"],
	    "Business" => ["gnucash-core-maint\@gnucash.bugs", "stimming\@tuhh.de", "warlord\@MIT.EDU"],
	    "Check Printing" => ["gnucash-reports-maint\@gnucash.bugs"],
	    "Currency and Commodity" => ["frank.h.ellenberger\@gmail.com", "gnucash-core-maint\@gnucash.bugs"],
	    "Documentation" => ["gnucash-documentation-maint\@gnucash.bugs"],
	    "Engine" => ["gnucash-core-maint\@gnucash.bugs", "jralls\@ceridwen.fremont.ca.us", "stimming\@tuhh.de", "warlord\@MIT.EDU"],
	    "General" => ["gnucash-general-maint\@gnucash.bugs", "stimming\@tuhh.de"],
	    "Import - AqBanking" => ["gnucash-import-maint\@gnucash.bugs", "micha\@lenk.info", "stimming\@tuhh.de"],
	    "Import - CSV" => ["gnucash-import-maint\@gnucash.bugs", "stimming\@tuhh.de"],
	    "Import - OFX" => ["benoitg\@coeus.ca", "gnucash-import-maint\@gnucash.bugs", "stimming\@tuhh.de"],
	    "Import - Other" => ["gnucash-import-maint\@gnucash.bugs", "stimming\@tuhh.de"],
	    "Import - QIF" => ["gnucash-import-maint\@gnucash.bugs", "warlord\@MIT.EDU"],
	    "Import - QSF" => ["gnucash-import-maint\@gnucash.bugs"],
	    "MacOS" => ["gnucash-mac-maint\@gnucash.bugs", "jralls\@ceridwen.fremont.ca.us"],
	    "Python Bindings" => ["gnucash-core-maint\@gnucash.bugs"],
	    "Regist-2" => ["14ubobit\@gmail.com", "gnucash-ui-maint\@gnucash.bugs"],
	    "Register" => ["gnucash-ui-maint\@gnucash.bugs", "stimming\@tuhh.de"],
	    "Reports" => ["gnucash-reports-maint\@gnucash.bugs", "stimming\@tuhh.de"],
	    "Scheduled Transactions" => ["gnucash-core-maint\@gnucash.bugs", "stimming\@tuhh.de"],
	    "Translations" => ["gnucash-documentation-maint\@gnucash.bugs", "stimming\@tuhh.de"],
	    "TXF Export" => ["alex.aycinena\@gmail.com", "frank.h.ellenberger\@gmail.com", "gnucash-import-maint\@gnucash.bugs"],
	    "User Interface General" => ["gnucash-ui-maint\@gnucash.bugs", "stimming\@tuhh.de"],
	    "Website" => ["gnucash-documentation-maint\@gnucash.bugs"],
	    "Windows" => ["gnucash-win-maint\@gnucash.bugs", "info\@kobaltwit.be", "jralls\@ceridwen.fremont.ca.us", "stimming\@tuhh.de"]
    };
    foreach my $comp (@{$gnc_prod->{components}}) {
	$comp->{initial_cc} = $initial_cc->{$comp->{name}};
    }
    
    #    Split out Documentation, Website, and Packaging into new products
    # Steps (each):
    #   1. duplicate the main hash
    #   2. update the name and description
    #   3. update the relevent components
    # Then remove the "duplicate" components from the main GnuCash product
    # and push these onto the response array.
    my @indices;
    my $index;

    # Documentation
    my %doc_prod = %{$gnc_prod};
    $doc_prod{name} = "Documentation";
    $doc_prod{description} = "GnuCash Documentation: User Guide, Help, Man Pages, Tip of the Day";
    $doc_prod{components} = [];
    $index = 0;
    foreach my $comp (@{$gnc_prod->{components}}) {
	if ($comp->{name} eq "Documentation") {
	    push @indices, $index;
	    foreach my $upd ({n=>"Help", d=>"Bugs in the User Help"},
			     {n=>"Guide", d=>"Bugs in the User Guide and Tutorial"},
			     {n=>"Man Pages", d=>"Bugs in the GnuCash Man Pages"},
			     {n=>"Tip of the Day", d=>"Bugs in the GnuCash Tips of the Day"}) {
		my %comp = %{$comp};
		$comp{name} = $upd->{n};
		$comp{description} = $upd->{d};
		push @{$doc_prod{components}}, \%comp;
	    }
	}
	$index++;
    }

    # Website
    my %web_prod = %{$gnc_prod};
    $web_prod{name} = "Website";
    $web_prod{description} = "Bugs specific to the GnuCash website (www.gnucash.org) and other web services";
    $web_prod{components} = [];
    $index = 0;
    foreach my $comp (@{$gnc_prod->{components}}) {
	if ($comp->{name} eq "Website") {
	    push @indices, $index;
	    my %comp = %{$comp};
	    $comp{name} = "Translations";
	    $comp{description} = "Bugs in the GnuCash Website translations";
	    push @{$web_prod{components}}, $comp;
	    push @{$web_prod{components}}, \%comp;
	}
	$index++;
    }

    # Packaging -- move MacOS and Windows
    my %pkg_prod = %{$gnc_prod};
    $pkg_prod{name} = "Packaging";
    $pkg_prod{description} = "Bugs specific the GnuCash OS-specific packaging";
    $pkg_prod{components} = [];
    $index = 0;
    foreach my $comp (@{$gnc_prod->{components}}) {
	if ($comp->{name} eq "MacOS" || $comp->{name} eq "Windows") {
	    #push @indices, $index;
	    my %comp = %{$comp};
	    push @{$pkg_prod{components}}, \%comp;
	}
	$index++;
    }

    # Push the new GnuCash products into the list
    push @{$json}, \%doc_prod, \%web_prod, \%pkg_prod;
    
    # Remove the indices of moved components from the gnucash product
    for ( sort { $b <=> $a } @indices ) {
        splice @{$gnc_prod->{components}}, $_, 1;
    }
    
    return $json;
}

################
# Reading Bugs #
################

sub bug_has_text(@@) {
    my ($bug, $text) = @_;
    return 1 if ($bug->{summary} =~ m/$text/i);
    for my $com (@{$bug->{comments}}) {
	return 1 if ($com->{text} =~ m/$text/i);
    }
    return 0;
}

sub _read_bugs {
    my $self = shift;
    my $path = $self->config('data_path');
    my @buglist = glob("$path/bugs/bug_*.json");
    my @bugs;
    my $count = 1;
    my $total = scalar(@buglist);

    ## XXX:
    #return [];
    
    foreach my $bugfile (@buglist) {
	$self->debug("Reading $bugfile");
	open (my $bug_fh, '<', $bugfile) || die "$bugfile: $!";
	my $json = decode_json <$bug_fh>;
	close($bug_fh);
        if (!$self->verbose) {
            indicate_progress({ current => $count++, every => 5,
                                total => $total });
	}

	# Attachment data must be stored as a new_tmpfile in IO::File
	foreach my $a (@{$json->{attachments}}) {
	    my $decoded = decode_base64($a->{data});
	    my $temp_fh = IO::File->new_tmpfile or die ("Can't create tempfile: $!");
	    $temp_fh->binmode;
	    print $temp_fh $decoded;
	    $a->{data} = $temp_fh;
	    
	    # Reset bogus mime-type
	    if ($a->{content_type} eq "application/gnucash data") {
		$a->{content_type} = "application/gnucash";
	    }
	}

	#remove null alias
	delete $json->{alias} unless (defined $json->{alias} && $json->{alias} != '');

	# GnuCash specific -- translate bugs into the new products/categories
	if ($json->{component} eq "Windows" || $json->{component} eq "MacOS") {
	    $json->{product} = "Packaging" if bug_has_text($json, "gnucash-on-");

	} elsif ($json->{component} eq "Website") {
	    $json->{product} = "Website";
	    $json->{component} = "Translations" if bug_has_text($json, "translation");

	} elsif ($json->{component} eq "Documentation") {
	    $json->{product} = "Documentation";
	    if (bug_has_text($json, "man page") || bug_has_text($json, "manpage")) {
		$json->{component} = "Man Pages";
	    } elsif (bug_has_text($json, "tips?[ -]?of[ -]?the[ -]?day") ||
		bug_has_text($json, "daily tip")) {
		$json->{component} = "Tip of the Day";
	    } elsif (bug_has_text($json, "help")) {
		$json->{component} = "Help";
	    } else {
		$json->{component} = "Guide";
	    }
	}
	
        push(@bugs, $json);
    }
    
    # Sort the list for proper ordering during import
    # Start by ordering by number
    @bugs = sort { $a->{id} <=> $b->{id} } @bugs;

    # Next: we need to make sure that "dependent" bugs get added later
    #  i.e., if bug#A blocks bug#B, then B must get inserted first
    #  so first, find all bugs with no external dependencies
    my @orderedbugs;
    my %usedbugs;
    my %gncbugs;
    # first, find all bugs that have no dependencies
    @buglist = ();
    foreach my $bug (@bugs) {
	$gncbugs{$bug->{id}} = 1;
	if (!$bug->{dupe_of} &&
	    #scalar(@{$bug->{blocks}}) == 0
	    #scalar(@{$bug->{see_also}}) == 0
	    scalar(@{$bug->{depends_on}}) == 0
	    ) {
	    push @orderedbugs, $bug;
	    $usedbugs{$bug->{id}} = 1;
	} else {
	    push @buglist, $bug;
	}
    }
    # Next, go through all the bugs in the buglist and remove and dupe_of
    # or depends_on links to bugs that are not in our list.  If that exists,
    # add a see_also link.
    foreach my $bug (@buglist) {
	if ($bug->{dupe_of}) {
	    unless ($gncbugs{$bug->{dupe_of}}) {
		push @{$bug->{see_also}}, "https://bugzilla.gnome.org/show_bug.cgi?id=" . $bug->{dupe_of};
		$bug->{dupe_of} = "";
		$bug->{resolution} = "NOTGNUCASH";
		# XXX: Add a comment/history that this changed?
	    }
	}
	my $idx = 0;
	foreach my $dep (@{$bug->{depends_on}}) {
	    unless ($gncbugs{$dep}) {
		push @{$bug->{see_also}}, "https://bugzilla.gnome.org/show_bug.cgi?id=$dep";
		splice @{$bug->{depends_on}},$idx,1;
		# XXX: Add a comment/history that this changed?
	    } else {$idx++};
	}
    }

    # Next, we keep iterating over the list of bugs that remain to satisfy
    # dependencies.  Hopefully there are no circular dependencies!
    print "Ordering bugs...\n";
    my $count = 0;
    my $lastcount = -1;
    while (scalar(@buglist) != 0) {
	$count++;
	print "Round $count: ($lastcount) " . scalar(@buglist) . ":";
	$lastcount = scalar(@buglist);
	@bugs = @buglist;
	@buglist = ();
	foreach my $bug (@bugs) {
	    print " " . $bug->{id};
	    my @deps;
	    push @deps, $bug->{dupe_of} if ($bug->{dupe_of});
	    #push @deps, @{$bug->{blocks}} if (scalar(@{$bug->{blocks}}) != 0);
	    #push @deps, @{$bug->{see_also}} if (scalar(@{$bug->{see_also}}) != 0);
	    push @deps, @{$bug->{depends_on}} if (scalar(@{$bug->{depends_on}}) != 0);
	    # Check if all the deps are in the result list, yet
	    my $clear = 1;
	    foreach my $id (@deps) {
		unless ($usedbugs{$id}) {
		    $clear = 0;
		    print "($id)";
		}
	    }
	    if ($clear) {
		push @orderedbugs, $bug;
		$usedbugs{$bug->{id}} = 1;
	    } else {
		push @buglist, $bug;
	    }
	}
	print "\n";
	die "Failed to reduce my list from $lastcount" if ($lastcount == scalar(@buglist));
    }
    print "Ordering complete\n";
    
    return \@orderedbugs;
}

####################
# Translating Bugs #
####################

sub _generate_description {
    my ($self, $bug, $fields) = @_;

    #my $json = JSON->new;
    #print "Bug: " . $json->pretty->encode($bug);
    #print "Fields: " . $json->pretty->encode($fields);
    return '';
}

sub translate_bug {
    my ($self, $fields) = @_;

    # translate from JSON back to internal names
    $fields->{blocked} = delete $fields->{blocks};
    $fields->{comment_is_private} = delete $fields->{commentprivacy};
    $fields->{creation_ts} = $self->SUPER::parse_date(delete $fields->{creation_time});
    $fields->{reporter} = delete $fields->{creator};
    $fields->{dependson} = delete $fields->{depends_on};
    $fields->{comment} = delete $fields->{description};
    $fields->{dup_id} = delete $fields->{dupe_of} if ($fields->{dupe_of});
    $fields->{bug_id} = delete $fields->{id};
    $fields->{everconfirmed} = delete $fields->{is_confirmed};
    $fields->{cclist_accessible} = delete $fields->{is_cc_accessible};
    $fields->{reporter_accessible} = delete $fields->{is_creator_accessible};
    $fields->{delta_ts} = $self->SUPER::parse_date(delete $fields->{last_change_time});
    $fields->{rep_platform} = delete $fields->{platform};
    $fields->{bug_severity} = delete $fields->{severity};
    $fields->{bug_status} = delete $fields->{status};
    $fields->{short_desc} = delete $fields->{summary};
    $fields->{bug_file_loc} = delete $fields->{url};
    $fields->{status_whiteboard} = delete $fields->{whiteboard};

    # Delete fields we don't/shouldn't handle    
    delete $fields->{cf_gnome_version};
    delete $fields->{cf_gnome_target};
    delete $fields->{is_open};
    delete $fields->{blocked};     # "dependson" will create blocked
    delete $fields->{dupe_of};     # make sure this does not exist
    delete $fields->{flags};

    # Translate gnome.bugs usernames in the bug top-level items:
    $fields->{reporter} =~ s/\@gnome.bugs/\@gnucash.bugs/;
    $fields->{qa_contact} =~ s/\@gnome.bugs/\@gnucash.bugs/;
    $fields->{assigned_to} =~ s/\@gnome.bugs/\@gnucash.bugs/;
    foreach my $name (@{$fields->{cc}}) {
	$name =~ s/\@gnome.bugs/\@gnucash.bugs/;
    }
    
    # Need to handle dependencies and duplicates later, so save them off for now
    $depends_on{$fields->{bug_id}} = delete $fields->{dependson} if (scalar @{$fields->{dependson}} > 0);
    $duplicates{$fields->{bug_id}} = $fields->{dup_id} if ($fields->{dup_id});
    delete $fields->{dup_id};      # duplicate data created later

    # Translate Status and Resolution fields
    #if ($fields->{bug_status} eq '') {}
    if ($fields->{resolution} eq 'NOTGNOME') {$fields->{resolution} = "NOTGNUCASH"; }

    # Translate comment fields
    foreach my $comment (@{$fields->{comments}}) {
	$comment->{who} = delete $comment->{creator};
	$comment->{thetext} = delete $comment->{text};
	$comment->{isprivate} = delete $comment->{is_private};
	$comment->{bug_when} = $self->SUPER::parse_date(delete $comment->{creation_time});
	if ($comment->{attachment_id}) {
	    if ($comment->{thetext} =~ m/^Created attachment /) {
		#$comment->{type} = CMT_ATTACHMENT_CREATED;
	    } else {
		$comment->{type} = CMT_ATTACHMENT_UPDATED;
		$comment->{extra_data} = $comment->{attachment_id};
	    }
	}

	delete $comment->{attachment_id};
	delete $comment->{count};
	delete $comment->{author};
	delete $comment->{id};
	delete $comment->{time}; # XXX: ???

	$comment->{who} =~ s/\@gnome.bugs/\@gnucash.bugs/;
    }

    # Translate the history list
    my @hlist;
    my $dmd = new Date::Manip::Date;
    foreach my $history (@{$fields->{history}}) {
	foreach my $change (@{$history->{changes}}) {
	    # Revert the field back to the "proper" name:
	    # perl -e 'use lib qw(/usr/share/bugzilla); use Bugzilla; print join " ", sort keys %{Bugzilla->fields({by_name => 1})}; print "\n"'
	    # alias assigned_to assigned_to_realname attach_data.thedata attachments.description attachments.filename attachments.isobsolete attachments.ispatch attachments.isprivate attachments.mimetype attachments.submitter blocked bug_file_loc bug_group bug_id bug_severity bug_status cc cclist_accessible classification comment_tag commenter component content creation_ts days_elapsed deadline delta_ts dependson estimated_time everconfirmed flagtypes.name keywords last_visit_ts longdesc longdescs.count longdescs.isprivate op_sys owner_idle_time percentage_complete priority product qa_contact qa_contact_realname remaining_time rep_platform reporter reporter_accessible reporter_realname requestees.login_name resolution see_also setters.login_name short_desc status_whiteboard tag target_milestone version work_time
	    my $field = $change->{field_name};
	    if ($field eq "blocks") { $field = "blocked"; }
	    if ($field eq "depends_on") { $field = "dependson"; }
	    if ($field eq "severity") { $field = "bug_severity"; }
	    if ($field eq "status") { $field = "bug_status"; }
	    if ($field eq "summary") { $field = "short_desc"; }
	    if ($field eq "is_confirmed") { $field = "everconfirmed"; }
	    if ($field eq "whiteboard") { $field = "status_whiteboard"; }
	    if ($field eq "url") { $field = "bug_file_loc"; }
	    if ($field eq "platform") { $field = "rep_platform"; }
	    if ($field eq "is_cc_accessible") { $field = "cclist_accessible"; }
	    if ($field eq "is_creator_accessible") { $field = "reporter_accessible"; }

	    $history->{who} =~ s/\@gnome.bugs/\@gnucash.bugs/;

	    ## XXX: Is this correct?
	    if ($field eq "groups") { $field = "bug_group"; }

	    #
	    # XXX: FIX THE FOLLOWING:
	    if ($field eq "attachments.gnome_attachment_status") {
		$field = "flagtypes.name";

		# keep track of the most recent attachment changes
		if ($change->{added} && $change->{added} ne '') {
		    my $attach_id = $change->{attachment_id};
		    my $when = $dmd->new;
		    my $time = $self->SUPER::parse_date($history->{when});
		    $when->parse($time);
		    if ($attach_id && 
			(! $attach_status{$attach_id} ||
			 ($attach_status{$attach_id}->{time}->cmp($when) < 0))) {
			$attach_status{$attach_id}->{time} = $when;
			$attach_status{$attach_id}->{when} = $time;
			$attach_status{$attach_id}->{status} = $change->{added};
			$attach_status{$attach_id}->{who} = $history->{who};
		    }
		}
	    }
	    if ($field eq "cf_gnome_version") { $field = "version"; }
	    if ($field eq "cf_gnome_target") {$field = "target_milestone"; }

	    # Translate Status and Resolution Fields in the History
	    # Change resolution NOTGNOME -> NOTGNUCASH
	    if ($field eq "resolution") {
		if ($change->{added} eq "NOTGNOME") {
		    $change->{added} = "NOTGNUCASH";
		}
		if ($change->{removed} eq "NOTGNOME") {
		    $change->{removed} = "NOTGNUCASH";
		}
	    }

	    push @hlist, { who => $history->{who},
			   bug_when => $self->SUPER::parse_date($history->{when}),
			   field => $field,
			   added => $change->{added},
			   removed => $change->{removed},
			   attachment_id => $change->{attachment_id}
	    };
	}
    }
    $fields->{history} = \@hlist;

    # Translate Attachments
    foreach my $attach (@{$fields->{attachments}}) {
	$attach->{submitter} = delete $attach->{creator};
	$attach->{filename} = delete $attach->{file_name};
	$attach->{ispatch} = delete $attach->{is_patch};
	$attach->{isprivate} = delete $attach->{is_private};
	$attach->{isobsolete} = delete $attach->{is_obsolete};
	$attach->{mimetype} = delete $attach->{content_type};
	$attach->{creation_ts} = $self->SUPER::parse_date(delete $attach->{creation_time});
	$attach->{attach_id} = delete $attach->{id};

	delete $attach->{attacher};
	delete $attach->{flags};
	delete $attach->{summary};
	delete $attach->{last_change_time};
	delete $attach->{size};

	$attach->{submitter} =~ s/\@gnome.bugs/\@gnucash.bugs/;

	# Set the attachment flag
#	if ($attach_status{$attach->{attach_id}}) {
#	    print "Got status " . $attach_status{$attach->{attach_id}}->{status} . " for attachment # " . $attach->{attach_id} . "\n";
#	}
    }

    #my $json = JSON->new;
    #print "Translated Bug: " . $json->pretty->encode($fields);
    
    return $fields;
}

1;