Committer: ailyin
LJSUP-9344 (clean up output of texttool.pl)U trunk/bin/upgrading/texttool.pl
Modified: trunk/bin/upgrading/texttool.pl
===================================================================
--- trunk/bin/upgrading/texttool.pl 2011-07-27 03:25:33 UTC (rev 19533)
+++ trunk/bin/upgrading/texttool.pl 2011-07-27 03:57:02 UTC (rev 19534)
@@ -15,6 +15,8 @@
my $opt_verbose;
my $opt_all; ## load texts for all known languages
my $force_override = 0;
+my $opt_force_popstruct = 0;
+my $opt_process_deadphrases = 0;
GetOptions(
"help" => \$opt_help,
"local-lang=s" => \$opt_local_lang,
@@ -22,6 +24,8 @@
"only=s" => \$opt_only,
"all" => \$opt_all,
"force-override" => \$force_override,
+ 'force-popstruct' => \$opt_force_popstruct,
+ 'process-deadphrases' => \$opt_process_deadphrases,
) or die "can't parse arguments";
my $mode = shift @ARGV;
@@ -50,11 +54,24 @@
Optionally:
--local-lang=.. If given, works on local site files too
- --all When loading texts, and no language is
+
+ --all
+ When loading texts, and no language is
specified, load all languages
-
- --force-override Force overriding existing keys when loading texts from disk.
-
+
+ --force-override
+ Force overriding existing keys when loading texts
+ from disk.
+
+ --force-popstruct
+ Force updating/replacing languages and language domains
+ if they are found to be unchanged
+
+ --process-deadphrases
+ Process data found in deadphrases.dat and
+ deadphrases-local.dat; this is disabled by default
+ because these data are outdated.
+
Examples:
texttool.pl load en en_LJ
texttool.pl --all load
@@ -199,9 +216,7 @@
my $l = $lang_code{$lang};
$out->("x", "Bogus language: $lang") unless $l;
my @children = grep { $_->{'parentlnid'} == $l->{'lnid'} } values %lang_code;
- foreach my $cl (@children) {
- $out->("$l->{'lncode'} -- $cl->{'lncode'}");
-
+ foreach my $cl ( sort { $a->{'lncode'} cmp $b->{'lncode'} } @children ) {
my %need;
# push downwards everything that has some valid text in some language (< 4)
$sth = $dbh->prepare("SELECT dmid, itid, txtid FROM ml_latest WHERE lnid=$l->{'lnid'} AND staleness < 4");
@@ -214,13 +229,21 @@
while (my ($dmid, $itid, $txtid) = $sth->fetchrow_array) {
delete $need{"$dmid:$itid"};
}
- while (my $k = each %need) {
+
+ if ( %need && !$opt_verbose ) {
+ my $count = scalar keys %need;
+ $out->("[$l->{'lncode'} => $cl->{'lncode'}] $count items");
+ }
+
+ foreach my $k ( sort keys %need ) {
my ($dmid, $itid) = split(/:/, $k);
my $txtid = $need{$k};
my $stale = $cl->{'parenttype'} eq "diff" ? 3 : 0;
$dbh->do("INSERT INTO ml_latest (lnid, dmid, itid, txtid, chgtime, staleness) VALUES ".
"($cl->{'lnid'}, $dmid, $itid, $txtid, NOW(), $stale)");
die $dbh->errstr if $dbh->err;
+
+ $out->("[$l->{'lncode'} => $cl->{'lncode'}] $itid") if $opt_verbose;
}
$rec->($cl->{'lncode'}, $rec);
}
@@ -321,15 +344,18 @@
# begin iterating, order doesn't matter...
foreach my $crumbkey (@crumbs) {
- $out->("inserting crumb.$crumbkey");
my $crumb = LJ::get_crumb($crumbkey);
my $local = $LJ::CRUMBS_LOCAL{$crumbkey} ? 1 : 0;
# see if it exists
my $itid = $dbh->selectrow_array("SELECT itid FROM ml_items
WHERE dmid = $genid AND itcode = 'crumb.$crumbkey'")+0;
- LJ::Lang::set_text($genid, $local ? $loclang : 'en', "crumb.$crumbkey", $crumb->[0])
- unless $itid;
+
+ unless ($itid) {
+ $out->("inserting crumb.$crumbkey");
+ my $lang = $local ? $loclang : 'en';
+ LJ::Lang::set_text( $genid, $lang, "crumb.$crumbkey", $crumb->[0] );
+ }
}
# done
@@ -339,19 +365,91 @@
sub popstruct
{
$out->("Populating structure...", '+');
- foreach my $l (values %lang_id) {
- $out->("Inserting language: $l->{'lnname'}");
- $dbh->do("REPLACE INTO ml_langs (lnid, lncode, lnname, parenttype, parentlnid) ".
- "VALUES (" . join(",", map { $dbh->quote($l->{$_}) } qw(lnid lncode lnname parenttype parentlnid)) . ")");
+
+ my $languages_changed = 0;
+ my $langdata = $dbh->selectall_arrayref(
+ 'SELECT * FROM ml_langs',
+ { 'Slice' => {} },
+ );
+
+ my %langid_present;
+ foreach my $langrow (@$langdata) {
+ $langid_present{ $langrow->{'lnid'} } = 1;
+ my $l = $lang_id{ $langrow->{'lnid'} };
+
+ $languages_changed ||= !$l;
+ $l ||= {};
+
+ foreach my $key ( qw( lncode lnname parenttype parentlnid ) ) {
+ $languages_changed ||= ( $l->{$key} ne $langrow->{$key} );
+ }
+
+ last if $languages_changed;
}
- foreach my $d (values %dom_id) {
- $out->("Inserting domain: $d->{'type'}\[$d->{'args'}\]");
- $dbh->do("REPLACE INTO ml_domains (dmid, type, args) ".
- "VALUES (" . join(",", map { $dbh->quote($d->{$_}) } qw(dmid type args)) . ")");
+ if ( grep { !$langid_present{$_} } keys %lang_id ) {
+ $languages_changed = 1;
}
- $out->("Inserting language domains ...");
+ if ( $languages_changed || $opt_force_popstruct ) {
+ $out->('Languages:', '+');
+ foreach my $l ( values %lang_id ) {
+ $out->("$l->{'lnname'} (lncode=$l->{'lncode'})");
+ $dbh->do(
+ qq{
+ REPLACE INTO ml_langs
+ (lnid, lncode, lnname, parenttype, parentlnid)
+ VALUES (?, ?, ?, ?, ?)
+ }, undef,
+ $l->{'lnid'}, $l->{'lncode'}, $l->{'lnname'},
+ $l->{'parenttype'}, $l->{'parentlnid'},
+ );
+ }
+ $out->('-');
+ } else {
+ $out->('Languages seem to be unchanged, not changing anything without --force-popstruct');
+ }
+
+ my $domains_changed = 0;
+ my $domdata = $dbh->selectall_arrayref(
+ 'SELECT * FROM ml_domains',
+ { 'Slice' => {} },
+ );
+
+ my %domid_present;
+ foreach my $domrow (@$domdata) {
+ $domid_present{ $domrow->{'dmid'} } = 1;
+ my $l = $dom_id{ $domrow->{'dmid'} };
+
+ $domains_changed ||= !$l;
+ $l ||= {};
+
+ foreach my $key ( qw( type args ) ) {
+ $domains_changed ||= ( $l->{$key} ne $domrow->{$key} );
+ }
+
+ last if $domains_changed;
+ }
+
+ if ( grep { !$domid_present{$_} } keys %dom_id ) {
+ $domains_changed = 1;
+ }
+
+ if ( $domains_changed || $opt_force_popstruct ) {
+ $out->('Domains:', '+');
+ foreach my $d (values %dom_id) {
+ $out->("$d->{'type'}\[$d->{'args'}\]");
+ $dbh->do(
+ 'REPLACE INTO ml_domains (dmid, type, args) VALUES (?, ?, ?)',
+ undef, $d->{'dmid'}, $d->{'type'}, $d->{'args'},
+ );
+ }
+ $out->('-');
+ } else {
+ $out->('Domains seem to be unchanged, not changing anything without --force-popstruct');
+ }
+
+ $out->('Inserting/updating language domains ...');
foreach my $ld (@lang_domains) {
$dbh->do("INSERT IGNORE INTO ml_langdomains (lnid, dmid, dmmaster) VALUES ".
"(" . join(",", map { $dbh->quote($ld->{$_}) } qw(lnid dmid dmmaster)) . ")");
@@ -373,7 +471,9 @@
}
}
- $out->("Populating text...", '+');
+ $out->('Populating text (reading all these files may take a while)...');
+ $out->(' hint: --verbose will output filenames') unless $opt_verbose;
+ $out->('+');
# learn about base files
my %source; # langcode -> absfilepath
@@ -381,7 +481,7 @@
my $file = "$ENV{'LJHOME'}/bin/upgrading/${lang}.dat";
next if $opt_only && $lang ne $opt_only;
next unless -e $file;
- $source{$file} = [$lang, ''];
+ $source{$file} = [ $lang, '', "bin/upgrading/${lang}.dat" ];
}
# learn about local files
@@ -399,16 +499,15 @@
$pfx =~ s!^htdocs/!!;
$pfx =~ s!\.text(\.local)?$!!;
$pfx = "/$pfx";
- $source{"$ENV{'LJHOME'}/$tf"} = [$lang, $pfx];
+ $source{"$ENV{'LJHOME'}/$tf"} = [ $lang, $pfx, $tf ];
}
my %existing_item; # langid -> code -> 1
- foreach my $file (keys %source)
- {
- my ($lang, $pfx) = @{$source{$file}};
+ foreach my $file ( sort keys %source ) {
+ my ( $lang, $pfx, $filename_short ) = @{$source{$file}};
- $out->("$lang", '+');
+ $out->("reading $filename_short...") if $opt_verbose;
my $ldf = LJ::LangDatFile->new($file);
my $l = $lang_code{$lang} or die "unknown language '$lang'";
@@ -421,15 +520,25 @@
my $text = $ldf->value($code);
$code = "$pfx$code";
- die "Code in file $file can't start with a dot: $code"
+ die "Code in file $filename_short can't start with a dot: $code"
if $code =~ /^\./;
# load existing items for target language
unless (exists $existing_item{$l->{'lnid'}}) {
$existing_item{$l->{'lnid'}} = {};
my $sth = $dbh->prepare(qq{
- SELECT i.itcode, t.text FROM ml_latest l, ml_items i, ml_text t WHERE i.dmid=1 AND l.dmid=1 AND i.itid=l.itid AND l.lnid=? AND t.lnid=l.lnid and t.txtid = l.txtid and i.dmid=i.dmid and t.dmid=i.dmid
- });
+ SELECT i.itcode, t.text
+ FROM ml_latest l, ml_items i, ml_text t
+ WHERE
+ i.dmid = 1 AND
+ l.dmid = 1 AND
+ i.itid = l.itid AND
+ l.lnid = ? AND
+ t.lnid = l.lnid AND
+ t.txtid = l.txtid AND
+ i.dmid = i.dmid AND
+ t.dmid = i.dmid
+ });
$sth->execute($l->{lnid});
die $sth->errstr if $sth->err;
while (my ($code, $oldtext) = $sth->fetchrow_array) {
@@ -447,8 +556,16 @@
return if exists $existing_item{$l->{'lnid'}}->{$code}
and not $force_override;
- unless ($existing_item{$l->{'lnid'}}->{$code} eq $text) {
+ my $old_text = $existing_item{$l->{'lnid'}}->{$code};
+
+ unless ( $old_text eq $text ) {
$addcount++;
+ if ($old_text) {
+ $out->("[$filename_short, lang=$lang] $code: $old_text => $text");
+ } else {
+ $out->("[$filename_short, lang=$lang] $code: setting to $text");
+ }
+
# if the text is changing, the staleness is at least 1
my $staleness = $metadata{'staleness'}+0 || 1;
@@ -456,18 +573,21 @@
{ 'staleness' => $staleness,
'notes' => $metadata{'notes'},
'changeseverity' => 2, });
- $out->("set: $code") if $opt_verbose;
+
unless ($res) {
$out->('x', "ERROR: " . LJ::Lang::last_error());
}
}
});
- $out->("added: $addcount", '-');
+
+ if ( $addcount > 0 ) {
+ $out->("added $addcount from $file");
+ }
}
$out->("-", "done.");
# dead phrase removal
- if ($LJ::IS_DEV_SERVER) {
+ if ($opt_process_deadphrases) {
$out->("Removing dead phrases...", '+');
foreach my $file ("deadphrases.dat", "deadphrases-local.dat") {
my $ffile = "$ENV{'LJHOME'}/bin/upgrading/$file";
@@ -687,12 +807,16 @@
return if $no_error && !$itid;
$out->("x", "Unknown item code $itcode.") unless $itid;
- $out->("Removing item $itcode from domain $dmcode ($itid)...", "+");
-
# need to delete everything from: ml_items ml_latest ml_text
- $dbh->do("DELETE FROM ml_items WHERE dmid=$dmid AND itid=$itid");
+ my $affected = $dbh->do("DELETE FROM ml_items WHERE dmid=$dmid AND itid=$itid");
+ # we're only outputting something if this is something
+ # significant, according to ml_items
+ if ( $affected > 0 ) {
+ $out->("Removing item $itcode from domain $dmcode ($itid)...");
+ }
+
my $txtids = "";
my $sth = $dbh->prepare("SELECT txtid FROM ml_latest WHERE dmid=$dmid AND itid=$itid");
$sth->execute;
@@ -702,9 +826,4 @@
}
$dbh->do("DELETE FROM ml_latest WHERE dmid=$dmid AND itid=$itid");
$dbh->do("DELETE FROM ml_text WHERE dmid=$dmid AND txtid IN ($txtids)") if $txtids;
-
- $out->("-","done.");
}
-
-
-
