Андрей (andy) wrote in changelog,
Андрей
andy
changelog

[livejournal] r19534: LJSUP-9344 (clean up output of texttool....

Committer: ailyin
LJSUP-9344 (clean up output of texttool.pl)
U   trunk/bin/upgrading/texttool.pl
Modified: trunk/bin/upgrading/texttool.pl
===================================================================
--- trunk/bin/upgrading/texttool.pl	2011-07-27 03:25:33 UTC (rev 19533)
+++ trunk/bin/upgrading/texttool.pl	2011-07-27 03:57:02 UTC (rev 19534)
@@ -15,6 +15,8 @@
 my $opt_verbose;
 my $opt_all;    ## load texts for all known languages
 my $force_override = 0;
+my $opt_force_popstruct = 0;
+my $opt_process_deadphrases = 0;
 GetOptions(
            "help" => \$opt_help,
            "local-lang=s" => \$opt_local_lang,
@@ -22,6 +24,8 @@
            "only=s" => \$opt_only,
            "all"    => \$opt_all,
            "force-override" => \$force_override,
+           'force-popstruct' => \$opt_force_popstruct,
+           'process-deadphrases' => \$opt_process_deadphrases,
            ) or die "can't parse arguments";
 
 my $mode = shift @ARGV;
@@ -50,11 +54,24 @@
 
 Optionally:
     --local-lang=..     If given, works on local site files too
-    --all               When loading texts, and no language is
+
+    --all
+                        When loading texts, and no language is
                         specified, load all languages
-    
-    --force-override    Force overriding existing keys when loading texts from disk.
-    
+
+    --force-override
+                        Force overriding existing keys when loading texts
+                        from disk.
+
+    --force-popstruct
+                        Force updating/replacing languages and language domains
+                        if they are found to be unchanged
+
+    --process-deadphrases
+                        Process data found in deadphrases.dat and
+                        deadphrases-local.dat; this is disabled by default
+                        because these data are outdated.
+
 Examples:
     texttool.pl load en en_LJ
     texttool.pl --all load
@@ -199,9 +216,7 @@
         my $l = $lang_code{$lang};
         $out->("x", "Bogus language: $lang") unless $l;
         my @children = grep { $_->{'parentlnid'} == $l->{'lnid'} } values %lang_code;
-        foreach my $cl (@children) {
-            $out->("$l->{'lncode'} -- $cl->{'lncode'}");
-
+        foreach my $cl ( sort { $a->{'lncode'} cmp $b->{'lncode'} } @children ) {
             my %need;
             # push downwards everything that has some valid text in some language (< 4)
             $sth = $dbh->prepare("SELECT dmid, itid, txtid FROM ml_latest WHERE lnid=$l->{'lnid'} AND staleness < 4");
@@ -214,13 +229,21 @@
             while (my ($dmid, $itid, $txtid) = $sth->fetchrow_array) {
                 delete $need{"$dmid:$itid"};
             }
-            while (my $k = each %need) {
+
+            if ( %need && !$opt_verbose ) {
+                my $count = scalar keys %need;
+                $out->("[$l->{'lncode'} => $cl->{'lncode'}] $count items");
+            }
+
+            foreach my $k ( sort keys %need ) {
                 my ($dmid, $itid) = split(/:/, $k);
                 my $txtid = $need{$k};
                 my $stale = $cl->{'parenttype'} eq "diff" ? 3 : 0;
                 $dbh->do("INSERT INTO ml_latest (lnid, dmid, itid, txtid, chgtime, staleness) VALUES ".
                          "($cl->{'lnid'}, $dmid, $itid, $txtid, NOW(), $stale)");
                 die $dbh->errstr if $dbh->err;
+
+                $out->("[$l->{'lncode'} => $cl->{'lncode'}] $itid") if $opt_verbose;
             }
             $rec->($cl->{'lncode'}, $rec);
         }
@@ -321,15 +344,18 @@
 
     # begin iterating, order doesn't matter...
     foreach my $crumbkey (@crumbs) {
-        $out->("inserting crumb.$crumbkey");
         my $crumb = LJ::get_crumb($crumbkey);
         my $local = $LJ::CRUMBS_LOCAL{$crumbkey} ? 1 : 0;
 
         # see if it exists
         my $itid = $dbh->selectrow_array("SELECT itid FROM ml_items
                                           WHERE dmid = $genid AND itcode = 'crumb.$crumbkey'")+0;
-        LJ::Lang::set_text($genid, $local ? $loclang : 'en', "crumb.$crumbkey", $crumb->[0])
-            unless $itid;
+
+        unless ($itid) {
+            $out->("inserting crumb.$crumbkey");
+            my $lang = $local ? $loclang : 'en';
+            LJ::Lang::set_text( $genid, $lang, "crumb.$crumbkey", $crumb->[0] );
+        }
     }
 
     # done
@@ -339,19 +365,91 @@
 sub popstruct
 {
     $out->("Populating structure...", '+');
-    foreach my $l (values %lang_id) {
-        $out->("Inserting language: $l->{'lnname'}");
-        $dbh->do("REPLACE INTO ml_langs (lnid, lncode, lnname, parenttype, parentlnid) ".
-                 "VALUES (" . join(",", map { $dbh->quote($l->{$_}) } qw(lnid lncode lnname parenttype parentlnid)) . ")");
+
+    my $languages_changed = 0;
+    my $langdata = $dbh->selectall_arrayref(
+        'SELECT * FROM ml_langs',
+        { 'Slice' => {} },
+    );
+
+    my %langid_present;
+    foreach my $langrow (@$langdata) {
+        $langid_present{ $langrow->{'lnid'} } = 1;
+        my $l = $lang_id{ $langrow->{'lnid'} };
+
+        $languages_changed ||= !$l;
+        $l ||= {};
+
+        foreach my $key ( qw( lncode lnname parenttype parentlnid ) ) {
+            $languages_changed ||= ( $l->{$key} ne $langrow->{$key} );
+        }
+
+        last if $languages_changed;
     }
 
-    foreach my $d (values %dom_id) {
-        $out->("Inserting domain: $d->{'type'}\[$d->{'args'}\]");
-        $dbh->do("REPLACE INTO ml_domains (dmid, type, args) ".
-                 "VALUES (" . join(",", map { $dbh->quote($d->{$_}) } qw(dmid type args)) . ")");
+    if ( grep { !$langid_present{$_} } keys %lang_id ) {
+        $languages_changed = 1;
     }
 
-    $out->("Inserting language domains ...");
+    if ( $languages_changed || $opt_force_popstruct ) {
+        $out->('Languages:', '+');
+        foreach my $l ( values %lang_id ) {
+            $out->("$l->{'lnname'} (lncode=$l->{'lncode'})");
+            $dbh->do(
+                qq{
+                    REPLACE INTO ml_langs
+                    (lnid, lncode, lnname, parenttype, parentlnid)
+                    VALUES (?, ?, ?, ?, ?)
+                }, undef,
+                $l->{'lnid'}, $l->{'lncode'}, $l->{'lnname'},
+                $l->{'parenttype'}, $l->{'parentlnid'},
+            );
+        }
+        $out->('-');
+    } else {
+        $out->('Languages seem to be unchanged, not changing anything without --force-popstruct');
+    }
+
+    my $domains_changed = 0;
+    my $domdata = $dbh->selectall_arrayref(
+        'SELECT * FROM ml_domains',
+        { 'Slice' => {} },
+    );
+
+    my %domid_present;
+    foreach my $domrow (@$domdata) {
+        $domid_present{ $domrow->{'dmid'} } = 1;
+        my $l = $dom_id{ $domrow->{'dmid'} };
+
+        $domains_changed ||= !$l;
+        $l ||= {};
+
+        foreach my $key ( qw( type args ) ) {
+            $domains_changed ||= ( $l->{$key} ne $domrow->{$key} );
+        }
+
+        last if $domains_changed;
+    }
+
+    if ( grep { !$domid_present{$_} } keys %dom_id ) {
+        $domains_changed = 1;
+    }
+
+    if ( $domains_changed || $opt_force_popstruct ) {
+        $out->('Domains:', '+');
+        foreach my $d (values %dom_id) {
+            $out->("$d->{'type'}\[$d->{'args'}\]");
+            $dbh->do(
+                'REPLACE INTO ml_domains (dmid, type, args) VALUES (?, ?, ?)',
+                undef, $d->{'dmid'}, $d->{'type'}, $d->{'args'},
+            );
+        }
+        $out->('-');
+    } else {
+        $out->('Domains seem to be unchanged, not changing anything without --force-popstruct');
+    }
+
+    $out->('Inserting/updating language domains ...');
     foreach my $ld (@lang_domains) {
         $dbh->do("INSERT IGNORE INTO ml_langdomains (lnid, dmid, dmmaster) VALUES ".
                  "(" . join(",", map { $dbh->quote($ld->{$_}) } qw(lnid dmid dmmaster)) . ")");
@@ -373,7 +471,9 @@
         }
     }
 
-    $out->("Populating text...", '+');
+    $out->('Populating text (reading all these files may take a while)...');
+    $out->('  hint: --verbose will output filenames') unless $opt_verbose;
+    $out->('+');
 
     # learn about base files
     my %source;   # langcode -> absfilepath
@@ -381,7 +481,7 @@
         my $file = "$ENV{'LJHOME'}/bin/upgrading/${lang}.dat";
         next if $opt_only && $lang ne $opt_only;
         next unless -e $file;
-        $source{$file} = [$lang, ''];
+        $source{$file} = [ $lang, '', "bin/upgrading/${lang}.dat" ];
     }
 
     # learn about local files
@@ -399,16 +499,15 @@
         $pfx =~ s!^htdocs/!!;
         $pfx =~ s!\.text(\.local)?$!!;
         $pfx = "/$pfx";
-        $source{"$ENV{'LJHOME'}/$tf"} = [$lang, $pfx];
+        $source{"$ENV{'LJHOME'}/$tf"} = [ $lang, $pfx, $tf ];
     }
 
     my %existing_item;  # langid -> code -> 1
 
-    foreach my $file (keys %source)
-    {
-        my ($lang, $pfx) = @{$source{$file}};
+    foreach my $file ( sort keys %source ) {
+        my ( $lang, $pfx, $filename_short ) = @{$source{$file}};
 
-        $out->("$lang", '+');
+        $out->("reading $filename_short...") if $opt_verbose;
         my $ldf = LJ::LangDatFile->new($file);
 
         my $l = $lang_code{$lang} or die "unknown language '$lang'";
@@ -421,15 +520,25 @@
             my $text = $ldf->value($code);
 
             $code = "$pfx$code";
-            die "Code in file $file can't start with a dot: $code"
+            die "Code in file $filename_short can't start with a dot: $code"
                 if $code =~ /^\./;
 
             # load existing items for target language
             unless (exists $existing_item{$l->{'lnid'}}) {
                 $existing_item{$l->{'lnid'}} = {};
                 my $sth = $dbh->prepare(qq{
-                    SELECT i.itcode, t.text   FROM ml_latest l, ml_items i, ml_text t WHERE i.dmid=1 AND l.dmid=1 AND i.itid=l.itid AND l.lnid=? AND t.lnid=l.lnid and t.txtid = l.txtid and i.dmid=i.dmid and t.dmid=i.dmid
-                    });
+                    SELECT i.itcode, t.text
+                    FROM ml_latest l, ml_items i, ml_text t
+                    WHERE
+                        i.dmid  = 1       AND
+                        l.dmid  = 1       AND
+                        i.itid  = l.itid  AND
+                        l.lnid  = ?       AND
+                        t.lnid  = l.lnid  AND
+                        t.txtid = l.txtid AND
+                        i.dmid  = i.dmid  AND
+                        t.dmid  = i.dmid
+                });
                 $sth->execute($l->{lnid});
                 die $sth->errstr if $sth->err;
                 while (my ($code, $oldtext) = $sth->fetchrow_array) {
@@ -447,8 +556,16 @@
             return if exists $existing_item{$l->{'lnid'}}->{$code}
                         and not $force_override;
             
-            unless ($existing_item{$l->{'lnid'}}->{$code} eq $text) {
+            my $old_text = $existing_item{$l->{'lnid'}}->{$code};
+
+            unless ( $old_text eq $text ) {
                 $addcount++;
+                if ($old_text) {
+                    $out->("[$filename_short, lang=$lang] $code: $old_text => $text");
+                } else {
+                    $out->("[$filename_short, lang=$lang] $code: setting to $text");
+                }
+
                 # if the text is changing, the staleness is at least 1
                 my $staleness = $metadata{'staleness'}+0 || 1;
 
@@ -456,18 +573,21 @@
                                              { 'staleness' => $staleness,
                                                'notes' => $metadata{'notes'},
                                                'changeseverity' => 2, });
-                $out->("set: $code") if $opt_verbose;
+
                 unless ($res) {
                     $out->('x', "ERROR: " . LJ::Lang::last_error());
                 }
             }
         });
-        $out->("added: $addcount", '-');
+
+        if ( $addcount > 0 ) {
+            $out->("added $addcount from $file");
+        }
     }
     $out->("-", "done.");
 
     # dead phrase removal
-    if ($LJ::IS_DEV_SERVER) {
+    if ($opt_process_deadphrases) {
         $out->("Removing dead phrases...", '+');
         foreach my $file ("deadphrases.dat", "deadphrases-local.dat") {
             my $ffile = "$ENV{'LJHOME'}/bin/upgrading/$file";
@@ -687,12 +807,16 @@
     return if $no_error && !$itid;
     $out->("x", "Unknown item code $itcode.") unless $itid;
 
-    $out->("Removing item $itcode from domain $dmcode ($itid)...", "+");
-
     # need to delete everything from: ml_items ml_latest ml_text
 
-    $dbh->do("DELETE FROM ml_items WHERE dmid=$dmid AND itid=$itid");
+    my $affected = $dbh->do("DELETE FROM ml_items WHERE dmid=$dmid AND itid=$itid");
 
+    # we're only outputting something if this is something
+    # significant, according to ml_items
+    if ( $affected > 0 ) {
+        $out->("Removing item $itcode from domain $dmcode ($itid)...");
+    }
+
     my $txtids = "";
     my $sth = $dbh->prepare("SELECT txtid FROM ml_latest WHERE dmid=$dmid AND itid=$itid");
     $sth->execute;
@@ -702,9 +826,4 @@
     }
     $dbh->do("DELETE FROM ml_latest WHERE dmid=$dmid AND itid=$itid");
     $dbh->do("DELETE FROM ml_text WHERE dmid=$dmid AND txtid IN ($txtids)") if $txtids;
-
-    $out->("-","done.");
 }
-
-
-

Tags: andy, livejournal, pl
Subscribe
  • Post a new comment

    Error

    Anonymous comments are disabled in this journal

    default userpic

    Your reply will be screened

    Your IP address will be recorded 

  • 0 comments