Андрей (andy) wrote in changelog,
Андрей
andy
changelog

[livejournal] r19888: LJSUP-9653 (ljlang refactoring)

Committer: ailyin
LJSUP-9653 (ljlang refactoring)
U   trunk/bin/misc/set_comm_supermaintainer.pl
U   trunk/bin/upgrading/texttool.pl
U   trunk/bin/worker/email-status
U   trunk/bin/worker/esn-cluster-subs
U   trunk/bin/worker/esn-filter-subs
U   trunk/bin/worker/esn-fired-event
U   trunk/bin/worker/esn-fired-mass
U   trunk/bin/worker/esn-process-sub
U   trunk/bin/worker/mark-suspended-entries
U   trunk/bin/worker/process-esn
U   trunk/bin/worker/process-sms
A   trunk/cgi-bin/LJ/Lang.pm
U   trunk/cgi-bin/LJ/NewWorker.pm
U   trunk/cgi-bin/ljlang.pl
U   trunk/cgi-bin/ljviews-s1-using-s2.pl
U   trunk/cgi-bin/ljviews.pl
U   trunk/cgi-bin/modperl_subs.pl
U   trunk/t/commafy.t
U   trunk/t/console-faqcat.t
U   trunk/t/faq.t
U   trunk/t/settings.t
Modified: trunk/bin/misc/set_comm_supermaintainer.pl
===================================================================
--- trunk/bin/misc/set_comm_supermaintainer.pl	2011-08-30 04:13:09 UTC (rev 19887)
+++ trunk/bin/misc/set_comm_supermaintainer.pl	2011-08-30 06:35:24 UTC (rev 19888)
@@ -6,7 +6,7 @@
 use lib "$ENV{LJHOME}/cgi-bin";
 require "ljlib.pl";
 require "ljdb.pl";
-require "ljlang.pl";
+use LJ::Lang;
 require 'ljprotocol.pl';
 use Getopt::Long;
 use LJ::DBUtil;

Modified: trunk/bin/upgrading/texttool.pl
===================================================================
--- trunk/bin/upgrading/texttool.pl	2011-08-30 04:13:09 UTC (rev 19887)
+++ trunk/bin/upgrading/texttool.pl	2011-08-30 06:35:24 UTC (rev 19888)
@@ -84,7 +84,7 @@
 }
 
 require 'ljlib.pl';
-require 'ljlang.pl';
+use LJ::Lang;
 require 'weblib.pl';
 
 my %dom_id;       # number -> {}

Modified: trunk/bin/worker/email-status
===================================================================
--- trunk/bin/worker/email-status	2011-08-30 04:13:09 UTC (rev 19887)
+++ trunk/bin/worker/email-status	2011-08-30 06:35:24 UTC (rev 19888)
@@ -6,7 +6,7 @@
 
 require 'ljlib.pl';
 require 'ljprotocol.pl';
-require 'ljlang.pl';
+use LJ::Lang;
 
 package LJ::NewWorker::TheSchwartz::EmailStatus;
 use base 'LJ::NewWorker::TheSchwartz';

Modified: trunk/bin/worker/esn-cluster-subs
===================================================================
--- trunk/bin/worker/esn-cluster-subs	2011-08-30 04:13:09 UTC (rev 19887)
+++ trunk/bin/worker/esn-cluster-subs	2011-08-30 06:35:24 UTC (rev 19888)
@@ -3,7 +3,7 @@
 use lib "$ENV{LJHOME}/cgi-bin";
 require 'ljlib.pl';
 require 'ljprotocol.pl';
-require 'ljlang.pl';
+use LJ::Lang;
 use LJ::NewWorker::TheSchwartz;
 use LJ::ESN;
 LJ::NewWorker::TheSchwartz::FindSubsByCluster->start();

Modified: trunk/bin/worker/esn-filter-subs
===================================================================
--- trunk/bin/worker/esn-filter-subs	2011-08-30 04:13:09 UTC (rev 19887)
+++ trunk/bin/worker/esn-filter-subs	2011-08-30 06:35:24 UTC (rev 19888)
@@ -3,7 +3,7 @@
 use lib "$ENV{LJHOME}/cgi-bin";
 require 'ljlib.pl';
 require 'ljprotocol.pl';
-require 'ljlang.pl';
+use LJ::Lang;
 use LJ::NewWorker::TheSchwartz;
 use LJ::ESN;
 LJ::NewWorker::TheSchwartz::FilterSubs->start();

Modified: trunk/bin/worker/esn-fired-event
===================================================================
--- trunk/bin/worker/esn-fired-event	2011-08-30 04:13:09 UTC (rev 19887)
+++ trunk/bin/worker/esn-fired-event	2011-08-30 06:35:24 UTC (rev 19888)
@@ -3,7 +3,7 @@
 use lib "$ENV{LJHOME}/cgi-bin";
 require 'ljlib.pl';
 require 'ljprotocol.pl';
-require 'ljlang.pl';
+use LJ::Lang;
 use LJ::NewWorker::TheSchwartz;
 use LJ::ESN;
 LJ::NewWorker::TheSchwartz::FiredEvent->start();

Modified: trunk/bin/worker/esn-fired-mass
===================================================================
--- trunk/bin/worker/esn-fired-mass	2011-08-30 04:13:09 UTC (rev 19887)
+++ trunk/bin/worker/esn-fired-mass	2011-08-30 06:35:24 UTC (rev 19888)
@@ -3,7 +3,7 @@
 use lib "$ENV{LJHOME}/cgi-bin";
 require 'ljlib.pl';
 require 'ljprotocol.pl';
-require 'ljlang.pl';
+use LJ::Lang;
 use LJ::NewWorker::TheSchwartz;
 use LJ::ESN;
 

Modified: trunk/bin/worker/esn-process-sub
===================================================================
--- trunk/bin/worker/esn-process-sub	2011-08-30 04:13:09 UTC (rev 19887)
+++ trunk/bin/worker/esn-process-sub	2011-08-30 06:35:24 UTC (rev 19888)
@@ -3,7 +3,7 @@
 use lib "$ENV{LJHOME}/cgi-bin";
 require 'ljlib.pl';
 require 'ljprotocol.pl';
-require 'ljlang.pl';
+use LJ::Lang;
 use LJ::NewWorker::TheSchwartz;
 use LJ::ESN;
 LJ::NewWorker::TheSchwartz::ProcessSub->start();

Modified: trunk/bin/worker/mark-suspended-entries
===================================================================
--- trunk/bin/worker/mark-suspended-entries	2011-08-30 04:13:09 UTC (rev 19887)
+++ trunk/bin/worker/mark-suspended-entries	2011-08-30 06:35:24 UTC (rev 19888)
@@ -3,7 +3,7 @@
 use lib "$ENV{LJHOME}/cgi-bin";
 require 'ljlib.pl';
 require 'ljprotocol.pl';
-require 'ljlang.pl';
+use LJ::Lang;
 
 package LJ::NewWorker::TheSchwartz::MarkSuspendedEntries;
 use base 'LJ::NewWorker::TheSchwartz';

Modified: trunk/bin/worker/process-esn
===================================================================
--- trunk/bin/worker/process-esn	2011-08-30 04:13:09 UTC (rev 19887)
+++ trunk/bin/worker/process-esn	2011-08-30 06:35:24 UTC (rev 19888)
@@ -3,7 +3,7 @@
 use lib "$ENV{LJHOME}/cgi-bin";
 require 'ljlib.pl';
 require 'ljprotocol.pl';
-require 'ljlang.pl';
+use LJ::Lang;
 
 package LJ::NewWorker::TheSchwartz::ESN;
 use strict;

Modified: trunk/bin/worker/process-sms
===================================================================
--- trunk/bin/worker/process-sms	2011-08-30 04:13:09 UTC (rev 19887)
+++ trunk/bin/worker/process-sms	2011-08-30 06:35:24 UTC (rev 19888)
@@ -8,7 +8,7 @@
 use base 'LJ::NewWorker::TheSchwartz';
 require 'ljlib.pl';
 require 'ljprotocol.pl';
-require 'ljlang.pl';
+use LJ::Lang;
 require 'ljfeed.pl';
 use LJ::SMS;
 

Added: trunk/cgi-bin/LJ/Lang.pm
===================================================================
--- trunk/cgi-bin/LJ/Lang.pm	                        (rev 0)
+++ trunk/cgi-bin/LJ/Lang.pm	2011-08-30 06:35:24 UTC (rev 19888)
@@ -0,0 +1,1014 @@
+package LJ::Lang;
+use strict;
+use warnings;
+
+use lib "$ENV{'LJHOME'}/cgi-bin";
+require "ljhooks.pl";
+
+use LJ::LangDatFile;
+use LJ::TimeUtil;
+
+use constant MAXIMUM_ITCODE_LENGTH => 80;
+
+my @day_short = qw( Sun Mon Tue Wed Thu Fri Sat );
+my @day_long = qw( Sunday Monday Tuesday Wednesday Thursday Friday Saturday );
+my @month_short = qw( Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec );
+my @month_long  = qw(
+    January February March April May June
+    July August September October November December
+);
+
+# get entire array of days and months
+sub day_list_short   { return @LJ::Lang::day_short; }
+sub day_list_long    { return @LJ::Lang::day_long; }
+sub month_list_short { return @LJ::Lang::month_short; }
+sub month_list_long  { return @LJ::Lang::month_long; }
+
+# access individual day or month given integer
+sub day_short   { return $day_short[ $_[0] - 1 ]; }
+sub day_long    { return $day_long[ $_[0] - 1 ]; }
+sub month_short { return $month_short[ $_[0] - 1 ]; }
+sub month_long  { return $month_long[ $_[0] - 1 ]; }
+
+# lang codes for individual day or month given integer
+sub day_short_langcode {
+    return "date.day." . lc( LJ::Lang::day_long(@_) ) . ".short";
+}
+
+sub day_long_langcode {
+    return "date.day." . lc( LJ::Lang::day_long(@_) ) . ".long";
+}
+
+sub month_short_langcode {
+    return "date.month." . lc( LJ::Lang::month_long(@_) ) . ".short";
+}
+
+sub month_long_langcode {
+    return "date.month." . lc( LJ::Lang::month_long(@_) ) . ".long";
+}
+
+sub month_long_genitive_langcode {
+    return "date.month." . lc( LJ::Lang::month_long(@_) ) . ".genitive";
+}
+
+## ordinal suffix
+sub day_ord {
+    my $day = shift;
+
+    # teens all end in 'th'
+    if ( $day =~ /1\d$/ ) { return "th"; }
+
+    # otherwise endings in 1, 2, 3 are special
+    if ( $day % 10 == 1 ) { return "st"; }
+    if ( $day % 10 == 2 ) { return "nd"; }
+    if ( $day % 10 == 3 ) { return "rd"; }
+
+    # everything else (0,4-9) end in "th"
+    return "th";
+}
+
+sub time_format {
+    my ( $hours, $h, $m, $formatstring ) = @_;
+
+    if ( $formatstring eq "short" ) {
+        if ( $hours == 12 ) {
+            my $ret;
+            my $ap = "a";
+            if    ( $h == 0 )  { $ret .= "12"; }
+            elsif ( $h < 12 )  { $ret .= ( $h + 0 ); }
+            elsif ( $h == 12 ) { $ret .= ( $h + 0 ); $ap = "p"; }
+            else               { $ret .= ( $h - 12 ); $ap = "p"; }
+            $ret .= sprintf( ":%02d$ap", $m );
+            return $ret;
+        } elsif ( $hours == 24 ) {
+            return sprintf( "%02d:%02d", $h, $m );
+        }
+    }
+    return "";
+}
+
+#### ml_ stuff:
+my $LS_CACHED = 0;
+my %DM_ID   = ();  # id -> { type, args, dmid, langs => { => 1, => 0, => 1 } }
+my %DM_UNIQ = ();  # "$type/$args" => ^^^
+my %LN_ID   = ();  # id -> { ..., ..., 'children' => [ $ids, .. ] }
+my %LN_CODE = ();  # $code -> ^^^^
+my $LAST_ERROR;
+my %TXT_CACHE;
+
+if ( $LJ::IS_DEV_SERVER || $LJ::IS_LJCOM_BETA ) {
+    our $_hook_is_installed;
+    LJ::register_hook( 'start_request', sub { %TXT_CACHE = (); } )
+        unless $_hook_is_installed++;
+}
+
+sub last_error {
+    return $LAST_ERROR;
+}
+
+sub set_error {
+    $LAST_ERROR = $_[0];
+    return 0;
+}
+
+sub get_lang {
+    my $code = shift;
+    load_lang_struct() unless $LS_CACHED;
+    return $LN_CODE{$code};
+}
+
+sub get_lang_id {
+    my $id = shift;
+    load_lang_struct() unless $LS_CACHED;
+    return $LN_ID{$id};
+}
+
+sub get_dom {
+    my $dmcode = shift;
+    load_lang_struct() unless $LS_CACHED;
+    return $DM_UNIQ{$dmcode};
+}
+
+sub get_dom_id {
+    my $dmid = shift;
+    load_lang_struct() unless $LS_CACHED;
+    return $DM_ID{$dmid};
+}
+
+sub get_domains {
+    load_lang_struct() unless $LS_CACHED;
+    return values %DM_ID;
+}
+
+sub get_root_lang {
+    my $dom = shift;    # from, say, get_dom
+    return undef unless ref $dom eq "HASH";
+
+    my $lang_override = LJ::run_hook( "root_lang_override", $dom );
+    return get_lang($lang_override) if $lang_override;
+
+    foreach ( keys %{ $dom->{'langs'} } ) {
+        if ( $dom->{'langs'}->{$_} ) {
+            return get_lang_id($_);
+        }
+    }
+    return undef;
+}
+
+sub load_lang_struct {
+    return 1 if $LS_CACHED;
+    my $dbr = LJ::get_db_reader();
+    return set_error("No database available") unless $dbr;
+    my $sth;
+
+    $sth = $dbr->prepare("SELECT dmid, type, args FROM ml_domains");
+    $sth->execute;
+    while ( my ( $dmid, $type, $args ) = $sth->fetchrow_array ) {
+        my $uniq = $args ? "$type/$args" : $type;
+        $DM_UNIQ{$uniq} = $DM_ID{$dmid} = {
+            'type' => $type,
+            'args' => $args,
+            'dmid' => $dmid,
+            'uniq' => $uniq,
+        };
+    }
+
+    $sth = $dbr->prepare(
+        "SELECT lnid, lncode, lnname, parenttype, parentlnid FROM ml_langs");
+    $sth->execute;
+    while ( my ( $id, $code, $name, $ptype, $pid ) = $sth->fetchrow_array ) {
+        $LN_ID{$id} = $LN_CODE{$code} = {
+            'lnid'       => $id,
+            'lncode'     => $code,
+            'lnname'     => $name,
+            'parenttype' => $ptype,
+            'parentlnid' => $pid,
+        };
+    }
+    foreach ( values %LN_CODE ) {
+        next unless $_->{'parentlnid'};
+        push @{ $LN_ID{ $_->{'parentlnid'} }->{'children'} }, $_->{'lnid'};
+    }
+
+    $sth = $dbr->prepare("SELECT lnid, dmid, dmmaster FROM ml_langdomains");
+    $sth->execute;
+    while ( my ( $lnid, $dmid, $dmmaster ) = $sth->fetchrow_array ) {
+        $DM_ID{$dmid}->{'langs'}->{$lnid} = $dmmaster;
+    }
+
+    $LS_CACHED = 1;
+}
+
+sub langdat_file_of_lang_itcode {
+    my ( $lang, $itcode, $want_cvs ) = @_;
+
+    my $langdat_file =
+        LJ::Lang::relative_langdat_file_of_lang_itcode( $lang, $itcode );
+
+    my $cvs_extra = "";
+    if ($want_cvs) {
+        if ( $lang eq "en" ) {
+            $cvs_extra = "/cvs/livejournal";
+        } else {
+            $cvs_extra = "/cvs/local";
+        }
+    }
+
+    return "$ENV{LJHOME}$cvs_extra/$langdat_file";
+}
+
+sub relative_langdat_file_of_lang_itcode {
+    my ( $lang, $itcode ) = @_;
+
+    my $root_lang       = "en";
+    my $root_lang_local = $LJ::DEFAULT_LANG;
+
+    my $base_file = "bin/upgrading/$lang\.dat";
+
+    # not a root or root_local lang, just return base file location
+    unless ( $lang eq $root_lang || $lang eq $root_lang_local ) {
+        return $base_file;
+    }
+
+    my $is_local = $lang eq $root_lang_local;
+
+    # is this a filename-based itcode?
+    if ( $itcode =~ m!^(/.+\.bml)! ) {
+        my $file = $1;
+
+        # given the filename of this itcode and the current
+        # source, what langdat file should we use?
+        my $langdat_file = "htdocs$file\.text";
+        $langdat_file .= $is_local ? ".local" : "";
+        return $langdat_file;
+    }
+
+    # not a bml file, goes into base .dat file
+    return $base_file;
+}
+
+sub itcode_for_langdat_file {
+    my ( $langdat_file, $itcode ) = @_;
+
+    # non-bml itcode, return full itcode path
+    unless ( $langdat_file =~ m!^/.+\.bml\.text(?:\.local)?$! ) {
+        return $itcode;
+    }
+
+    # bml itcode, strip filename and return
+    if ( $itcode =~ m!^/.+\.bml(\..+)! ) {
+        return $1;
+    }
+
+    # fallback -- full $itcode
+    return $itcode;
+}
+
+sub get_chgtime_unix {
+    my ( $lncode, $dmid, $itcode ) = @_;
+    load_lang_struct() unless $LS_CACHED;
+
+    $dmid = int( $dmid || 1 );
+
+    my $l = get_lang($lncode) or return "No lang info for lang $lncode";
+    my $lnid = $l->{'lnid'}
+        or die "Could not get lang_id for lang $lncode";
+
+    my $itid = LJ::Lang::get_itemid( $dmid, $itcode )
+        or return 0;
+
+    my $dbr     = LJ::get_db_reader();
+    my $chgtime = $dbr->selectrow_array(
+        'SELECT chgtime FROM ml_latest WHERE dmid=? AND itid=? AND lnid=?',
+        undef, $dmid, $itid, $lnid, );
+    die $dbr->errstr if $dbr->err;
+
+    return $chgtime ? LJ::TimeUtil->mysqldate_to_time($chgtime) : 0;
+}
+
+sub get_itemid {
+    my ( $dmid, $itcode, $opts ) = @_;
+    load_lang_struct() unless $LS_CACHED;
+
+    if ( length $itcode > MAXIMUM_ITCODE_LENGTH ) {
+        warn "'$itcode' exceeds maximum code length, truncating to "
+            . MAXIMUM_ITCODE_LENGTH
+            . " symbols";
+        $itcode = substr( $itcode, 0, MAXIMUM_ITCODE_LENGTH );
+    }
+
+    my $dbr  = LJ::get_db_reader();
+    my $itid = $dbr->selectrow_array(
+        "SELECT itid FROM ml_items WHERE dmid=? AND itcode=?",
+        undef, $dmid, $itcode, );
+    return $itid if defined $itid;
+
+    my $dbh = LJ::get_db_writer();
+    return 0 unless $dbh;
+
+    # allocate a new id
+    LJ::get_lock( $dbh, 'global', 'mlitem_dmid' ) || return 0;
+    $itid = $dbh->selectrow_array(
+        "SELECT MAX(itid)+1 FROM ml_items WHERE dmid=?",
+        undef, $dmid, );
+    $itid ||= 1;    # if the table is empty, NULL+1 == NULL
+
+    my $affected = $dbh->do(
+        qq{
+            INSERT IGNORE INTO ml_items (dmid, itid, itcode, notes)
+            VALUES (?, ?, ?, ?)
+        },
+        undef, $dmid, $itid, $itcode, $opts->{'notes'},
+    );
+    LJ::release_lock( $dbh, 'global', 'mlitem_dmid' );
+
+    die $dbh->errstr if $dbh->err;
+    unless ($affected) {
+        $itid = $dbh->selectrow_array(
+            "SELECT itid FROM ml_items WHERE dmid=? AND itcode=?",
+            undef, $dmid, $itcode, );
+    }
+
+    return $itid;
+}
+
+# this is called when editing text from a web UI.
+# first try and run a local hook to save the text,
+# if that fails then just call set_text
+
+# returns ($success, $responsemsg) where responsemsg can be output
+# from whatever saves the text
+sub web_set_text {
+    my ( $dmid, $lncode, $itcode, $text, $opts ) = @_;
+
+    my $resp     = '';
+    my $hook_ran = 0;
+
+    if ( LJ::are_hooks('web_set_text') ) {
+        $hook_ran =
+            LJ::run_hook( 'web_set_text', $dmid, $lncode, $itcode, $text,
+            $opts, );
+    }
+
+    # save in the db
+    my $save_success =
+        LJ::Lang::set_text( $dmid, $lncode, $itcode, $text, $opts );
+
+    $resp = LJ::Lang::last_error() unless $save_success;
+    warn $resp if !$save_success && $LJ::IS_DEV_SERVER;
+
+    return ( $save_success, $resp );
+}
+
+sub set_text {
+    my ( $dmid, $lncode, $itcode, $text, $opts ) = @_;
+    load_lang_struct() unless $LS_CACHED;
+
+    my $l = $LN_CODE{$lncode} or return set_error("Language not defined.");
+    my $lnid = $l->{'lnid'};
+
+    # is this domain/language request even possible?
+    return set_error("Bogus domain")
+        unless exists $DM_ID{$dmid};
+
+    return set_error("Bogus lang for that domain")
+        unless exists $DM_ID{$dmid}->{'langs'}->{$lnid};
+
+    my $itid = get_itemid( $dmid, $itcode, { 'notes' => $opts->{'notes'} } );
+    return set_error("Couldn't allocate itid.") unless $itid;
+
+    my $dbh   = LJ::get_db_writer();
+    my $txtid = 0;
+
+    my $oldtextid = $dbh->selectrow_array(
+        "SELECT MAX(txtid) FROM ml_text WHERE lnid=? AND dmid=? AND itid=?",
+        undef, $lnid, $dmid, $itid, );
+
+    if ( defined $text ) {
+        my $userid = $opts->{'userid'} + 0;
+
+        # Strip bad characters
+        $text =~ s/\r//;
+
+        LJ::get_lock( $dbh, 'global', 'ml_text_txtid' ) || return 0;
+
+        $txtid = $dbh->selectrow_array(
+            "SELECT MAX(txtid)+1 FROM ml_text WHERE dmid=?",
+            undef, $dmid, );
+        $txtid ||= 1;
+
+        $dbh->do(
+            qq{
+                INSERT INTO ml_text (dmid, txtid, lnid, itid, text, userid)
+                VALUES (?, ?, ?, ?, ?, ?)
+            },
+            undef, $dmid, $txtid, $lnid, $itid, $text, $userid,
+        );
+        LJ::release_lock( $dbh, 'global', 'ml_text_txtid' );
+
+        return set_error( "Error inserting ml_text: " . $dbh->errstr )
+            if $dbh->err;
+    }
+
+    if ( $opts->{'txtid'} ) {
+        $txtid = $opts->{'txtid'} + 0;
+    }
+
+    my $revid     = LJ::alloc_global_counter("ml_latest_updates_counter");
+    my $staleness = int $opts->{'staleness'};
+    $dbh->do(
+        qq{
+            REPLACE INTO ml_latest
+            (lnid, dmid, itid, txtid, chgtime, staleness, revid)
+            VALUES (?, ?, ?, ?, NOW(), ?, ?)
+        },
+        undef, $lnid, $dmid, $itid, $txtid, $staleness, $revid,
+    );
+
+    return set_error( "Error inserting ml_latest: " . $dbh->errstr )
+        if $dbh->err;
+
+    LJ::MemCache::set( "ml.${lncode}.${dmid}.${itcode}", $text )
+        if defined $text;
+
+    my @langids;
+    my $langids;
+    my $vals;
+
+    my $rec;
+    $rec = sub {
+        my $l   = shift;
+        my $rec = shift;
+        foreach my $cid ( @{ $l->{'children'} } ) {
+            my $clid = $LN_ID{$cid};
+            if ( $opts->{'childrenlatest'} ) {
+                $revid =
+                    LJ::alloc_global_counter("ml_latest_updates_counter");
+                my $stale = $clid->{'parenttype'} eq "diff" ? 3 : 0;
+
+                # set descendants to use this mapping:
+                $dbh->do(
+                    qq{
+                        INSERT IGNORE INTO ml_latest
+                        (lnid, dmid, itid, txtid, chgtime, staleness, revid)
+                        VALUES (?, ?, ?, ?, NOW(), ?, ?)
+                    },
+                    undef, $cid, $dmid, $itid, $txtid, $stale, $revid,
+                );
+            }
+            push @langids, $cid;
+
+            LJ::MemCache::delete("ml.$clid->{'lncode'}.${dmid}.${itcode}");
+            $rec->($clid);
+        }
+    };
+    $rec->($l);
+
+    my $langids_in = join( ',', map { int $_ } @langids );
+
+    # update languages that have no translation yet
+    $revid = LJ::alloc_global_counter("ml_latest_updates_counter");
+    if (@langids) {
+        if ($oldtextid) {
+            $dbh->do(
+                qq{
+                    UPDATE ml_latest
+                    SET txtid=?, revid=?
+                    WHERE
+                        dmid=? AND
+                        lnid IN ($langids_in)
+                        AND itid=? AND
+                        txtid=
+                },
+                undef, $txtid, $revid, $dmid, $itid, $oldtextid,
+            );
+        } else {
+            $dbh->do(
+                qq{
+                    UPDATE ml_latest
+                    SET txtid=?, revid=?
+                    WHERE
+                        dmid=> AND
+                        lnid IN ($langids_in) AND
+                        itid=? AND
+                        staleness >= 3
+                },
+                undef, $txtid, $revid, $dmid, $itid,
+            );
+        }
+    }
+
+    if ( $opts->{'changeseverity'} && @langids ) {
+        my $newstale = $opts->{'changeseverity'} == 2 ? 2 : 1;
+        $dbh->do(
+            qq{
+                UPDATE ml_latest
+                SET staleness=?
+                WHERE
+                    lnid IN ($langids_in) AND
+                    dmid=? AND
+                    itid=? AND
+                    txtid<>? AND
+                    staleness < ?
+            },
+            undef, $newstale, $dmid, $itid, $txtid, $newstale,
+        );
+    }
+
+    LJ::MemCache::set( 'ml_latest_updates_counter', $revid );
+
+    return 1;
+}
+
+sub remove_text {
+    my ( $dmid, $itcode, $lncode ) = @_;
+
+    my $dbh = LJ::get_db_writer();
+
+    my $itid = $dbh->selectrow_array(
+        'SELECT itid FROM ml_items WHERE dmid=? AND itcode=?',
+        undef, $dmid, $itcode, );
+    die "Unknown item code $itcode." unless $itid;
+
+    # need to delete everything from: ml_items ml_latest ml_text
+
+    $dbh->do( 'DELETE FROM ml_items WHERE dmid=? AND itid=?',
+        undef, $dmid, $itid, );
+
+    my $txtids = $dbh->selectcol_arrayref(
+        'SELECT txtid FROM ml_latest WHERE dmid=? AND itid=?',
+        undef, $dmid, $itid, );
+
+    $dbh->do( 'DELETE FROM ml_latest WHERE dmid=? AND itid=?',
+        undef, $dmid, $itid, );
+
+    if (@$txtids) {
+        my $txtid_bind = join( ",", map {'?'} @$txtids );
+        $dbh->do(
+            "DELETE FROM ml_text WHERE dmid=? AND txtid IN ($txtid_bind)",
+            undef, $dmid, @$txtids, );
+    }
+
+    # delete from memcache if lncode is defined
+    LJ::MemCache::delete("ml.${lncode}.${dmid}.${itcode}") if $lncode;
+
+    return 1;
+}
+
+sub get_effective_lang {
+
+    return LJ::run_hook('effective_lang')
+        if LJ::are_hooks('effective_lang');
+
+    my $lang;
+    if ( LJ::is_web_context() ) {
+        $lang = BML::get_language();
+    }
+
+    if ( my $remote = LJ::get_remote() ) {
+
+        # we have a user; try their browse language
+        $lang ||= $remote->prop("browselang");
+    }
+
+    load_lang_struct() unless $LS_CACHED;
+
+    # did we get a valid language code?
+    if ( $lang && $LN_CODE{$lang} ) {
+        return $lang;
+    }
+
+    # had no language code, or invalid.  return default
+    return $LJ::DEFAULT_LANG;
+}
+
+sub get_remote_lang {
+    if ( my $remote = LJ::get_remote() ) {
+        return $remote->prop('browselang')
+            || $LJ::DEFAULT_LANG;
+    }
+
+    if ( LJ::is_web_context() ) {
+        return BML::get_language();
+    }
+
+    return $LJ::DEFAULT_LANG;
+}
+
+sub ml {
+    my ( $code, $vars ) = @_;
+
+    if ( LJ::is_web_context() ) {
+
+        # this means we should use BML::ml and not do our own handling
+        my $text = BML::ml( $code, $vars );
+        $LJ::_ML_USED_STRINGS{$code} = $text if $LJ::IS_DEV_SERVER;
+        return $text;
+    }
+
+    my $lang = LJ::Lang::get_effective_lang();
+    return get_text( $lang, $code, undef, $vars );
+}
+
+sub string_exists {
+    my ( $code, $vars ) = @_;
+
+    my $string = LJ::Lang::ml( $code, $vars );
+    return LJ::Lang::is_missing_string($string) ? 0 : 1;
+}
+
+# LJ::Lang::ml will return a number of values for "invalid string"
+# -- this function will tell you if the value is one of
+#    those values.  gross.
+sub is_missing_string {
+    my $string = shift;
+
+    return (   $string eq ""
+            || $string =~ /^\[missing string/
+            || $string =~ /^\[uhhh:/ ) ? 1 : 0;
+}
+
+sub get_text {
+    my ( $lang, $code, $dmid, $vars ) = @_;
+    $lang ||= $LJ::DEFAULT_LANG;
+    $dmid ||= 1;
+
+    my $from_db = sub {
+        my $text = get_text_multi( $lang, $dmid, [$code] );
+        return $text->{$code};
+    };
+
+    my $_from_files = sub {
+        my ( $localcode, @files );
+        if ( $code =~ m!^(/.+\.bml)(\..+)! ) {
+            my $file;
+            ( $file, $localcode ) = ( "$LJ::HTDOCS$1", $2 );
+            @files = ( "$file.text.local", "$file.text" );
+        } else {
+            $localcode = $code;
+            @files     = (
+                "$LJ::HOME/bin/upgrading/$LJ::DEFAULT_LANG.dat",
+                "$LJ::HOME/bin/upgrading/en.dat"
+            );
+        }
+
+        my $dbmodtime = LJ::Lang::get_chgtime_unix( $lang, $dmid, $code );
+        foreach my $tf (@files) {
+            next unless -e $tf;
+
+            # compare file modtime to when the string was updated in the DB.
+            # whichever is newer is authoritative
+            my $fmodtime = ( stat $tf )[9];
+            return $from_db->() if !$fmodtime || $dbmodtime > $fmodtime;
+
+            my $ldf = $LJ::REQ_LANGDATFILE{$tf} ||= LJ::LangDatFile->new($tf);
+            my $val = $ldf->value($localcode);
+            return $val if $val;
+        }
+        return "[missing string $code]";
+    };
+
+    my $from_files = sub {
+        my $cache_key = "ml.${lang}.${dmid}.${code}";
+        return $TXT_CACHE{$cache_key} ||= $_from_files->();
+    };
+
+    ##
+    my $gen_mld = LJ::Lang::get_dom('general');
+    my $is_gen_dmid = defined $dmid ? $dmid == $gen_mld->{dmid} : 1;
+    my $text;
+
+    if (   $LJ::IS_DEV_SERVER
+        && $is_gen_dmid
+        && ( $lang eq "en" || $lang eq $LJ::DEFAULT_LANG ) )
+    {
+        $text = $from_files->();
+    } else {
+        $text = $from_db->();
+    }
+
+    if ($vars) {
+
+        # the following regexp parses the [[?num|singular|plural1|...]] syntax
+        $text =~ s{
+            \[\[\?      # opening literal '[[?'
+            ([\w\-]+)   # the number key
+            \|          # the pipe delimiter
+            (.+?)       # singular/plural variants
+            \]\]        # closing literal ']]'
+        }
+        {resolve_plural($lang, $vars, $1, $2)}xeg;
+
+        # and the following merely substitutes the keys:
+        $text =~ s{
+            \[\[        # opening literal '[['
+            ([^\[]+?)   # the key
+            \]\]        # closing literal ']]'
+        }
+        {$vars->{$1}}xg;
+    }
+
+    $LJ::_ML_USED_STRINGS{$code} = $text if $LJ::IS_DEV_SERVER;
+
+    return $text || ( $LJ::IS_DEV_SERVER ? "[uhhh: $code]" : "" );
+}
+
+# Loads multiple language strings at once.  These strings
+# cannot however contain variables, if you have variables
+# you wouldn't be calling this anyway!
+# args: $lang, $dmid, array ref of lang codes
+sub get_text_multi {
+    my ( $lang, $dmid, $codes ) = @_;
+
+    return {} unless $codes;
+    return { map { $_ => $_ } @$codes }
+        if $lang eq 'debug';
+
+    $dmid = int( $dmid || 1 );
+    $lang ||= $LJ::DEFAULT_LANG;
+    load_lang_struct() unless $LS_CACHED;
+
+    ## %strings: code --> text
+    my %strings;
+
+    ## normalize the codes: all chars must be in lower case
+    ## MySQL string comparison isn't case-sensitive, but memcaches keys are.
+    ## Caller will get %strings with keys in original case.
+    ##
+    ## Final note about case:
+    ##  Codes in disk .text files, mysql and bml files may be mixed-cased
+    ##  Codes in memcache and %TXT_CACHE are lower-case
+    ##  Codes are not case-sensitive
+
+    ## %lc_code: lower-case code --> original code
+    my %lc_codes = map { lc($_) => $_ } @$codes;
+
+    ## %memkeys: lower-case code --> memcache key
+    my %memkeys;
+    foreach my $code ( keys %lc_codes ) {
+        my $cache_key = "ml.${lang}.${dmid}.${code}";
+        my $text      = undef;
+        $text = $TXT_CACHE{$cache_key} unless $LJ::NO_ML_CACHE;
+
+        if ( defined $text ) {
+            $strings{ $lc_codes{$code} } = $text;
+            $LJ::_ML_USED_STRINGS{$code} = $text if $LJ::IS_DEV_SERVER;
+        } else {
+            $memkeys{$cache_key} = $code;
+        }
+    }
+
+    return \%strings unless %memkeys;
+
+    my $mem = LJ::MemCache::get_multi( keys %memkeys ) || {};
+
+    ## %dbload: lower-case key --> text; text may be empty (but defined) string
+    my %dbload;
+    foreach my $cache_key ( keys %memkeys ) {
+        my $code = $memkeys{$cache_key};
+        my $text = $mem->{$cache_key};
+
+        if ( defined $text ) {
+            $strings{ $lc_codes{$code} } = $text;
+            $LJ::_ML_USED_STRINGS{$code} = $text if $LJ::IS_DEV_SERVER;
+            $TXT_CACHE{$cache_key} = $text;
+        } else {
+
+            # we need to cache nonexistant/empty strings because
+            # otherwise we're running a lot of queries all the time
+            # to cache nonexistant strings, value of %dbload must be defined
+            $dbload{$code} = '';
+        }
+    }
+
+    return \%strings unless %dbload;
+
+    my $l = $LN_CODE{$lang};
+
+    # This shouldn't happen!
+    die "Unable to load language code: $lang" unless $l;
+
+    my $dbr = LJ::get_db_reader();
+    my $bind = join( ',', map {'?'} keys %dbload );
+
+    my $rows = $dbr->selectall_arrayref(
+        qq{
+            SELECT i.itcode, t.text
+            FROM ml_text t, ml_latest l, ml_items i
+            WHERE
+                t.dmid=? AND
+                t.txtid=l.txtid AND
+                l.dmid=? AND
+                l.lnid=? AND
+                l.itid=i.itid AND
+                i.dmid=? AND
+                i.itcode IN ($bind)
+        },
+        { 'Slice' => {} },
+        $dmid, $dmid, $l->{'lnid'}, $dmid, keys %dbload,
+    );
+
+    # now replace the empty strings with the defined ones
+    # that we got back from the database
+    foreach my $row (@$rows) {
+
+        # some MySQL codes might be mixed-case
+        $dbload{ lc $row->{'itcode'} } = $row->{'text'};
+    }
+
+    while ( my ( $code, $text ) = each %dbload ) {
+        $strings{ $lc_codes{$code} } = $text;
+        $LJ::_ML_USED_STRINGS{$code} = $text if $LJ::IS_DEV_SERVER;
+
+        my $cache_key = "ml.${lang}.${dmid}.${code}";
+        $TXT_CACHE{$cache_key} = $text;
+
+        if ($text) {
+            LJ::MemCache::set( $cache_key, $text );
+        } else {
+            ## Do not cache empty values forever - they may be inserted later.
+            ## This is a hack, what we actually need is a mechanism to delete
+            ## the entire language tree for a given $code if it's updated.
+            LJ::MemCache::set( $cache_key, $text, 24 * 3600 );
+        }
+    }
+
+    return \%strings;
+}
+
+sub get_lang_names {
+    my @langs = @_;
+    push @langs, @LJ::LANGS unless @langs;
+
+    my $list = LJ::MemCache::get("langnames");
+    return $list if $list;
+
+    $list = [];
+    foreach my $code (@langs) {
+        my $l = LJ::Lang::get_lang($code);
+        next unless $l;
+
+        my $item = "langname.$code";
+
+        ## Native lang name
+        my $namenative = LJ::Lang::get_text( $l->{'lncode'}, $item );
+
+        push @$list, $code, $namenative;
+    }
+
+    ## cache name on 5 min
+    LJ::MemCache::set( 'langnames' => $list, 3660 );
+
+    return $list;
+}
+
+sub set_lang {
+    my $lang = shift;
+
+    my $l      = LJ::Lang::get_lang($lang);
+    my $remote = LJ::get_remote();
+
+    # default cookie value to set
+    my $cval = $l->{'lncode'} . '/' . time();
+
+    # if logged in, change userprop and make cookie expiration
+    # the same as their login expiration
+    if ($remote) {
+        $remote->set_prop( 'browselang' => $l->{lncode} );
+
+        if ( $remote->{'_session'}->{'exptype'} eq 'long' ) {
+            $cval = [ $cval, $remote->{'_session'}->{'timeexpire'} ];
+        }
+    }
+
+    # set cookie
+    $BML::COOKIE{'langpref'} = $cval;
+
+    # set language through BML so it will apply immediately
+    BML::set_language( $l->{'lncode'} );
+
+    return;
+}
+
+# The translation system supports the ability to add multiple plural forms of
+# the word given different rules in a languge. This functionality is much like
+# the plural support in the S2 styles code. To use this code you must use the
+# BML::ml function and pass the number of items as one of the variables. To
+# make sure that you are allowing the utmost compatibility for each language
+# you should not hardcode the placement of the number of items in relation to
+# the noun.  Let the translation string do this for you. A translation string
+# is in the format of, with num being the variable storing the number of items.
+# =[[num]] [[?num|singular|plural1|plural2|pluralx]]
+
+sub resolve_plural {
+    my ( $lang, $vars, $varname, $wordlist ) = @_;
+
+    my $count       = $vars->{$varname};
+    my @wlist       = split( /\|/, $wordlist );
+    my $plural_form = plural_form( $lang, $count );
+    return $wlist[$plural_form];
+}
+
+my %PLURAL_FORMS_HANDLERS = (
+    'be' => \&plural_form_ru,
+    'en' => \&plural_form_en,
+    'fr' => \&plural_form_fr,
+    'hu' => \&plural_form_singular,
+    'is' => \&plural_form_is,
+    'ja' => \&plural_form_singular,
+    'lt' => \&plural_form_lt,
+    'lv' => \&plural_form_lv,
+    'pl' => \&plural_form_pl,
+    'pt' => \&plural_form_fr,
+    'ru' => \&plural_form_ru,
+    'tr' => \&plural_form_singular,
+    'uk' => \&plural_form_ru,
+);
+
+sub plural_form {
+    my ( $lang, $count ) = @_;
+
+    my $lang_short = substr( $lang, 0, 2 );
+    my $handler = $PLURAL_FORMS_HANDLERS{$lang_short} || \&plural_form_en;
+
+    return $handler->($count);
+}
+
+# English, Danish, German, Norwegian, Swedish, Estonian, Finnish, Greek,
+# Hebrew, Italian, Spanish, Esperanto
+sub plural_form_en {
+    my ($count) = @_;
+
+    return 0 if $count == 1;
+    return 1;
+}
+
+# French, Portugese, Brazilian Portuguese
+sub plural_form_fr {
+    my ($count) = @_;
+
+    return 1 if $count > 1;
+    return 0;
+}
+
+# Croatian, Czech, Russian, Slovak, Ukrainian, Belarusian
+sub plural_form_ru {
+    my ($count) = @_;
+
+    return 0 if ( $count % 10 == 1 && $count % 100 != 11 );
+    return 1
+        if ( $count % 10 >= 2 && $count % 10 <= 4 )
+        && ( $count % 100 < 10 || $count % 100 >= 20 );
+
+    return 2;
+}
+
+# Polish
+sub plural_form_pl {
+    my ($count) = @_;
+
+    return 0 if ( $count == 1 );
+
+    return 1
+        if ( $count % 10 >= 2 && $count % 10 <= 4 )
+        && ( $count % 100 < 10 || $count % 100 >= 20 );
+
+    return 2;
+}
+
+# Lithuanian
+sub plural_form_lt {
+    my ($count) = @_;
+
+    return 0 if ( $count % 10 == 1 && $count % 100 != 11 );
+
+    return 1
+        if ( $count % 10 >= 2 )
+        && ( $count % 100 < 10 || $count % 100 >= 20 );
+
+    return 2;
+}
+
+# Hungarian, Japanese, Korean (not supported), Turkish
+sub plural_form_singular {
+    return 0;
+}
+
+# Latvian
+sub plural_form_lv {
+    my ($count) = @_;
+
+    return 0 if ( $count % 10 == 1 && $count % 100 != 11 );
+    return 1 if ( $count != 0 );
+    return 2;
+}
+
+# Icelandic
+sub plural_form_is {
+    my ($count) = @_;
+
+    return 0 if ( $count % 10 == 1 and $count % 100 != 11 );
+    return 1;
+}
+
+1;

Modified: trunk/cgi-bin/LJ/NewWorker.pm
===================================================================
--- trunk/cgi-bin/LJ/NewWorker.pm	2011-08-30 04:13:09 UTC (rev 19887)
+++ trunk/cgi-bin/LJ/NewWorker.pm	2011-08-30 06:35:24 UTC (rev 19888)
@@ -7,7 +7,7 @@
 
 require 'ljlib.pl';
 require 'ljprotocol.pl';
-require 'ljlang.pl';
+use LJ::Lang;
 
 my $name        = 0;    # Name of this worker
 my $quantity    = 0;    # Quantity of workers (0 means no-daemons)

Modified: trunk/cgi-bin/ljlang.pl
===================================================================
--- trunk/cgi-bin/ljlang.pl	2011-08-30 04:13:09 UTC (rev 19887)
+++ trunk/cgi-bin/ljlang.pl	2011-08-30 06:35:24 UTC (rev 19888)
@@ -1,872 +1 @@
-#!/usr/bin/perl
-#
-
-use strict;
-use lib "$ENV{LJHOME}/cgi-bin";
-require "ljhooks.pl";
-
-use Class::Autouse qw(
-                      LJ::LangDatFile
-                      );
-
-package LJ::Lang;
-#use LJ::ML;
-use constant MAXIMUM_ITCODE_LENGTH => 80;
-use LJ::TimeUtil;
-
-my @day_short   = (qw[Sun Mon Tue Wed Thu Fri Sat]);
-my @day_long    = (qw[Sunday Monday Tuesday Wednesday Thursday Friday Saturday]);
-my @month_short = (qw[Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec]);
-my @month_long  = (qw[January February March April May June July August September October November December]);
-
-# get entire array of days and months
-sub day_list_short   { return @LJ::Lang::day_short;   }
-sub day_list_long    { return @LJ::Lang::day_long;    }
-sub month_list_short { return @LJ::Lang::month_short; }
-sub month_list_long  { return @LJ::Lang::month_long;  }
-
-# access individual day or month given integer
-sub day_short   { return   $day_short[$_[0] - 1]; }
-sub day_long    { return    $day_long[$_[0] - 1]; }
-sub month_short { return $month_short[$_[0] - 1]; }
-sub month_long  { return  $month_long[$_[0] - 1]; }
-
-# lang codes for individual day or month given integer
-sub day_short_langcode   { return "date.day."   . lc(LJ::Lang::day_long(@_))    . ".short"; }
-sub day_long_langcode    { return "date.day."   . lc(LJ::Lang::day_long(@_))    . ".long";  }
-sub month_short_langcode { return "date.month." . lc(LJ::Lang::month_long(@_))  . ".short"; }
-sub month_long_langcode  { return "date.month." . lc(LJ::Lang::month_long(@_))  . ".long";  }
-sub month_long_genitive_langcode  { return "date.month." . lc(LJ::Lang::month_long(@_))  . ".genitive";  }
-
-## ordinal suffix
-sub day_ord {
-    my $day = shift;
-
-    # teens all end in 'th'
-    if ($day =~ /1\d$/) { return "th"; }
-
-    # otherwise endings in 1, 2, 3 are special
-    if ($day % 10 == 1) { return "st"; }
-    if ($day % 10 == 2) { return "nd"; }
-    if ($day % 10 == 3) { return "rd"; }
-
-    # everything else (0,4-9) end in "th"
-    return "th";
-}
-
-sub time_format
-{
-    my ($hours, $h, $m, $formatstring) = @_;
-
-    if ($formatstring eq "short") {
-        if ($hours == 12) {
-            my $ret;
-            my $ap = "a";
-            if ($h == 0) { $ret .= "12"; }
-            elsif ($h < 12) { $ret .= ($h+0); }
-            elsif ($h == 12) { $ret .= ($h+0); $ap = "p"; }
-            else { $ret .= ($h-12); $ap = "p"; }
-            $ret .= sprintf(":%02d$ap", $m);
-            return $ret;
-        } elsif ($hours == 24) {
-            return sprintf("%02d:%02d", $h, $m);
-        }
-    }
-    return "";
-}
-
-#### ml_ stuff:
-my $LS_CACHED = 0;
-my %DM_ID = ();     # id -> { type, args, dmid, langs => { => 1, => 0, => 1 } }
-my %DM_UNIQ = ();   # "$type/$args" => ^^^
-my %LN_ID = ();     # id -> { ..., ..., 'children' => [ $ids, .. ] }
-my %LN_CODE = ();   # $code -> ^^^^
-my $LAST_ERROR;
-my %TXT_CACHE;
-
-if ($LJ::IS_DEV_SERVER || $LJ::IS_LJCOM_BETA) {
-    our $_hook_is_installed;
-    LJ::register_hook('start_request', sub { %TXT_CACHE = (); }) unless $_hook_is_installed++;
-}
-
-sub last_error
-{
-    return $LAST_ERROR;
-}
-
-sub set_error
-{
-    $LAST_ERROR = $_[0];
-    return 0;
-}
-
-sub get_lang
-{
-    my $code = shift;
-    load_lang_struct() unless $LS_CACHED;
-    return $LN_CODE{$code};
-}
-
-sub get_lang_id
-{
-    my $id = shift;
-    load_lang_struct() unless $LS_CACHED;
-    return $LN_ID{$id};
-}
-
-sub get_dom
-{
-    my $dmcode = shift;
-    load_lang_struct() unless $LS_CACHED;
-    return $DM_UNIQ{$dmcode};
-}
-
-sub get_dom_id
-{
-    my $dmid = shift;
-    load_lang_struct() unless $LS_CACHED;
-    return $DM_ID{$dmid};
-}
-
-sub get_domains
-{
-    load_lang_struct() unless $LS_CACHED;
-    return values %DM_ID;
-}
-
-sub get_root_lang
-{
-    my $dom = shift;  # from, say, get_dom
-    return undef unless ref $dom eq "HASH";
-
-    my $lang_override = LJ::run_hook("root_lang_override", $dom);
-    return get_lang($lang_override) if $lang_override;
-
-    foreach (keys %{$dom->{'langs'}}) {
-        if ($dom->{'langs'}->{$_}) {
-            return get_lang_id($_);
-        }
-    }
-    return undef;
-}
-
-sub load_lang_struct
-{
-    return 1 if $LS_CACHED;
-    my $dbr = LJ::get_db_reader();
-    return set_error("No database available") unless $dbr;
-    my $sth;
-
-    $sth = $dbr->prepare("SELECT dmid, type, args FROM ml_domains");
-    $sth->execute;
-    while (my ($dmid, $type, $args) = $sth->fetchrow_array) {
-        my $uniq = $args ? "$type/$args" : $type;
-        $DM_UNIQ{$uniq} = $DM_ID{$dmid} = {
-            'type' => $type, 'args' => $args, 'dmid' => $dmid,
-            'uniq' => $uniq,
-        };
-    }
-
-    $sth = $dbr->prepare("SELECT lnid, lncode, lnname, parenttype, parentlnid FROM ml_langs");
-    $sth->execute;
-    while (my ($id, $code, $name, $ptype, $pid) = $sth->fetchrow_array) {
-        $LN_ID{$id} = $LN_CODE{$code} = {
-            'lnid' => $id,
-            'lncode' => $code,
-            'lnname' => $name,
-            'parenttype' => $ptype,
-            'parentlnid' => $pid,
-        };
-    }
-    foreach (values %LN_CODE) {
-        next unless $_->{'parentlnid'};
-        push @{$LN_ID{$_->{'parentlnid'}}->{'children'}}, $_->{'lnid'};
-    }
-
-    $sth = $dbr->prepare("SELECT lnid, dmid, dmmaster FROM ml_langdomains");
-    $sth->execute;
-    while (my ($lnid, $dmid, $dmmaster) = $sth->fetchrow_array) {
-        $DM_ID{$dmid}->{'langs'}->{$lnid} = $dmmaster;
-    }
-
-    $LS_CACHED = 1;
-}
-
-sub langdat_file_of_lang_itcode
-{
-    my ($lang, $itcode, $want_cvs) = @_;
-
-    my $langdat_file = LJ::Lang::relative_langdat_file_of_lang_itcode($lang, $itcode);
-    my $cvs_extra = "";
-    if ($want_cvs) {
-        if ($lang eq "en") {
-            $cvs_extra = "/cvs/livejournal";
-        } else {
-            $cvs_extra = "/cvs/local";
-        }
-    }
-    return "$ENV{LJHOME}$cvs_extra/$langdat_file";
-}
-
-sub relative_langdat_file_of_lang_itcode
-{
-    my ($lang, $itcode) = @_;
-
-    my $root_lang = "en";
-    my $root_lang_local = $LJ::DEFAULT_LANG;
-
-    my $base_file = "bin/upgrading/$lang\.dat";
-
-    # not a root or root_local lang, just return base file location
-    unless ($lang eq $root_lang || $lang eq $root_lang_local) {
-        return $base_file;
-    }
-
-    my $is_local = $lang eq $root_lang_local;
-
-    # is this a filename-based itcode?
-    if ($itcode =~ m!^(/.+\.bml)!) {
-        my $file = $1;
-
-        # given the filename of this itcode and the current
-        # source, what langdat file should we use?
-        my $langdat_file = "htdocs$file\.text";
-        $langdat_file .= $is_local ? ".local" : "";
-        return $langdat_file;
-    }
-
-    # not a bml file, goes into base .dat file
-    return $base_file;
-}
-
-sub itcode_for_langdat_file {
-    my ($langdat_file, $itcode) = @_;
-
-    # non-bml itcode, return full itcode path
-    unless ($langdat_file =~ m!^/.+\.bml\.text(?:\.local)?$!) {
-        return $itcode;
-    }
-
-    # bml itcode, strip filename and return
-    if ($itcode =~ m!^/.+\.bml(\..+)!) {
-        return $1;
-    }
-
-    # fallback -- full $itcode
-    return $itcode;
-}
-
-sub get_chgtime_unix
-{
-    my ($lncode, $dmid, $itcode) = @_;
-    load_lang_struct() unless $LS_CACHED;
-
-    $dmid = int($dmid || 1);
-
-    my $l = get_lang($lncode) or return "No lang info for lang $lncode";
-    my $lnid = $l->{'lnid'}
-        or die "Could not get lang_id for lang $lncode";
-
-    my $itid = LJ::Lang::get_itemid($dmid, $itcode)
-        or return 0;
-
-    my $dbr = LJ::get_db_reader();
-    $dmid += 0;
-    my $chgtime = $dbr->selectrow_array("SELECT chgtime FROM ml_latest WHERE dmid=? AND itid=? AND lnid=?",
-                                        undef, $dmid, $itid, $lnid);
-    die $dbr->errstr if $dbr->err;
-    return $chgtime ? LJ::TimeUtil->mysqldate_to_time($chgtime) : 0;
-}
-
-sub get_itemid
-{
-    &LJ::nodb;
-    my ($dmid, $itcode, $opts) = @_;
-    load_lang_struct() unless $LS_CACHED;
-
-    if (length $itcode > MAXIMUM_ITCODE_LENGTH) {
-        warn "'$itcode' exceeds maximum code length, truncating to " . MAXIMUM_ITCODE_LENGTH . " symbols";
-        $itcode = substr($itcode, 0, MAXIMUM_ITCODE_LENGTH);
-    }
-
-    my $dbr = LJ::get_db_reader();
-    $dmid += 0;
-    my $itid = $dbr->selectrow_array("SELECT itid FROM ml_items WHERE dmid=$dmid AND itcode=?", undef, $itcode);
-    return $itid if defined $itid;
-
-    my $dbh = LJ::get_db_writer();
-    return 0 unless $dbh;
-
-    # allocate a new id
-    LJ::get_lock($dbh, 'global', 'mlitem_dmid') || return 0;
-    $itid = $dbh->selectrow_array("SELECT MAX(itid)+1 FROM ml_items WHERE dmid=?", undef, $dmid);
-    $itid ||= 1; # if the table is empty, NULL+1 == NULL
-    $dbh->do("INSERT INTO ml_items (dmid, itid, itcode, notes) ".
-             "VALUES (?, ?, ?, ?)", undef, $dmid, $itid, $itcode, $opts->{'notes'});
-    LJ::release_lock($dbh, 'global', 'mlitem_dmid');
-
-    if ($dbh->err) {
-        return $dbh->selectrow_array("SELECT itid FROM ml_items WHERE dmid=$dmid AND itcode=?",
-                                     undef, $itcode);
-    }
-    return $itid;
-}
-
-# this is called when editing text from a web UI.
-# first try and run a local hook to save the text,
-# if that fails then just call set_text
-
-# returns ($success, $responsemsg) where responsemsg can be output
-# from whatever saves the text
-sub web_set_text {
-    my ($dmid, $lncode, $itcode, $text, $opts) = @_;
-
-    my $resp = '';
-    my $hook_ran = 0;
-
-    if (LJ::are_hooks('web_set_text')) {
-        $hook_ran = LJ::run_hook('web_set_text', $dmid, $lncode, $itcode, $text, $opts);
-    }
-
-    # save in the db
-    my $save_success = LJ::Lang::set_text($dmid, $lncode, $itcode, $text, $opts);
-    $resp = LJ::Lang::last_error() unless $save_success;
-    warn $resp if ! $save_success && $LJ::IS_DEV_SERVER;
-
-    return ($save_success, $resp);
-}
-
-sub set_text
-{
-    &LJ::nodb;
-    my ($dmid, $lncode, $itcode, $text, $opts) = @_;
-    load_lang_struct() unless $LS_CACHED;
-
-    my $l = $LN_CODE{$lncode} or return set_error("Language not defined.");
-    my $lnid = $l->{'lnid'};
-    $dmid += 0;
-
-    # is this domain/language request even possible?
-    return set_error("Bogus domain")
-        unless exists $DM_ID{$dmid};
-    return set_error("Bogus lang for that domain")
-        unless exists $DM_ID{$dmid}->{'langs'}->{$lnid};
-
-    my $itid = get_itemid($dmid, $itcode, { 'notes' => $opts->{'notes'}});
-    return set_error("Couldn't allocate itid.") unless $itid;
-
-    my $dbh = LJ::get_db_writer();
-    my $txtid = 0;
-
-    my $oldtextid = $dbh->selectrow_array("SELECT MAX(txtid) FROM ml_text WHERE lnid=? AND dmid=? AND itid=?", undef, $lnid, $dmid, $itid);
-
-    if (defined $text) {
-        my $userid = $opts->{'userid'} + 0;
-        # Strip bad characters
-        $text =~ s/\r//;
-        my $qtext = $dbh->quote($text);
-        LJ::get_lock( $dbh, 'global', 'ml_text_txtid' ) || return 0;
-        $txtid = $dbh->selectrow_array("SELECT MAX(txtid)+1 FROM ml_text WHERE dmid=?", undef, $dmid);
-        $txtid ||= 1;
-        $dbh->do("INSERT INTO ml_text (dmid, txtid, lnid, itid, text, userid) ".
-                 "VALUES ($dmid, $txtid, $lnid, $itid, $qtext, $userid)");
-        LJ::release_lock( $dbh, 'global', 'ml_text_txtid' );
-        return set_error("Error inserting ml_text: ".$dbh->errstr) if $dbh->err;
-    }
-    if ($opts->{'txtid'}) {
-        $txtid = $opts->{'txtid'}+0;
-    }
-
-
-    my $revid     = LJ::alloc_global_counter("ml_latest_updates_counter");
-    my $staleness = int $opts->{'staleness'};
-    $dbh->do("REPLACE INTO ml_latest (lnid, dmid, itid, txtid, chgtime, staleness, revid) ".
-             "VALUES ($lnid, $dmid, $itid, $txtid, NOW(), $staleness, $revid)");
-    warn ("REPLACE INTO ml_latest (lnid, dmid, itid, txtid, chgtime, staleness, revid) ".
-          "VALUES ($lnid, $dmid, $itid, $txtid, NOW(), $staleness, $revid)")
-	     if $dbh->err;
-    return set_error("Error inserting ml_latest: ".$dbh->errstr) if $dbh->err;
-    LJ::MemCache::set("ml.${lncode}.${dmid}.${itcode}", $text) if defined $text;
-
-    my $langids;
-    {
-        my $vals;
-        my $rec = sub {
-            my $l = shift;
-            my $rec = shift;
-            foreach my $cid (@{$l->{'children'}}) {
-                my $clid = $LN_ID{$cid};
-                if ($opts->{'childrenlatest'}) {
-                    $revid = LJ::alloc_global_counter("ml_latest_updates_counter");
-                    my $stale = $clid->{'parenttype'} eq "diff" ? 3 : 0;
-                    $vals .= "," if $vals;
-                    $vals .= "($cid, $dmid, $itid, $txtid, NOW(), $stale, $revid)";
-                }
-                $langids .= "," if $langids;
-                $langids .= $cid+0;
-                LJ::MemCache::delete("ml.$clid->{'lncode'}.${dmid}.${itcode}");
-                $rec->($clid, $rec);
-            }
-        };
-        $rec->($l, $rec);
-
-        # set descendants to use this mapping
-        $dbh->do("INSERT IGNORE INTO ml_latest (lnid, dmid, itid, txtid, chgtime, staleness, revid) ".
-                 "VALUES $vals") if $vals;
-
-        # update languages that have no translation yet
-        if ($oldtextid) {
-            $revid = LJ::alloc_global_counter("ml_latest_updates_counter");
-            $dbh->do("UPDATE ml_latest SET txtid=$txtid, revid=$revid WHERE dmid=$dmid ".
-                 "AND lnid IN ($langids) AND itid=$itid AND txtid=$oldtextid") if $langids;
-        } else {
-            $revid = LJ::alloc_global_counter("ml_latest_updates_counter");
-            $dbh->do("UPDATE ml_latest SET txtid=$txtid, revid=$revid WHERE dmid=$dmid ".
-                 "AND lnid IN ($langids) AND itid=$itid AND staleness >= 3") if $langids;
-        }
-    }
-
-    if ($opts->{'changeseverity'} && $langids) {
-        my $newstale = $opts->{'changeseverity'} == 2 ? 2 : 1;
-        $dbh->do("UPDATE ml_latest SET staleness=$newstale WHERE lnid IN ($langids) AND ".
-                 "dmid=$dmid AND itid=$itid AND txtid<>$txtid AND staleness < $newstale");
-    }
-
-    #
-    LJ::MemCache::set('ml_latest_updates_counter', $revid);
-    
-    return 1;
-}
-
-sub remove_text {
-    my ($dmid, $itcode, $lncode) = @_;
-
-    my $dbh = LJ::get_db_writer();
-
-    my $itid = $dbh->selectrow_array("SELECT itid FROM ml_items WHERE dmid=? AND itcode=?",
-                                     undef, $dmid, $itcode);
-    die "Unknown item code $itcode." unless $itid;
-
-    # need to delete everything from: ml_items ml_latest ml_text
-
-    $dbh->do("DELETE FROM ml_items WHERE dmid=? AND itid=?",
-             undef, $dmid, $itid);
-
-    my @txtids = ();
-    my $sth = $dbh->prepare("SELECT txtid FROM ml_latest WHERE dmid=? AND itid=?");
-    $sth->execute($dmid, $itid);
-    while (my $txtid = $sth->fetchrow_array) {
-        push @txtids, $txtid;
-    }
-
-    $dbh->do("DELETE FROM ml_latest WHERE dmid=? AND itid=?",
-             undef, $dmid, $itid);
-
-    my $txtid_bind = join(",", map { "?" } @txtids);
-    $dbh->do("DELETE FROM ml_text WHERE dmid=? AND txtid IN ($txtid_bind)",
-             undef, $dmid, @txtids);
-
-    # delete from memcache if lncode is defined
-    LJ::MemCache::delete("ml.${lncode}.${dmid}.${itcode}") if $lncode;
-
-    return 1;
-}
-
-sub get_effective_lang {
-
-    return LJ::run_hook('effective_lang') if LJ::are_hooks('effective_lang');
-
-    my $lang;
-    if (LJ::is_web_context()) {
-        $lang = BML::get_language();
-    } 
-    if (my $remote = LJ::get_remote()) {
-        # we have a user; try their browse language
-        $lang ||= $remote->prop("browselang");
-    }
-
-    load_lang_struct() unless $LS_CACHED;
-
-    # did we get a valid language code?
-    if ($lang && $LN_CODE{$lang}) {
-        return $lang;
-    }
-
-    # had no language code, or invalid.  return default
-    return $LJ::DEFAULT_LANG;
-}
-
-sub get_remote_lang {
-    if ( my $remote = LJ::get_remote() ) {
-        return $remote->prop('browselang')
-            || $LJ::DEFAULT_LANG;
-    }
-
-    if ( LJ::is_web_context() ) {
-        return BML::get_language();
-    }
-
-    return $LJ::DEFAULT_LANG;
-}
-
-sub ml {
-    my ($code, $vars) = @_;
-
-    if (LJ::is_web_context()) {
-        # this means we should use BML::ml and not do our own handling
-        my $text = BML::ml($code, $vars);
-        $LJ::_ML_USED_STRINGS{$code} = $text if $LJ::IS_DEV_SERVER;
-        return $text;
-    }
-
-    my $lang = LJ::Lang::get_effective_lang();
-    return get_text($lang, $code, undef, $vars);
-}
-
-sub string_exists {
-    my ($code, $vars) = @_;
-
-    my $string = LJ::Lang::ml($code, $vars);
-    return LJ::Lang::is_missing_string($string) ? 0 : 1;
-}
-
-# LJ::Lang::ml will return a number of values for "invalid string"
-# -- this function will tell you if the value is one of
-#    those values.  gross.
-sub is_missing_string {
-    my $string = shift;
-
-    return ( $string eq "" ||
-             $string =~ /^\[missing string/ ||
-             $string =~ /^\[uhhh:/ ) ? 1 : 0;
-}
-
-sub get_text
-{
-    my ($lang, $code, $dmid, $vars) = @_;
-    $lang ||= $LJ::DEFAULT_LANG;
-    
-    my $from_db = sub {
-        my $text = get_text_multi($lang, $dmid, [ $code ]);
-        return $text->{$code};
-    };
-
-    my $_from_files = sub {
-        my ($localcode, @files);
-        if ($code =~ m!^(/.+\.bml)(\..+)!) {
-            my $file;
-            ($file, $localcode) = ("$LJ::HTDOCS$1", $2);
-            @files = ("$file.text.local", "$file.text");
-        } else {
-            $localcode = $code;
-            @files = ("$LJ::HOME/bin/upgrading/$LJ::DEFAULT_LANG.dat",
-                      "$LJ::HOME/bin/upgrading/en.dat");
-        }
-
-        my $dbmodtime = LJ::Lang::get_chgtime_unix($lang, $dmid, $code);
-        foreach my $tf (@files) {
-            next unless -e $tf;
-
-            # compare file modtime to when the string was updated in the DB.
-            # whichever is newer is authoritative
-            my $fmodtime = (stat $tf)[9];
-            return $from_db->() if !$fmodtime || $dbmodtime > $fmodtime;
-
-            my $ldf = $LJ::REQ_LANGDATFILE{$tf} ||= LJ::LangDatFile->new($tf);
-            my $val = $ldf->value($localcode);
-            return $val if $val;
-        }
-        return "[missing string $code]";
-    };
-
-    my $from_files = sub {
-        my $cache_key = "ml.${lang}.${dmid}.${code}";
-        return $TXT_CACHE{$cache_key} ||= $_from_files->();
-    };
- 
-
-    ##
-    my $gen_mld = LJ::Lang::get_dom('general');
-    my $is_gen_dmid = defined $dmid ? $dmid == $gen_mld->{dmid} : 1;
-    my $text = ($LJ::IS_DEV_SERVER && $is_gen_dmid &&
-                                      ($lang eq "en" ||
-                                       $lang eq $LJ::DEFAULT_LANG)) ?
-                                       $from_files->() :
-                                       $from_db->();
-=head
-    my $text = 
-        0 ? LJ::ML->get_text($lang, $dmid, $code) :
-        ($LJ::IS_DEV_SERVER && $is_gen_dmid &&
-                                      ($lang eq "en" ||
-                                       $lang eq $LJ::DEFAULT_LANG)) 
-                                       ? $from_files->()
-                                       : $from_db->();
-                                       #: LJ::ML->get_text($lang, $dmid, $code);
-=cut
-
-    if ($vars) {
-        $text =~ s/\[\[\?([\w\-]+)\|(.+?)\]\]/resolve_plural($lang, $vars, $1, $2)/eg;
-        $text =~ s/\[\[([^\[]+?)\]\]/$vars->{$1}/g;
-    }
-
-    $LJ::_ML_USED_STRINGS{$code} = $text if $LJ::IS_DEV_SERVER;
-
-    return $text || ($LJ::IS_DEV_SERVER ? "[uhhh: $code]" : "");
-}
-
-# Loads multiple language strings at once.  These strings
-# cannot however contain variables, if you have variables
-# you wouldn't be calling this anyway!
-# args: $lang, $dmid, array ref of lang codes
-sub get_text_multi
-{
-    my ($lang, $dmid, $codes) = @_;
-
-    return {} unless $codes;
-    return { map {$_ => $_} @$codes }
-        if $lang eq 'debug';
-
-    $dmid = int($dmid || 1);
-    $lang ||= $LJ::DEFAULT_LANG;
-    load_lang_struct() unless $LS_CACHED;
-    ## %strings: code --> text
-    my %strings;
-
-    ## normalize the codes: all chars must be in lower case
-    ## MySQL string comparison isn't case-sensitive, but memcaches keys are.
-    ## Caller will get %strings with keys in original case.
-    ##
-    ## Final note about case:  
-    ##  Codes in disk .text files, mysql and bml files may be mixed-cased
-    ##  Codes in memcache and %TXT_CACHE are lower-case
-    ##  Codes are not case-sensitive
-    
-    ## %lc_code: lower-case code --> original code
-    my %lc_codes = map { lc($_) => $_ } @$codes;
-    
-    ## %memkeys: lower-case code --> memcache key
-    my %memkeys; 
-    foreach my $code (keys %lc_codes) {
-        my $cache_key = "ml.${lang}.${dmid}.${code}";
-        my $text = undef;
-        $text = $TXT_CACHE{$cache_key} unless $LJ::NO_ML_CACHE;
-        
-        if (defined $text) {
-            $strings{ $lc_codes{$code} } = $text;
-            $LJ::_ML_USED_STRINGS{$code} = $text if $LJ::IS_DEV_SERVER;
-        } else {
-            $memkeys{$cache_key} = $code;
-        }
-    }
-
-    return \%strings unless %memkeys;
-
-    my $mem = LJ::MemCache::get_multi(keys %memkeys) || {};
-
-    ## %dbload: lower-case key --> text; text may be empty (but defined) string
-    my %dbload;
-    foreach my $cache_key (keys %memkeys) {
-        my $code = $memkeys{$cache_key};
-        my $text = $mem->{$cache_key};
-        
-        if (defined $text) {
-            $strings{ $lc_codes{$code} } = $text;
-            $LJ::_ML_USED_STRINGS{$code} = $text if $LJ::IS_DEV_SERVER;
-            $TXT_CACHE{$cache_key} = $text;
-        } else {
-            # we need to cache nonexistant/empty strings because otherwise we're running a lot of queries all the time
-            # to cache nonexistant strings, value of %dbload must be defined
-            $dbload{$code} = '';
-        }
-    }
-
-    return \%strings unless %dbload;
-
-    my $l = $LN_CODE{$lang};
-
-    # This shouldn't happen!
-    die ("Unable to load language code: $lang") unless $l;
-
-    my $dbr = LJ::get_db_reader();
-    my $bind = join(',', map { '?' } keys %dbload);
-    my $sth = $dbr->prepare("SELECT i.itcode, t.text".
-                            " FROM ml_text t, ml_latest l, ml_items i".
-                            " WHERE t.dmid=? AND t.txtid=l.txtid".
-                            " AND l.dmid=? AND l.lnid=? AND l.itid=i.itid".
-                            " AND i.dmid=? AND i.itcode IN ($bind)");
-    $sth->execute($dmid, $dmid, $l->{lnid}, $dmid, keys %dbload);
-
-    # now replace the empty strings with the defined ones that we got back from the database
-    while (my ($code, $text) = $sth->fetchrow_array) {
-        # some MySQL codes might be mixed-case
-        $dbload{ lc($code) } = $text;
-    }
-
-    while (my ($code, $text) = each %dbload) {
-        $strings{ $lc_codes{$code} } = $text;
-        $LJ::_ML_USED_STRINGS{$code} = $text if $LJ::IS_DEV_SERVER;
-
-        my $cache_key = "ml.${lang}.${dmid}.${code}";
-        $TXT_CACHE{$cache_key} = $text;
-        if ($text) {
-            LJ::MemCache::set($cache_key, $text);
-        } else {
-            ## Do not cache empty values forever - they may be inserted later.
-            ## This is a hack, what we actually need is a mechanism to delete
-            ## the entire language tree for a given $code if it's updated.
-            LJ::MemCache::set($cache_key, $text, 24*3600);
-        }
-    }
-
-    return \%strings;
-}
-
-sub get_lang_names {
-    my @langs = @_;
-    push @langs, @LJ::LANGS unless @langs;
-
-    my $list = LJ::MemCache::get("langnames");
-    return $list if $list;
-
-    $list = [];
-    foreach my $code (@langs) {
-        my $l = LJ::Lang::get_lang($code);
-        next unless $l;
-
-        my $item = "langname.$code";
-
-        ## Language name in current lang
-        # my $namethislang = LJ::Lang::ml($item);
-
-        ## Native lang name
-        my $namenative = LJ::Lang::get_text($l->{'lncode'}, $item);
-
-        push @$list, $code, $namenative;
-    }
-
-    ## cache name on 5 min
-    LJ::MemCache::set("langnames" => $list, 3660);
-
-    return $list;
-}
-
-sub set_lang {
-    my $lang = shift;
-
-    my $l = LJ::Lang::get_lang($lang);
-    my $remote = LJ::get_remote();
-
-    # default cookie value t...
 (truncated)
Tags: andy, livejournal, pl, pm, t
Subscribe
  • Post a new comment

    Error

    Anonymous comments are disabled in this journal

    default userpic

    Your reply will be screened

    Your IP address will be recorded 

  • 0 comments