Андрей (andy) wrote in changelog,
Андрей
andy
changelog

[livejournal] r17810: LJSUP-7462 (refactor userprops)

Committer: ailyin
LJSUP-7462 (refactor userprops)
U   trunk/bin/upgrading/update-db-general.pl
U   trunk/cgi-bin/LJ/MemCache.pm
A   trunk/cgi-bin/LJ/User/PropStorage/
A   trunk/cgi-bin/LJ/User/PropStorage/DB.pm
A   trunk/cgi-bin/LJ/User/PropStorage/Global.pm
A   trunk/cgi-bin/LJ/User/PropStorage/GlobalIndexed.pm
A   trunk/cgi-bin/LJ/User/PropStorage/Packed.pm
A   trunk/cgi-bin/LJ/User/PropStorage/UserClusterBlob.pm
A   trunk/cgi-bin/LJ/User/PropStorage/UserClusterLite.pm
A   trunk/cgi-bin/LJ/User/PropStorage.pm
U   trunk/cgi-bin/LJ/User.pm
U   trunk/cgi-bin/ljdb.pl
Modified: trunk/bin/upgrading/update-db-general.pl
===================================================================
--- trunk/bin/upgrading/update-db-general.pl	2010-11-29 04:54:45 UTC (rev 17809)
+++ trunk/bin/upgrading/update-db-general.pl	2010-11-29 07:47:35 UTC (rev 17810)
@@ -653,9 +653,8 @@
 CREATE TABLE user (
   userid int(10) unsigned NOT NULL auto_increment,
   user char(15) default NULL,
-  caps SMALLINT UNSIGNED NOT NULL DEFAULT 0,
-  email char(50) default NULL,
-  password char(30) default NULL,
+  caps BIGINT UNSIGNED NOT NULL DEFAULT 0,
+  packed_props BIGINT UNSIGNED NOT NULL DEFAULT 0,
   status char(1) NOT NULL default 'N',
   statusvis char(1) NOT NULL default 'V',
   statusvisdate datetime default NULL,
@@ -681,12 +680,21 @@
   lang char(2) NOT NULL default 'EN',
   PRIMARY KEY  (userid),
   UNIQUE KEY user (user),
-  KEY (email),
   KEY (status),
   KEY (statusvis)
 )  PACK_KEYS=1
 EOC
 
+if ( column_type('user', 'caps') =~ /smallint/i ) {
+    do_alter('user', qq{
+        ALTER TABLE user
+            DROP COLUMN email,
+            DROP COLUMN password,
+            MODIFY COLUMN caps BIGINT UNSIGNED NOT NULL DEFAULT 0,
+            ADD COLUMN packed_props BIGINT UNSIGNED NOT NULL DEFAULT 0
+    });
+}
+
 register_tablecreate("userbio", <<'EOC');
 CREATE TABLE userbio (
   userid int(10) unsigned NOT NULL default '0',

Modified: trunk/cgi-bin/LJ/MemCache.pm
===================================================================
--- trunk/cgi-bin/LJ/MemCache.pm	2010-11-29 04:54:45 UTC (rev 17809)
+++ trunk/cgi-bin/LJ/MemCache.pm	2010-11-29 07:47:35 UTC (rev 17810)
@@ -32,11 +32,11 @@
 
 %LJ::MEMCACHE_ARRAYFMT = (
                           'user' =>
-                          [qw[1 userid user caps clusterid dversion email password status statusvis statusvisdate
+                          [qw[2 userid user caps clusterid dversion status statusvis statusvisdate
                               name bdate themeid moodthemeid opt_forcemoodtheme allow_infoshow allow_contactshow
                               allow_getljnews opt_showtalklinks opt_whocanreply opt_gettalkemail opt_htmlemail
                               opt_mangleemail useoverrides defaultpicid has_bio txtmsg_status is_system
-                              journaltype lang oldenc]],
+                              journaltype lang oldenc packed_props]],
                           'fgrp' => [qw[1 userid groupnum groupname sortorder is_public]],
                           # version #101 because old userpic format in memcached was an arrayref of
                           # [width, height, ...] and widths could have been 1 before, although unlikely

Added: trunk/cgi-bin/LJ/User/PropStorage/DB.pm
===================================================================
--- trunk/cgi-bin/LJ/User/PropStorage/DB.pm	                        (rev 0)
+++ trunk/cgi-bin/LJ/User/PropStorage/DB.pm	2010-11-29 07:47:35 UTC (rev 17810)
@@ -0,0 +1,91 @@
+package LJ::User::PropStorage::DB;
+use strict;
+use warnings;
+
+use base qw( LJ::User::PropStorage );
+
+sub can_handle { 0 }
+sub use_memcache { 'lite' }
+
+sub fetch_props_db {
+    my ( $class, $u, $dbh, $table, $props ) = @_;
+
+    my $userid = int $u->userid;
+
+    LJ::load_props('user');
+
+    my @propids;
+    foreach my $k (@$props) {
+        my $propinfo = LJ::get_prop('user', $k);
+        my $propid   = $propinfo->{'id'};
+
+        push @propids, $propid;
+    }
+
+    my %propid_map = %{ $LJ::CACHE_PROPID{'user'} };
+
+    my %ret = map { $_ => undef } @$props;
+    my $sql;
+    if ( my $propids_in = join(',', @propids) ) {
+        $sql = qq{
+            SELECT upropid, value
+            FROM $table
+            WHERE userid=? AND upropid IN ($propids_in)
+        };
+    } else {
+        # well, it seems that they didn't pass us any props, so
+        # let's use a somewhat different SQL query to load everything
+        # from the given table
+        $sql = qq{
+            SELECT upropid, value
+            FROM $table
+            WHERE userid=?
+        };
+    }
+
+    my $res = $dbh->selectall_arrayref($sql, { 'Slice' => {} }, $userid);
+
+    foreach my $row (@$res) {
+        my $propname = $propid_map{ $row->{'upropid'} }->{'name'};
+        $ret{$propname} = $row->{'value'};
+    }
+
+    return \%ret;
+}
+
+sub store_props_db {
+    my ( $class, $u, $dbh, $table, $propmap ) = @_;
+
+    my $userid = int $u->userid;
+
+    my ( @values_sql, @subst, @propid_delete );
+    foreach my $k (keys %$propmap) {
+        my $propinfo = LJ::get_prop('user', $k);
+        my $propid   = $propinfo->{'id'};
+
+        unless ( defined $propmap->{$k} && $propmap->{$k} ) {
+            push @propid_delete, $propid;
+            next;
+        }
+
+        push @values_sql, '(?, ?, ?)';
+        push @subst, $userid, $propid, $propmap->{$k};
+    }
+
+    if ( my $values_sql = join(',', @values_sql) ) {
+        $dbh->do(qq{
+            REPLACE INTO $table
+            (userid, upropid, value)
+            VALUES $values_sql
+        }, undef, @subst);
+    }
+
+    if ( my $props_delete_in = join(',', @propid_delete) ) {
+        $dbh->do(qq{
+            DELETE FROM $table
+            WHERE userid=? AND upropid IN ($props_delete_in)
+        }, undef, $userid);
+    }
+}
+
+1;

Added: trunk/cgi-bin/LJ/User/PropStorage/Global.pm
===================================================================
--- trunk/cgi-bin/LJ/User/PropStorage/Global.pm	                        (rev 0)
+++ trunk/cgi-bin/LJ/User/PropStorage/Global.pm	2010-11-29 07:47:35 UTC (rev 17810)
@@ -0,0 +1,53 @@
+package LJ::User::PropStorage::Global;
+use strict;
+use warnings;
+
+use base qw( LJ::User::PropStorage::DB );
+
+sub memcache_key {
+    my ( $class, $u ) = @_;
+
+    return 'uprop2:global:' . $u->id;
+}
+
+sub can_handle {
+    my ( $class, $propname ) = @_;
+
+    my $propinfo = LJ::get_prop( 'user', $propname );
+    return 0 unless $propinfo;
+
+    return 0 if $propinfo->{'datatype'} =~ /^bit/;
+
+    return 0 if $propinfo->{'multihomed'};
+    return 0 if $propinfo->{'indexed'};
+
+    return 0 if $propinfo->{'cldversion'};
+    return 0 if $propinfo->{'datatype'} eq 'blobchar';
+
+    return 1;
+}
+
+sub get_props {
+    my ( $class, $u, $props, $opts ) = @_;
+
+    my $dbh = $opts->{'use_master'} ? LJ::get_db_writer()
+                                    : LJ::get_db_reader();
+
+    Carp::croak 'cannot get a database handle for global cluster'
+        unless $dbh;
+
+    return $class->fetch_props_db( $u, $dbh, 'userproplite', $props );
+}
+
+sub set_props {
+    my ( $class, $u, $propmap, $opts ) = @_;
+
+    my $dbh = LJ::get_db_writer();
+
+    Carp::croak 'cannot get a database handle for global cluster'
+        unless $dbh;
+
+    return $class->store_props_db( $u, $dbh, 'userproplite', $propmap );
+}
+
+1;

Added: trunk/cgi-bin/LJ/User/PropStorage/GlobalIndexed.pm
===================================================================
--- trunk/cgi-bin/LJ/User/PropStorage/GlobalIndexed.pm	                        (rev 0)
+++ trunk/cgi-bin/LJ/User/PropStorage/GlobalIndexed.pm	2010-11-29 07:47:35 UTC (rev 17810)
@@ -0,0 +1,49 @@
+package LJ::User::PropStorage::GlobalIndexed;
+use strict;
+use warnings;
+
+use base qw(LJ::User::PropStorage::DB);
+
+sub memcache_key {
+    my ( $class, $u ) = @_;
+
+    return 'uprop2:indexed:' . $u->id;
+}
+
+sub can_handle {
+    my ( $class, $propname ) = @_;
+
+    my $propinfo = LJ::get_prop( 'user', $propname );
+    return 0 unless $propinfo;
+
+    return 0 if $propinfo->{'datatype'} =~ /^bit/;
+
+    return 0 unless $propinfo->{'multihomed'} || $propinfo->{'indexed'};
+
+    return 1;
+}
+
+sub get_props {
+    my ( $class, $u, $props, $opts ) = @_;
+
+    my $dbh = $opts->{'use_master'} ? LJ::get_db_writer()
+                                    : LJ::get_db_reader();
+
+    Carp::croak 'cannot get a database handle for global cluster'
+        unless $dbh;
+
+    return $class->fetch_props_db( $u, $dbh, 'userprop', $props );
+}
+
+sub set_props {
+    my ( $class, $u, $propmap, $opts ) = @_;
+
+    my $dbh = LJ::get_db_writer();
+
+    Carp::croak 'cannot get a database handle for global cluster'
+        unless $dbh;
+
+    return $class->store_props_db( $u, $dbh, 'userprop', $propmap );
+}
+
+1;

Added: trunk/cgi-bin/LJ/User/PropStorage/Packed.pm
===================================================================
--- trunk/cgi-bin/LJ/User/PropStorage/Packed.pm	                        (rev 0)
+++ trunk/cgi-bin/LJ/User/PropStorage/Packed.pm	2010-11-29 07:47:35 UTC (rev 17810)
@@ -0,0 +1,65 @@
+package LJ::User::PropStorage::Packed;
+use strict;
+use warnings;
+
+use base qw(LJ::User::PropStorage);
+
+use Carp qw();
+
+sub use_memcache { return; }
+
+sub can_handle {
+    my ($class, $propname) = @_;
+
+    my $propinfo = LJ::get_prop( 'user', $propname );
+    return 0 unless $propinfo;
+
+    return 0 unless $propinfo->{'datatype'} =~ /^bit/;
+
+    return 1;
+}
+
+sub get_bit {
+    my ($class, $propname) = @_;
+
+    my $propinfo = LJ::get_prop( 'user', $propname );
+
+    Carp::croak "cannot get userprop info for $propname"
+        unless $propinfo;
+
+    if ($propinfo->{'datatype'} =~ /^bit(\d+)$/) {
+        return $1;
+    }
+
+    Carp::croak "$propname is not a packed userprop";
+}
+
+sub get_props {
+    my ($class, $u, $props) = @_;
+
+    my %ret;
+
+    foreach my $propname (@$props) {
+        my $bit = $class->get_bit($propname);
+        $ret{$propname} = ( $u->packed_props & (1 << $bit) ) ? 1 : 0;
+    }
+}
+
+sub set_props {
+    my ($class, $u, $propmap) = @_;
+
+    my $newprops = $u->packed_props;
+
+    foreach my $propname ( keys %$propmap ) {
+        my $bit = $class->get_bit($propname);
+        if ( $propmap->{$propname} ) {
+            $newprops |= (1 << $bit);
+        } else {
+            $newprops &= ~(1 << $bit);
+        }
+    }
+
+    $u->set_packed_props($newprops);
+}
+
+1;

Added: trunk/cgi-bin/LJ/User/PropStorage/UserClusterBlob.pm
===================================================================
--- trunk/cgi-bin/LJ/User/PropStorage/UserClusterBlob.pm	                        (rev 0)
+++ trunk/cgi-bin/LJ/User/PropStorage/UserClusterBlob.pm	2010-11-29 07:47:35 UTC (rev 17810)
@@ -0,0 +1,57 @@
+package LJ::User::PropStorage::UserClusterBlob;
+use strict;
+use warnings;
+
+use base qw(LJ::User::PropStorage::DB);
+
+sub use_memcache { 'blob' }
+
+sub memcache_key {
+    my ( $class, $u, $propname ) = @_;
+
+    my $propinfo = LJ::get_prop( 'user', $propname );
+
+    return 'uprop2:blob:' . $u->id . ':' . $propinfo->{'id'};
+}
+
+sub can_handle {
+    my ( $class, $propname ) = @_;
+
+    my $propinfo = LJ::get_prop( 'user', $propname );
+    return 0 unless $propinfo;
+
+    return 0 if $propinfo->{'datatype'} =~ /^bit/;
+
+    return 0 if $propinfo->{'multihomed'};
+    return 0 if $propinfo->{'indexed'};
+
+    return 0 unless $propinfo->{'cldversion'};
+    return 0 unless $propinfo->{'datatype'} eq 'blobchar';
+
+    return 1;
+}
+
+sub get_props {
+    my ( $class, $u, $props, $opts ) = @_;
+
+    my $dbh = $opts->{'use_master'} ? LJ::get_cluster_master($u)
+                                    : LJ::get_cluster_reader($u);
+
+    Carp::croak 'cannot get a database handle for user #' . $u->userid
+        unless $dbh;
+
+    return $class->fetch_props_db( $u, $dbh, 'userpropblob', $props );
+}
+
+sub set_props {
+    my ( $class, $u, $propmap, $opts ) = @_;
+
+    my $dbh = LJ::get_cluster_master($u);
+
+    Carp::croak 'cannot get a database handle for user #' . $u->userid
+        unless $dbh;
+
+    return $class->store_props_db( $u, $dbh, 'userpropblob', $propmap );
+}
+
+1;

Added: trunk/cgi-bin/LJ/User/PropStorage/UserClusterLite.pm
===================================================================
--- trunk/cgi-bin/LJ/User/PropStorage/UserClusterLite.pm	                        (rev 0)
+++ trunk/cgi-bin/LJ/User/PropStorage/UserClusterLite.pm	2010-11-29 07:47:35 UTC (rev 17810)
@@ -0,0 +1,53 @@
+package LJ::User::PropStorage::UserClusterLite;
+use strict;
+use warnings;
+
+use base qw(LJ::User::PropStorage::DB);
+
+sub memcache_key {
+    my ( $class, $u ) = @_;
+
+    return 'uprop2:lite:' . $u->id;
+}
+
+sub can_handle {
+    my ( $class, $propname ) = @_;
+
+    my $propinfo = LJ::get_prop( 'user', $propname );
+    return 0 unless $propinfo;
+
+    return 0 if $propinfo->{'datatype'} =~ /^bit/;
+
+    return 0 if $propinfo->{'multihomed'};
+    return 0 if $propinfo->{'indexed'};
+
+    return 0 unless $propinfo->{'cldversion'};
+    return 0 if $propinfo->{'datatype'} eq 'blobchar';
+
+    return 1;
+}
+
+sub get_props {
+    my ( $class, $u, $props, $opts ) = @_;
+
+    my $dbh = $opts->{'use_master'} ? LJ::get_cluster_master($u)
+                                    : LJ::get_cluster_reader($u);
+
+    Carp::croak 'cannot get a database handle for user #' . $u->userid
+        unless $dbh;
+
+    return $class->fetch_props_db( $u, $dbh, 'userproplite2', $props );
+}
+
+sub set_props {
+    my ( $class, $u, $propmap, $opts ) = @_;
+
+    my $dbh = LJ::get_cluster_master($u);
+
+    Carp::croak 'cannot get a database handle for user #' . $u->userid
+        unless $dbh;
+
+    return $class->store_props_db( $u, $dbh, 'userproplite2', $propmap );
+}
+
+1;

Added: trunk/cgi-bin/LJ/User/PropStorage.pm
===================================================================
--- trunk/cgi-bin/LJ/User/PropStorage.pm	                        (rev 0)
+++ trunk/cgi-bin/LJ/User/PropStorage.pm	2010-11-29 07:47:35 UTC (rev 17810)
@@ -0,0 +1,142 @@
+package LJ::User::PropStorage;
+use strict;
+use warnings;
+
+use LJ::ModuleLoader;
+use Carp qw();
+
+# initialization code. do not touch this.
+my @SUBCLASSES = LJ::ModuleLoader->module_subclasses(__PACKAGE__);
+use LJ::User::PropStorage::DB;
+foreach my $class (@SUBCLASSES) {
+    eval "use $class";
+    Carp::confess "Error loading module '$class': $@" if $@;
+}
+
+sub get_handler {
+    my ($class, $propname) = @_;
+
+    foreach my $class (@SUBCLASSES) {
+        next unless $class->can_handle($propname);
+        return $class;
+    }
+
+    Carp::croak 'cannot get a handler for prop=' . $propname;
+}
+
+sub get_handler_multi {
+    my ($class, $props) = @_;
+
+    my %ret;
+
+    foreach my $prop (@$props) {
+        my $handler = $class->get_handler($prop);
+
+        $ret{$handler} = [] unless exists $ret{$handler};
+        push @{ $ret{$handler} }, $prop;
+    }
+
+    return \%ret;
+}
+
+sub get_props {
+    my ( $class, $u, $props, $opts ) = @_;
+
+    Carp::croak $class . 'does not have a "get" method defined';
+}
+
+sub set_props {
+    my ( $class, $u, $propmap, $opts ) = @_;
+
+    Carp::croak $class . 'does not have a "set" method defined';
+}
+
+sub can_handle {
+    my ( $class, $propname ) = @_;
+
+    Carp::croak $class . 'does not have a "can_handle" method defined';
+}
+
+sub use_memcache {
+    my ($class) = @_;
+
+    Carp::croak $class . 'does not have a "use_memcache" method defined';
+}
+
+sub memcache_key {
+    my ( $class, $u, $propname ) = @_;
+
+    Carp::croak $class . 'does not have a "memcache_key" method defined';
+}
+
+sub fetch_props_memcache {
+    my ( $class, $u, $props ) = @_;
+
+    my $userid = int $u->userid;
+
+    my ( %propid_map, @memkeys );
+    foreach my $k (@$props) {
+        my $propinfo = LJ::get_prop('user', $k);
+        my $propid   = $propinfo->{'id'};
+        my $memkey   = $class->memcache_key($u, $k);
+
+        $propid_map{$propid} = $k;
+        push @memkeys, [ $userid, $memkey ];
+    }
+
+    my $from_memcache = LJ::MemCache::get_multi(@memkeys);
+
+    my %ret;
+    foreach my $k (@$props) {
+        my $memkey = $class->memcache_key($u, $k);
+
+        next unless exists $from_memcache->{$memkey};
+
+        $ret{ $propid_map{$k} } = $from_memcache->{$memkey};
+    }
+
+    return \%ret;
+}
+
+sub store_props_memcache {
+    my ( $class, $u, $propmap ) = @_;
+
+    my $userid = int $u->userid;
+    my $expire = time + 3600 * 24;
+
+    foreach my $k (keys %$propmap) {
+        my $memkey = $class->memcache_key($u, $k);
+
+        LJ::MemCache::set( [ $userid, $memkey ],
+                           $propmap->{$k} || '',
+                           $expire );
+    }
+}
+
+sub pack_for_memcache {
+    my ( $class, $propmap ) = @_;
+
+    my %ret;
+    foreach my $propname (keys %$propmap) {
+        my $propinfo = LJ::get_prop( 'user', $propname );
+        my $propid   = $propinfo->{'id'};
+
+        $ret{$propid} = $propmap->{$propname};
+    }
+
+    return \%ret;
+}
+
+sub unpack_from_memcache {
+    my ( $class, $packed ) = @_;
+
+    my %ret;
+    foreach my $propid (keys %$packed) {
+        my $propname = $LJ::CACHE_PROPID{'user'}->{$propid}->{'name'};
+        $ret{$propname} = $packed->{$propid};
+    }
+
+    return \%ret;
+}
+
+1;

Modified: trunk/cgi-bin/LJ/User.pm
===================================================================
--- trunk/cgi-bin/LJ/User.pm	2010-11-29 04:54:45 UTC (rev 17809)
+++ trunk/cgi-bin/LJ/User.pm	2010-11-29 07:47:35 UTC (rev 17810)
@@ -26,6 +26,7 @@
 use LJ::JSON;
 use HTTP::Date qw(str2time);
 use LJ::TimeUtil;
+use LJ::User::PropStorage;
 
 use Class::Autouse qw(
                       URI
@@ -1879,11 +1880,73 @@
 # sets prop, and also updates $u's cached version
 sub set_prop {
     my ($u, $prop, $value) = @_;
-    return 0 unless LJ::set_userprop($u, $prop, $value);  # FIXME: use exceptions
-    $u->{$prop} = $value;
 
-    LJ::run_hooks("props_changed", $u, {$prop => $value});
+    my $propmap = ref $prop ? $prop : { $prop => $value };
 
+    # filter out props that do not change
+    foreach my $propname (keys %$propmap) {
+        # it's not loaded, so let's not check it
+        next unless exists $u->{$propname};
+
+        if ( (!$propmap->{$propname} && !$u->{$propname})
+          || $propmap->{$propname} eq $u->{$propname} )
+        {
+            delete $propmap->{$propname};
+        }
+    }
+
+    my @props_affected = keys %$propmap;
+    my $groups = LJ::User::PropStorage->get_handler_multi(\@props_affected);
+    my $memcache_available = @LJ::MEMCACHE_SERVERS;
+
+    my $memc_expire = time + 3600 * 24;
+
+    foreach my $handler (keys %$groups) {
+        my $propnames_handled = $groups->{$handler};
+        my %propmap_handled   = map { $_ => $propmap->{$_} }
+                                @$propnames_handled;
+
+        # first, actually save stuff to the database;
+        # then, delete it from memcache, depending on the memcache
+        # policy of the handler
+        $handler->set_props( $u, \%propmap_handled );
+
+        # if there is no memcache, or if the handler doesn't wish to use
+        # memcache, we don't need to deal with it, yay
+        if ( !$memcache_available || !defined $handler->use_memcache )
+        {
+            next;
+        }
+
+        # now let's find out what we're going to do with memcache
+        my $memcache_policy = $handler->use_memcache;
+
+        if ( $memcache_policy eq 'lite' ) {
+            # the handler loads everything from the corresponding
+            # table and uses only one memcache key to cache that
+
+            my $memkey = $handler->memcache_key($u);
+            LJ::MemCache::delete([ $u->userid, $memkey ]);
+        } elsif ( $memcache_policy eq 'blob' ) {
+            # the handler uses one memcache key for each prop,
+            # so let's delete them all
+
+            foreach my $propname (@$propnames_handled) {
+                my $memkey = $handler->memcache_key( $u, $propname );
+                LJ::MemCache::delete([ $u->userid, $memkey ]);
+            }
+        }
+    }
+
+    # now, actually reflect that we've changed the props in the
+    # user object
+    foreach my $propname (keys %$propmap) {
+        $u->{$propname} = $propmap->{$propname};
+    }
+
+    # and run the hooks, too
+    LJ::run_hooks( 'props_changed', $u, $propmap );
+
     return $value;
 }
 
@@ -5631,7 +5694,7 @@
     my ($u, $url) = @_;
     my $uid = $u->id;
 
-    LJ::set_userprop($u, 'custom_usericon', $url);
+    $u->set_prop( 'custom_usericon' => $url );
 }
 
 sub _subscriptions_count {
@@ -5653,6 +5716,18 @@
     return $count;
 }
 
+sub packed_props {
+    my ($u) = @_;
+    return $u->{'packed_props'};
+}
+
+sub set_packed_props {
+    my ($u, $newprops) = @_;
+
+    LJ::update_user($u, { 'packed_props' => $newprops });
+    $u->{'packed_props'} = 1;
+}
+
 sub reset_cache {
     my $u = shift;
 
@@ -5912,8 +5987,7 @@
 # des-opts: hashref of opts.  set key 'cache' to use memcache.
 # des-propname: the name of a property from the [dbtable[userproplist]] table.
 # </LJFUNC>
-sub load_user_props
-{
+sub load_user_props {
     &nodb;
 
     my $u = shift;
@@ -5923,155 +5997,91 @@
     my $opts = ref $_[0] ? shift : {};
     my (@props) = @_;
 
-    my ($sql, $sth);
-    LJ::load_props("user");
+    my $extend_user_object = sub {
+        my ($propmap) = @_;
 
-    ## user reference
-    my $uid = $u->{'userid'}+0;
-    $uid = LJ::get_userid($u->{'user'}) unless $uid;
+        foreach my $propname ( keys %$propmap ) {
+            $u->{$propname} = $propmap->{$propname};
+        }
+    };
 
-    my $mem = {};
-    my $use_master = 0;
-    my $used_slave = 0;  # set later if we ended up using a slave
+    my $groups = LJ::User::PropStorage->get_handler_multi(\@props);
+    my $memcache_available = @LJ::MEMCACHE_SERVERS;
+    my $use_master = $memcache_available || $opts->{'use_master'};
 
-    if (@LJ::MEMCACHE_SERVERS) {
-        my @keys;
-        foreach (@props) {
-            next if exists $u->{$_};
-            my $p = LJ::get_prop("user", $_);
-            die "Invalid userprop $_ passed to LJ::load_user_props." unless $p;
-            push @keys, [$uid,"uprop:$uid:$p->{'id'}"];
-        }
-        $mem = LJ::MemCache::get_multi(@keys) || {};
-        $use_master = 1;
-    }
+    my $memc_expire = time() + 3600 * 24;
 
-    $use_master = 1 if $opts->{'use_master'};
+    foreach my $handler (keys %$groups) {
+        # if there is no memcache, or if the handler doesn't wish to use
+        # memcache, hit the storage directly, update the user object,
+        # and get straight to the next handler
+        if ( !$memcache_available || !defined $handler->use_memcache )
+        {
+            my $propmap
+                = $handler->get_props( $u, $groups->{$handler},
+                                       { 'use_master' => $use_master } );
 
-    my @needwrite;  # [propid, propname] entries we need to save to memcache later
+            $extend_user_object->($propmap);
 
-    my %loadfrom;
-    my %multihomed; # ( $propid => 0/1 ) # 0 if we haven't loaded it, 1 if we have
-    unless (@props) {
-        # case 1: load all props for a given user.
-        # multihomed props are stored on userprop and userproplite2, but since they
-        # should always be in sync, it doesn't matter which gets loaded first, the
-        # net results should be the same.  see doc/server/lj.int.multihomed_userprops.html
-        # for more information.
-        $loadfrom{'userprop'} = 1;
-        $loadfrom{'userproplite'} = 1;
-        $loadfrom{'userproplite2'} = 1;
-        $loadfrom{'userpropblob'} = 1;
-    } else {
-        # case 2: load only certain things
-        foreach (@props) {
-            next if exists $u->{$_};
-            my $p = LJ::get_prop("user", $_);
-            die "Invalid userprop $_ passed to LJ::load_user_props." unless $p;
-            if (defined $mem->{"uprop:$uid:$p->{'id'}"}) {
-                $u->{$_} = $mem->{"uprop:$uid:$p->{'id'}"};
-                next;
-            }
-            push @needwrite, [ $p->{'id'}, $_ ];
-            my $source = $p->{'indexed'} ? "userprop" : "userproplite";
-            if ($p->{datatype} eq 'blobchar') {
-                $source = "userpropblob"; # clustered blob
-            }
-            elsif ($p->{'cldversion'} && $u->{'dversion'} >= $p->{'cldversion'}) {
-                $source = "userproplite2";  # clustered
-            }
-            elsif ($p->{multihomed}) {
-                $multihomed{$p->{id}} = 0;
-                $source = "userproplite2";
-            }
-            push @{$loadfrom{$source}}, $p->{'id'};
+            next;
         }
-    }
 
-    foreach my $table (qw{userproplite userproplite2 userpropblob userprop}) {
-        next unless exists $loadfrom{$table};
-        my $db;
-        if ($use_master) {
-            $db = ($table =~ m{userprop(lite2|blob)}) ?
-                LJ::get_cluster_master($u) :
-                LJ::get_db_writer();
-        }
-        unless ($db) {
-            $db = ($table =~ m{userprop(lite2|blob)}) ?
-                LJ::get_cluster_reader($u) :
-                LJ::get_db_reader();
-            $used_slave = 1;
-        }
-        $sql = "SELECT upropid, value FROM $table WHERE userid=$uid";
-        if (ref $loadfrom{$table}) {
-            $sql .= " AND upropid IN (" . join(",", @{$loadfrom{$table}}) . ")";
-        }
-        $sth = $db->prepare($sql);
-        $sth->execute;
-        while (my ($id, $v) = $sth->fetchrow_array) {
-            delete $multihomed{$id} if $table eq 'userproplite2';
-            $u->{$LJ::CACHE_PROPID{'user'}->{$id}->{'name'}} = $v;
-        }
+        # now let's find out what we're going to do with memcache
+        my $memcache_policy = $handler->use_memcache;
 
-        # push back multihomed if necessary
-        if ($table eq 'userproplite2') {
-            push @{$loadfrom{userprop}}, $_ foreach keys %multihomed;
-        }
-    }
+        if ( $memcache_policy eq 'lite' ) {
+            # the handler loads everything from the corresponding
+            # table and uses only one memcache key to cache that
 
-    # see if we failed to get anything above and need to hit the master.
-    # this usually happens the first time a multihomed prop is hit.  this
-    # code will propagate that prop down to the cluster.
-    if (%multihomed) {
+            my $memkey = $handler->memcache_key($u);
+            if ( my $packed = LJ::MemCache::get([ $u->userid, $memkey ]) ) {
+                my $propmap
+                    = LJ::User::PropStorage->unpack_from_memcache($packed);
 
-        # verify that we got the database handle before we try propagating data
-        if ($u->writer) {
-            my @values;
-            foreach my $id (keys %multihomed) {
-                my $pname = $LJ::CACHE_PROPID{user}{$id}{name};
-                if (defined $u->{$pname} && $u->{$pname}) {
-                    push @values, "($uid, $id, " . $u->quote($u->{$pname}) . ")";
-                } else {
-                    push @values, "($uid, $id, '')";
-                }
+                $extend_user_object->($propmap);
+
+                # we've loaded everything handled by this handler,
+                # let's get to the next one
+                next;
             }
-            $u->do("REPLACE INTO userproplite2 VALUES " . join ',', @values);
-        }
-    }
 
-    # Add defaults to user object.
+            # actually load everything from DB, cache it, and update
+            # the user object
+            my $propmap
+                = $handler->get_props( $u, [],
+                                       { 'use_master' => $use_master } );
 
-    # defaults for S1 style IDs in config file are magic: really
-    # uniq strings representing style IDs, so on first use, we need
-    # to map them
-    unless ($LJ::CACHED_S1IDMAP) {
+            my $packed = LJ::User::PropStorage->pack_for_memcache($propmap);
+            LJ::MemCache::set( [$u->userid, $memkey],
+                               $packed, $memc_expire );
 
-        my $pubsty = LJ::S1::get_public_styles();
-        foreach (values %$pubsty) {
-            my $k = "s1_$_->{'type'}_style";
-            next unless $LJ::USERPROP_DEF{$k} eq "$_->{'type'}/$_->{'styledes'}";
+            $extend_user_object->($propmap);
+        } elsif ( $memcache_policy eq 'blob' ) {
+            # the handler uses one memcache key for each prop
+            # hit memcache for them first, then hit db and update
+            # memcache
 
-            $LJ::USERPROP_DEF{$k} = $_->{'styleid'};
-        }
+            my $handled_props = $groups->{$handler};
 
-        $LJ::CACHED_S1IDMAP = 1;
-    }
+            my $propmap_memc
+                = $handler->fetch_props_memcache( $u, $handled_props );
 
-    # If this was called with no @props, then the function tried
-    # to load all metadata.  but we don't know what's missing, so
-    # try to apply all defaults.
-    unless (@props) { @props = keys %LJ::USERPROP_DEF; }
+            $extend_user_object->($propmap_memc);
 
-    foreach my $prop (@props) {
-        next if (defined $u->{$prop});
-        $u->{$prop} = $LJ::USERPROP_DEF{$prop};
-    }
+            my @load_from_db = grep { !exists $propmap_memc->{$_} }
+                               @$handled_props;
 
-    unless ($used_slave) {
-        my $expire = time() + 3600*24;
-        foreach my $wr (@needwrite) {
-            my ($id, $name) = ($wr->[0], $wr->[1]);
-            LJ::MemCache::set([$uid,"uprop:$uid:$id"], $u->{$name} || "", $expire);
+            # if we can avoid hitting the db, avoid it
+            next unless @load_from_db;
+
+            my $propmap_db
+                = $handler->get_props( $u, \@load_from_db,
+                                       { 'use_master' => $use_master } );
+
+            $extend_user_object->($propmap_db);
+
+            # now, update memcache
+            $handler->store_props_memcache( $u, $propmap_db );
         }
     }
 }
@@ -6850,117 +6860,6 @@
 }
 
 # <LJFUNC>
-# name: LJ::set_userprop
-# des: Sets/deletes a userprop by name for a user.
-# info: This adds or deletes from the
-#       [dbtable[userprop]]/[dbtable[userproplite]] tables.  One
-#       crappy thing about this interface is that it doesn't allow
-#       a batch of userprops to be updated at once, which is the
-#       common thing to do.
-# args: dbarg?, uuserid, propname, value, memonly?
-# des-uuserid: The userid of the user or a user hashref.
-# des-propname: The name of the property.  Or a hashref of propname keys and corresponding values.
-# des-value: The value to set to the property.  If undefined or the
-#            empty string, then property is deleted.
-# des-memonly: if true, only writes to memcache, and not to database.
-# </LJFUNC>
-sub set_userprop
-{
-    &nodb;
-    my ($u, $propname, $value, $memonly) = @_;
-    $u = ref $u ? $u : LJ::load_userid($u);
-    my $userid = $u->{'userid'}+0;
-
-    my $hash = ref $propname eq "HASH" ? $propname : { $propname => $value };
-
-    my %action;  # $table -> {"replace"|"delete"} -> [ "($userid, $propid, $qvalue)" | propid ]
-    my %multihomed;  # { $propid => $value }
-
-    foreach $propname (keys %$hash) {
-        LJ::run_hook("setprop", prop => $propname,
-                     u => $u, value => $value);
-
-        my $p = LJ::get_prop("user", $propname) or
-            die "Invalid userprop $propname passed to LJ::set_userprop.";
-        if ($p->{multihomed}) {
-            # collect into array for later handling
-            $multihomed{$p->{id}} = $hash->{$propname};
-            next;
-        }
-        my $table = $p->{'indexed'} ? "userprop" : "userproplite";
-        if ($p->{datatype} eq 'blobchar') {
-            $table = 'userpropblob';
-        }
-        elsif ($p->{'cldversion'} && $u->{'dversion'} >= $p->{'cldversion'}) {
-            $table = "userproplite2";
-        }
-        unless ($memonly) {
-            my $db = $action{$table}->{'db'} ||= (
-                $table !~ m{userprop(lite2|blob)}
-                    ? LJ::get_db_writer()
-                    : $u->writer );
-            return 0 unless $db;
-        }
-        $value = $hash->{$propname};
-        if (defined $value && $value) {
-            push @{$action{$table}->{"replace"}}, [ $p->{'id'}, $value ];
-        } else {
-            push @{$action{$table}->{"delete"}}, $p->{'id'};
-        }
-    }
-
-    my $expire = time() + 3600*24;
-    foreach my $table (keys %action) {
-        my $db = $action{$table}->{'db'};
-        if (my $list = $action{$table}->{"replace"}) {
-            if ($db) {
-                my $vals = join(',', map { "($userid,$_->[0]," . $db->quote($_->[1]) . ")" } @$list);
-                $db->do("REPLACE INTO $table (userid, upropid, value) VALUES $vals");
-            }
-            LJ::MemCache::set([$userid,"uprop:$userid:$_->[0]"], $_->[1], $expire) foreach (@$list);
-        }
-        if (my $list = $action{$table}->{"delete"}) {
-            if ($db) {
-                my $in = join(',', @$list);
-                $db->do("DELETE FROM $table WHERE userid=$userid AND upropid IN ($in)");
-            }
-            LJ::MemCache::set([$userid,"uprop:$userid:$_"], "", $expire) foreach (@$list);
-        }
-    }
-
-    # if we had any multihomed props, set them here
-    if (%multihomed) {
-        my $dbh = LJ::get_db_writer();
-        return 0 unless $dbh && $u->writer;
-        while (my ($propid, $pvalue) = each %multihomed) {
-            if (defined $pvalue && $pvalue) {
-                # replace data into master
-                $dbh->do("REPLACE INTO userprop VALUES (?, ?, ?)",
-                         undef, $userid, $propid, $pvalue);
-            } else {
-                # delete data from master, but keep in cluster
-                $dbh->do("DELETE FROM userprop WHERE userid = ? AND upropid = ?",
-                         undef, $userid, $propid);
-            }
-
-            # fail out?
-            return 0 if $dbh->err;
-
-            # put data in cluster
-            $pvalue ||= '';
-            $u->do("REPLACE INTO userproplite2 VALUES (?, ?, ?)",
-                   undef, $userid, $propid, $pvalue);
-            return 0 if $u->err;
-
-            # set memcache
-            LJ::MemCache::set([$userid,"uprop:$userid:$propid"], $pvalue, $expire);
-        }
-    }
-
-    return 1;
-}
-
-# <LJFUNC>
 # name: LJ::get_shared_journals
 # des: Gets an array of shared journals a user has access to.
 # returns: An array of shared journals.

Modified: trunk/cgi-bin/ljdb.pl
===================================================================
--- trunk/cgi-bin/ljdb.pl	2010-11-29 04:54:45 UTC (rev 17809)
+++ trunk/cgi-bin/ljdb.pl	2010-11-29 07:47:35 UTC (rev 17810)
@@ -403,6 +403,8 @@
 sub get_dbirole_dbh {
     my $dbh = $LJ::DBIRole->get_dbh( @_ ) or return undef;
 
+    $dbh->{'RaiseError'} = 1 if $LJ::IS_DEV_SERVER;
+
     if ( $LJ::DB_LOG_HOST && $LJ::HAVE_DBI_PROFILE ) {
         $LJ::DB_REPORT_HANDLES{ $dbh->{Name} } = $dbh;
 

Tags: andy, livejournal, pl, pm
Subscribe
  • Post a new comment

    Error

    Anonymous comments are disabled in this journal

    default userpic

    Your reply will be screened

    Your IP address will be recorded 

  • 0 comments