Committer: ailyin
LJSUP-7462 (refactor userprops)U trunk/bin/upgrading/update-db-general.pl U trunk/cgi-bin/LJ/MemCache.pm A trunk/cgi-bin/LJ/User/PropStorage/ A trunk/cgi-bin/LJ/User/PropStorage/DB.pm A trunk/cgi-bin/LJ/User/PropStorage/Global.pm A trunk/cgi-bin/LJ/User/PropStorage/GlobalIndexed.pm A trunk/cgi-bin/LJ/User/PropStorage/Packed.pm A trunk/cgi-bin/LJ/User/PropStorage/UserClusterBlob.pm A trunk/cgi-bin/LJ/User/PropStorage/UserClusterLite.pm A trunk/cgi-bin/LJ/User/PropStorage.pm U trunk/cgi-bin/LJ/User.pm U trunk/cgi-bin/ljdb.pl
Modified: trunk/bin/upgrading/update-db-general.pl =================================================================== --- trunk/bin/upgrading/update-db-general.pl 2010-11-29 04:54:45 UTC (rev 17809) +++ trunk/bin/upgrading/update-db-general.pl 2010-11-29 07:47:35 UTC (rev 17810) @@ -653,9 +653,8 @@ CREATE TABLE user ( userid int(10) unsigned NOT NULL auto_increment, user char(15) default NULL, - caps SMALLINT UNSIGNED NOT NULL DEFAULT 0, - email char(50) default NULL, - password char(30) default NULL, + caps BIGINT UNSIGNED NOT NULL DEFAULT 0, + packed_props BIGINT UNSIGNED NOT NULL DEFAULT 0, status char(1) NOT NULL default 'N', statusvis char(1) NOT NULL default 'V', statusvisdate datetime default NULL, @@ -681,12 +680,21 @@ lang char(2) NOT NULL default 'EN', PRIMARY KEY (userid), UNIQUE KEY user (user), - KEY (email), KEY (status), KEY (statusvis) ) PACK_KEYS=1 EOC +if ( column_type('user', 'caps') =~ /smallint/i ) { + do_alter('user', qq{ + ALTER TABLE user + DROP COLUMN email, + DROP COLUMN password, + MODIFY COLUMN caps BIGINT UNSIGNED NOT NULL DEFAULT 0, + ADD COLUMN packed_props BIGINT UNSIGNED NOT NULL DEFAULT 0 + }); +} + register_tablecreate("userbio", <<'EOC'); CREATE TABLE userbio ( userid int(10) unsigned NOT NULL default '0', Modified: trunk/cgi-bin/LJ/MemCache.pm =================================================================== --- trunk/cgi-bin/LJ/MemCache.pm 2010-11-29 04:54:45 UTC (rev 17809) +++ trunk/cgi-bin/LJ/MemCache.pm 2010-11-29 07:47:35 UTC (rev 17810) @@ -32,11 +32,11 @@ %LJ::MEMCACHE_ARRAYFMT = ( 'user' => - [qw[1 userid user caps clusterid dversion email password status statusvis statusvisdate + [qw[2 userid user caps clusterid dversion status statusvis statusvisdate name bdate themeid moodthemeid opt_forcemoodtheme allow_infoshow allow_contactshow allow_getljnews opt_showtalklinks opt_whocanreply opt_gettalkemail opt_htmlemail opt_mangleemail useoverrides defaultpicid has_bio txtmsg_status is_system - journaltype lang oldenc]], + journaltype lang oldenc packed_props]], 'fgrp' => [qw[1 userid groupnum groupname sortorder is_public]], # version #101 because old userpic format in memcached was an arrayref of # [width, height, ...] and widths could have been 1 before, although unlikely Added: trunk/cgi-bin/LJ/User/PropStorage/DB.pm =================================================================== --- trunk/cgi-bin/LJ/User/PropStorage/DB.pm (rev 0) +++ trunk/cgi-bin/LJ/User/PropStorage/DB.pm 2010-11-29 07:47:35 UTC (rev 17810) @@ -0,0 +1,91 @@ +package LJ::User::PropStorage::DB; +use strict; +use warnings; + +use base qw( LJ::User::PropStorage ); + +sub can_handle { 0 } +sub use_memcache { 'lite' } + +sub fetch_props_db { + my ( $class, $u, $dbh, $table, $props ) = @_; + + my $userid = int $u->userid; + + LJ::load_props('user'); + + my @propids; + foreach my $k (@$props) { + my $propinfo = LJ::get_prop('user', $k); + my $propid = $propinfo->{'id'}; + + push @propids, $propid; + } + + my %propid_map = %{ $LJ::CACHE_PROPID{'user'} }; + + my %ret = map { $_ => undef } @$props; + my $sql; + if ( my $propids_in = join(',', @propids) ) { + $sql = qq{ + SELECT upropid, value + FROM $table + WHERE userid=? AND upropid IN ($propids_in) + }; + } else { + # well, it seems that they didn't pass us any props, so + # let's use a somewhat different SQL query to load everything + # from the given table + $sql = qq{ + SELECT upropid, value + FROM $table + WHERE userid=? + }; + } + + my $res = $dbh->selectall_arrayref($sql, { 'Slice' => {} }, $userid); + + foreach my $row (@$res) { + my $propname = $propid_map{ $row->{'upropid'} }->{'name'}; + $ret{$propname} = $row->{'value'}; + } + + return \%ret; +} + +sub store_props_db { + my ( $class, $u, $dbh, $table, $propmap ) = @_; + + my $userid = int $u->userid; + + my ( @values_sql, @subst, @propid_delete ); + foreach my $k (keys %$propmap) { + my $propinfo = LJ::get_prop('user', $k); + my $propid = $propinfo->{'id'}; + + unless ( defined $propmap->{$k} && $propmap->{$k} ) { + push @propid_delete, $propid; + next; + } + + push @values_sql, '(?, ?, ?)'; + push @subst, $userid, $propid, $propmap->{$k}; + } + + if ( my $values_sql = join(',', @values_sql) ) { + $dbh->do(qq{ + REPLACE INTO $table + (userid, upropid, value) + VALUES $values_sql + }, undef, @subst); + } + + if ( my $props_delete_in = join(',', @propid_delete) ) { + $dbh->do(qq{ + DELETE FROM $table + WHERE userid=? AND upropid IN ($props_delete_in) + }, undef, $userid); + } +} + +1; Added: trunk/cgi-bin/LJ/User/PropStorage/Global.pm =================================================================== --- trunk/cgi-bin/LJ/User/PropStorage/Global.pm (rev 0) +++ trunk/cgi-bin/LJ/User/PropStorage/Global.pm 2010-11-29 07:47:35 UTC (rev 17810) @@ -0,0 +1,53 @@ +package LJ::User::PropStorage::Global; +use strict; +use warnings; + +use base qw( LJ::User::PropStorage::DB ); + +sub memcache_key { + my ( $class, $u ) = @_; + + return 'uprop2:global:' . $u->id; +} + +sub can_handle { + my ( $class, $propname ) = @_; + + my $propinfo = LJ::get_prop( 'user', $propname ); + return 0 unless $propinfo; + + return 0 if $propinfo->{'datatype'} =~ /^bit/; + + return 0 if $propinfo->{'multihomed'}; + return 0 if $propinfo->{'indexed'}; + + return 0 if $propinfo->{'cldversion'}; + return 0 if $propinfo->{'datatype'} eq 'blobchar'; + + return 1; +} + +sub get_props { + my ( $class, $u, $props, $opts ) = @_; + + my $dbh = $opts->{'use_master'} ? LJ::get_db_writer() + : LJ::get_db_reader(); + + Carp::croak 'cannot get a database handle for global cluster' + unless $dbh; + + return $class->fetch_props_db( $u, $dbh, 'userproplite', $props ); +} + +sub set_props { + my ( $class, $u, $propmap, $opts ) = @_; + + my $dbh = LJ::get_db_writer(); + + Carp::croak 'cannot get a database handle for global cluster' + unless $dbh; + + return $class->store_props_db( $u, $dbh, 'userproplite', $propmap ); +} + +1; Added: trunk/cgi-bin/LJ/User/PropStorage/GlobalIndexed.pm =================================================================== --- trunk/cgi-bin/LJ/User/PropStorage/GlobalIndexed.pm (rev 0) +++ trunk/cgi-bin/LJ/User/PropStorage/GlobalIndexed.pm 2010-11-29 07:47:35 UTC (rev 17810) @@ -0,0 +1,49 @@ +package LJ::User::PropStorage::GlobalIndexed; +use strict; +use warnings; + +use base qw(LJ::User::PropStorage::DB); + +sub memcache_key { + my ( $class, $u ) = @_; + + return 'uprop2:indexed:' . $u->id; +} + +sub can_handle { + my ( $class, $propname ) = @_; + + my $propinfo = LJ::get_prop( 'user', $propname ); + return 0 unless $propinfo; + + return 0 if $propinfo->{'datatype'} =~ /^bit/; + + return 0 unless $propinfo->{'multihomed'} || $propinfo->{'indexed'}; + + return 1; +} + +sub get_props { + my ( $class, $u, $props, $opts ) = @_; + + my $dbh = $opts->{'use_master'} ? LJ::get_db_writer() + : LJ::get_db_reader(); + + Carp::croak 'cannot get a database handle for global cluster' + unless $dbh; + + return $class->fetch_props_db( $u, $dbh, 'userprop', $props ); +} + +sub set_props { + my ( $class, $u, $propmap, $opts ) = @_; + + my $dbh = LJ::get_db_writer(); + + Carp::croak 'cannot get a database handle for global cluster' + unless $dbh; + + return $class->store_props_db( $u, $dbh, 'userprop', $propmap ); +} + +1; Added: trunk/cgi-bin/LJ/User/PropStorage/Packed.pm =================================================================== --- trunk/cgi-bin/LJ/User/PropStorage/Packed.pm (rev 0) +++ trunk/cgi-bin/LJ/User/PropStorage/Packed.pm 2010-11-29 07:47:35 UTC (rev 17810) @@ -0,0 +1,65 @@ +package LJ::User::PropStorage::Packed; +use strict; +use warnings; + +use base qw(LJ::User::PropStorage); + +use Carp qw(); + +sub use_memcache { return; } + +sub can_handle { + my ($class, $propname) = @_; + + my $propinfo = LJ::get_prop( 'user', $propname ); + return 0 unless $propinfo; + + return 0 unless $propinfo->{'datatype'} =~ /^bit/; + + return 1; +} + +sub get_bit { + my ($class, $propname) = @_; + + my $propinfo = LJ::get_prop( 'user', $propname ); + + Carp::croak "cannot get userprop info for $propname" + unless $propinfo; + + if ($propinfo->{'datatype'} =~ /^bit(\d+)$/) { + return $1; + } + + Carp::croak "$propname is not a packed userprop"; +} + +sub get_props { + my ($class, $u, $props) = @_; + + my %ret; + + foreach my $propname (@$props) { + my $bit = $class->get_bit($propname); + $ret{$propname} = ( $u->packed_props & (1 << $bit) ) ? 1 : 0; + } +} + +sub set_props { + my ($class, $u, $propmap) = @_; + + my $newprops = $u->packed_props; + + foreach my $propname ( keys %$propmap ) { + my $bit = $class->get_bit($propname); + if ( $propmap->{$propname} ) { + $newprops |= (1 << $bit); + } else { + $newprops &= ~(1 << $bit); + } + } + + $u->set_packed_props($newprops); +} + +1; Added: trunk/cgi-bin/LJ/User/PropStorage/UserClusterBlob.pm =================================================================== --- trunk/cgi-bin/LJ/User/PropStorage/UserClusterBlob.pm (rev 0) +++ trunk/cgi-bin/LJ/User/PropStorage/UserClusterBlob.pm 2010-11-29 07:47:35 UTC (rev 17810) @@ -0,0 +1,57 @@ +package LJ::User::PropStorage::UserClusterBlob; +use strict; +use warnings; + +use base qw(LJ::User::PropStorage::DB); + +sub use_memcache { 'blob' } + +sub memcache_key { + my ( $class, $u, $propname ) = @_; + + my $propinfo = LJ::get_prop( 'user', $propname ); + + return 'uprop2:blob:' . $u->id . ':' . $propinfo->{'id'}; +} + +sub can_handle { + my ( $class, $propname ) = @_; + + my $propinfo = LJ::get_prop( 'user', $propname ); + return 0 unless $propinfo; + + return 0 if $propinfo->{'datatype'} =~ /^bit/; + + return 0 if $propinfo->{'multihomed'}; + return 0 if $propinfo->{'indexed'}; + + return 0 unless $propinfo->{'cldversion'}; + return 0 unless $propinfo->{'datatype'} eq 'blobchar'; + + return 1; +} + +sub get_props { + my ( $class, $u, $props, $opts ) = @_; + + my $dbh = $opts->{'use_master'} ? LJ::get_cluster_master($u) + : LJ::get_cluster_reader($u); + + Carp::croak 'cannot get a database handle for user #' . $u->userid + unless $dbh; + + return $class->fetch_props_db( $u, $dbh, 'userpropblob', $props ); +} + +sub set_props { + my ( $class, $u, $propmap, $opts ) = @_; + + my $dbh = LJ::get_cluster_master($u); + + Carp::croak 'cannot get a database handle for user #' . $u->userid + unless $dbh; + + return $class->store_props_db( $u, $dbh, 'userpropblob', $propmap ); +} + +1; Added: trunk/cgi-bin/LJ/User/PropStorage/UserClusterLite.pm =================================================================== --- trunk/cgi-bin/LJ/User/PropStorage/UserClusterLite.pm (rev 0) +++ trunk/cgi-bin/LJ/User/PropStorage/UserClusterLite.pm 2010-11-29 07:47:35 UTC (rev 17810) @@ -0,0 +1,53 @@ +package LJ::User::PropStorage::UserClusterLite; +use strict; +use warnings; + +use base qw(LJ::User::PropStorage::DB); + +sub memcache_key { + my ( $class, $u ) = @_; + + return 'uprop2:lite:' . $u->id; +} + +sub can_handle { + my ( $class, $propname ) = @_; + + my $propinfo = LJ::get_prop( 'user', $propname ); + return 0 unless $propinfo; + + return 0 if $propinfo->{'datatype'} =~ /^bit/; + + return 0 if $propinfo->{'multihomed'}; + return 0 if $propinfo->{'indexed'}; + + return 0 unless $propinfo->{'cldversion'}; + return 0 if $propinfo->{'datatype'} eq 'blobchar'; + + return 1; +} + +sub get_props { + my ( $class, $u, $props, $opts ) = @_; + + my $dbh = $opts->{'use_master'} ? LJ::get_cluster_master($u) + : LJ::get_cluster_reader($u); + + Carp::croak 'cannot get a database handle for user #' . $u->userid + unless $dbh; + + return $class->fetch_props_db( $u, $dbh, 'userproplite2', $props ); +} + +sub set_props { + my ( $class, $u, $propmap, $opts ) = @_; + + my $dbh = LJ::get_cluster_master($u); + + Carp::croak 'cannot get a database handle for user #' . $u->userid + unless $dbh; + + return $class->store_props_db( $u, $dbh, 'userproplite2', $propmap ); +} + +1; Added: trunk/cgi-bin/LJ/User/PropStorage.pm =================================================================== --- trunk/cgi-bin/LJ/User/PropStorage.pm (rev 0) +++ trunk/cgi-bin/LJ/User/PropStorage.pm 2010-11-29 07:47:35 UTC (rev 17810) @@ -0,0 +1,142 @@ +package LJ::User::PropStorage; +use strict; +use warnings; + +use LJ::ModuleLoader; +use Carp qw(); + +# initialization code. do not touch this. +my @SUBCLASSES = LJ::ModuleLoader->module_subclasses(__PACKAGE__); +use LJ::User::PropStorage::DB; +foreach my $class (@SUBCLASSES) { + eval "use $class"; + Carp::confess "Error loading module '$class': $@" if $@; +} + +sub get_handler { + my ($class, $propname) = @_; + + foreach my $class (@SUBCLASSES) { + next unless $class->can_handle($propname); + return $class; + } + + Carp::croak 'cannot get a handler for prop=' . $propname; +} + +sub get_handler_multi { + my ($class, $props) = @_; + + my %ret; + + foreach my $prop (@$props) { + my $handler = $class->get_handler($prop); + + $ret{$handler} = [] unless exists $ret{$handler}; + push @{ $ret{$handler} }, $prop; + } + + return \%ret; +} + +sub get_props { + my ( $class, $u, $props, $opts ) = @_; + + Carp::croak $class . 'does not have a "get" method defined'; +} + +sub set_props { + my ( $class, $u, $propmap, $opts ) = @_; + + Carp::croak $class . 'does not have a "set" method defined'; +} + +sub can_handle { + my ( $class, $propname ) = @_; + + Carp::croak $class . 'does not have a "can_handle" method defined'; +} + +sub use_memcache { + my ($class) = @_; + + Carp::croak $class . 'does not have a "use_memcache" method defined'; +} + +sub memcache_key { + my ( $class, $u, $propname ) = @_; + + Carp::croak $class . 'does not have a "memcache_key" method defined'; +} + +sub fetch_props_memcache { + my ( $class, $u, $props ) = @_; + + my $userid = int $u->userid; + + my ( %propid_map, @memkeys ); + foreach my $k (@$props) { + my $propinfo = LJ::get_prop('user', $k); + my $propid = $propinfo->{'id'}; + my $memkey = $class->memcache_key($u, $k); + + $propid_map{$propid} = $k; + push @memkeys, [ $userid, $memkey ]; + } + + my $from_memcache = LJ::MemCache::get_multi(@memkeys); + + my %ret; + foreach my $k (@$props) { + my $memkey = $class->memcache_key($u, $k); + + next unless exists $from_memcache->{$memkey}; + + $ret{ $propid_map{$k} } = $from_memcache->{$memkey}; + } + + return \%ret; +} + +sub store_props_memcache { + my ( $class, $u, $propmap ) = @_; + + my $userid = int $u->userid; + my $expire = time + 3600 * 24; + + foreach my $k (keys %$propmap) { + my $memkey = $class->memcache_key($u, $k); + + LJ::MemCache::set( [ $userid, $memkey ], + $propmap->{$k} || '', + $expire ); + } +} + +sub pack_for_memcache { + my ( $class, $propmap ) = @_; + + my %ret; + foreach my $propname (keys %$propmap) { + my $propinfo = LJ::get_prop( 'user', $propname ); + my $propid = $propinfo->{'id'}; + + $ret{$propid} = $propmap->{$propname}; + } + + return \%ret; +} + +sub unpack_from_memcache { + my ( $class, $packed ) = @_; + + my %ret; + foreach my $propid (keys %$packed) { + my $propname = $LJ::CACHE_PROPID{'user'}->{$propid}->{'name'}; + $ret{$propname} = $packed->{$propid}; + } + + return \%ret; +} + +1; Modified: trunk/cgi-bin/LJ/User.pm =================================================================== --- trunk/cgi-bin/LJ/User.pm 2010-11-29 04:54:45 UTC (rev 17809) +++ trunk/cgi-bin/LJ/User.pm 2010-11-29 07:47:35 UTC (rev 17810) @@ -26,6 +26,7 @@ use LJ::JSON; use HTTP::Date qw(str2time); use LJ::TimeUtil; +use LJ::User::PropStorage; use Class::Autouse qw( URI @@ -1879,11 +1880,73 @@ # sets prop, and also updates $u's cached version sub set_prop { my ($u, $prop, $value) = @_; - return 0 unless LJ::set_userprop($u, $prop, $value); # FIXME: use exceptions - $u->{$prop} = $value; - LJ::run_hooks("props_changed", $u, {$prop => $value}); + my $propmap = ref $prop ? $prop : { $prop => $value }; + # filter out props that do not change + foreach my $propname (keys %$propmap) { + # it's not loaded, so let's not check it + next unless exists $u->{$propname}; + + if ( (!$propmap->{$propname} && !$u->{$propname}) + || $propmap->{$propname} eq $u->{$propname} ) + { + delete $propmap->{$propname}; + } + } + + my @props_affected = keys %$propmap; + my $groups = LJ::User::PropStorage->get_handler_multi(\@props_affected); + my $memcache_available = @LJ::MEMCACHE_SERVERS; + + my $memc_expire = time + 3600 * 24; + + foreach my $handler (keys %$groups) { + my $propnames_handled = $groups->{$handler}; + my %propmap_handled = map { $_ => $propmap->{$_} } + @$propnames_handled; + + # first, actually save stuff to the database; + # then, delete it from memcache, depending on the memcache + # policy of the handler + $handler->set_props( $u, \%propmap_handled ); + + # if there is no memcache, or if the handler doesn't wish to use + # memcache, we don't need to deal with it, yay + if ( !$memcache_available || !defined $handler->use_memcache ) + { + next; + } + + # now let's find out what we're going to do with memcache + my $memcache_policy = $handler->use_memcache; + + if ( $memcache_policy eq 'lite' ) { + # the handler loads everything from the corresponding + # table and uses only one memcache key to cache that + + my $memkey = $handler->memcache_key($u); + LJ::MemCache::delete([ $u->userid, $memkey ]); + } elsif ( $memcache_policy eq 'blob' ) { + # the handler uses one memcache key for each prop, + # so let's delete them all + + foreach my $propname (@$propnames_handled) { + my $memkey = $handler->memcache_key( $u, $propname ); + LJ::MemCache::delete([ $u->userid, $memkey ]); + } + } + } + + # now, actually reflect that we've changed the props in the + # user object + foreach my $propname (keys %$propmap) { + $u->{$propname} = $propmap->{$propname}; + } + + # and run the hooks, too + LJ::run_hooks( 'props_changed', $u, $propmap ); + return $value; } @@ -5631,7 +5694,7 @@ my ($u, $url) = @_; my $uid = $u->id; - LJ::set_userprop($u, 'custom_usericon', $url); + $u->set_prop( 'custom_usericon' => $url ); } sub _subscriptions_count { @@ -5653,6 +5716,18 @@ return $count; } +sub packed_props { + my ($u) = @_; + return $u->{'packed_props'}; +} + +sub set_packed_props { + my ($u, $newprops) = @_; + + LJ::update_user($u, { 'packed_props' => $newprops }); + $u->{'packed_props'} = 1; +} + sub reset_cache { my $u = shift; @@ -5912,8 +5987,7 @@ # des-opts: hashref of opts. set key 'cache' to use memcache. # des-propname: the name of a property from the [dbtable[userproplist]] table. # </LJFUNC> -sub load_user_props -{ +sub load_user_props { &nodb; my $u = shift; @@ -5923,155 +5997,91 @@ my $opts = ref $_[0] ? shift : {}; my (@props) = @_; - my ($sql, $sth); - LJ::load_props("user"); + my $extend_user_object = sub { + my ($propmap) = @_; - ## user reference - my $uid = $u->{'userid'}+0; - $uid = LJ::get_userid($u->{'user'}) unless $uid; + foreach my $propname ( keys %$propmap ) { + $u->{$propname} = $propmap->{$propname}; + } + }; - my $mem = {}; - my $use_master = 0; - my $used_slave = 0; # set later if we ended up using a slave + my $groups = LJ::User::PropStorage->get_handler_multi(\@props); + my $memcache_available = @LJ::MEMCACHE_SERVERS; + my $use_master = $memcache_available || $opts->{'use_master'}; - if (@LJ::MEMCACHE_SERVERS) { - my @keys; - foreach (@props) { - next if exists $u->{$_}; - my $p = LJ::get_prop("user", $_); - die "Invalid userprop $_ passed to LJ::load_user_props." unless $p; - push @keys, [$uid,"uprop:$uid:$p->{'id'}"]; - } - $mem = LJ::MemCache::get_multi(@keys) || {}; - $use_master = 1; - } + my $memc_expire = time() + 3600 * 24; - $use_master = 1 if $opts->{'use_master'}; + foreach my $handler (keys %$groups) { + # if there is no memcache, or if the handler doesn't wish to use + # memcache, hit the storage directly, update the user object, + # and get straight to the next handler + if ( !$memcache_available || !defined $handler->use_memcache ) + { + my $propmap + = $handler->get_props( $u, $groups->{$handler}, + { 'use_master' => $use_master } ); - my @needwrite; # [propid, propname] entries we need to save to memcache later + $extend_user_object->($propmap); - my %loadfrom; - my %multihomed; # ( $propid => 0/1 ) # 0 if we haven't loaded it, 1 if we have - unless (@props) { - # case 1: load all props for a given user. - # multihomed props are stored on userprop and userproplite2, but since they - # should always be in sync, it doesn't matter which gets loaded first, the - # net results should be the same. see doc/server/lj.int.multihomed_userprops.html - # for more information. - $loadfrom{'userprop'} = 1; - $loadfrom{'userproplite'} = 1; - $loadfrom{'userproplite2'} = 1; - $loadfrom{'userpropblob'} = 1; - } else { - # case 2: load only certain things - foreach (@props) { - next if exists $u->{$_}; - my $p = LJ::get_prop("user", $_); - die "Invalid userprop $_ passed to LJ::load_user_props." unless $p; - if (defined $mem->{"uprop:$uid:$p->{'id'}"}) { - $u->{$_} = $mem->{"uprop:$uid:$p->{'id'}"}; - next; - } - push @needwrite, [ $p->{'id'}, $_ ]; - my $source = $p->{'indexed'} ? "userprop" : "userproplite"; - if ($p->{datatype} eq 'blobchar') { - $source = "userpropblob"; # clustered blob - } - elsif ($p->{'cldversion'} && $u->{'dversion'} >= $p->{'cldversion'}) { - $source = "userproplite2"; # clustered - } - elsif ($p->{multihomed}) { - $multihomed{$p->{id}} = 0; - $source = "userproplite2"; - } - push @{$loadfrom{$source}}, $p->{'id'}; + next; } - } - foreach my $table (qw{userproplite userproplite2 userpropblob userprop}) { - next unless exists $loadfrom{$table}; - my $db; - if ($use_master) { - $db = ($table =~ m{userprop(lite2|blob)}) ? - LJ::get_cluster_master($u) : - LJ::get_db_writer(); - } - unless ($db) { - $db = ($table =~ m{userprop(lite2|blob)}) ? - LJ::get_cluster_reader($u) : - LJ::get_db_reader(); - $used_slave = 1; - } - $sql = "SELECT upropid, value FROM $table WHERE userid=$uid"; - if (ref $loadfrom{$table}) { - $sql .= " AND upropid IN (" . join(",", @{$loadfrom{$table}}) . ")"; - } - $sth = $db->prepare($sql); - $sth->execute; - while (my ($id, $v) = $sth->fetchrow_array) { - delete $multihomed{$id} if $table eq 'userproplite2'; - $u->{$LJ::CACHE_PROPID{'user'}->{$id}->{'name'}} = $v; - } + # now let's find out what we're going to do with memcache + my $memcache_policy = $handler->use_memcache; - # push back multihomed if necessary - if ($table eq 'userproplite2') { - push @{$loadfrom{userprop}}, $_ foreach keys %multihomed; - } - } + if ( $memcache_policy eq 'lite' ) { + # the handler loads everything from the corresponding + # table and uses only one memcache key to cache that - # see if we failed to get anything above and need to hit the master. - # this usually happens the first time a multihomed prop is hit. this - # code will propagate that prop down to the cluster. - if (%multihomed) { + my $memkey = $handler->memcache_key($u); + if ( my $packed = LJ::MemCache::get([ $u->userid, $memkey ]) ) { + my $propmap + = LJ::User::PropStorage->unpack_from_memcache($packed); - # verify that we got the database handle before we try propagating data - if ($u->writer) { - my @values; - foreach my $id (keys %multihomed) { - my $pname = $LJ::CACHE_PROPID{user}{$id}{name}; - if (defined $u->{$pname} && $u->{$pname}) { - push @values, "($uid, $id, " . $u->quote($u->{$pname}) . ")"; - } else { - push @values, "($uid, $id, '')"; - } + $extend_user_object->($propmap); + + # we've loaded everything handled by this handler, + # let's get to the next one + next; } - $u->do("REPLACE INTO userproplite2 VALUES " . join ',', @values); - } - } - # Add defaults to user object. + # actually load everything from DB, cache it, and update + # the user object + my $propmap + = $handler->get_props( $u, [], + { 'use_master' => $use_master } ); - # defaults for S1 style IDs in config file are magic: really - # uniq strings representing style IDs, so on first use, we need - # to map them - unless ($LJ::CACHED_S1IDMAP) { + my $packed = LJ::User::PropStorage->pack_for_memcache($propmap); + LJ::MemCache::set( [$u->userid, $memkey], + $packed, $memc_expire ); - my $pubsty = LJ::S1::get_public_styles(); - foreach (values %$pubsty) { - my $k = "s1_$_->{'type'}_style"; - next unless $LJ::USERPROP_DEF{$k} eq "$_->{'type'}/$_->{'styledes'}"; + $extend_user_object->($propmap); + } elsif ( $memcache_policy eq 'blob' ) { + # the handler uses one memcache key for each prop + # hit memcache for them first, then hit db and update + # memcache - $LJ::USERPROP_DEF{$k} = $_->{'styleid'}; - } + my $handled_props = $groups->{$handler}; - $LJ::CACHED_S1IDMAP = 1; - } + my $propmap_memc + = $handler->fetch_props_memcache( $u, $handled_props ); - # If this was called with no @props, then the function tried - # to load all metadata. but we don't know what's missing, so - # try to apply all defaults. - unless (@props) { @props = keys %LJ::USERPROP_DEF; } + $extend_user_object->($propmap_memc); - foreach my $prop (@props) { - next if (defined $u->{$prop}); - $u->{$prop} = $LJ::USERPROP_DEF{$prop}; - } + my @load_from_db = grep { !exists $propmap_memc->{$_} } + @$handled_props; - unless ($used_slave) { - my $expire = time() + 3600*24; - foreach my $wr (@needwrite) { - my ($id, $name) = ($wr->[0], $wr->[1]); - LJ::MemCache::set([$uid,"uprop:$uid:$id"], $u->{$name} || "", $expire); + # if we can avoid hitting the db, avoid it + next unless @load_from_db; + + my $propmap_db + = $handler->get_props( $u, \@load_from_db, + { 'use_master' => $use_master } ); + + $extend_user_object->($propmap_db); + + # now, update memcache + $handler->store_props_memcache( $u, $propmap_db ); } } } @@ -6850,117 +6860,6 @@ } # <LJFUNC> -# name: LJ::set_userprop -# des: Sets/deletes a userprop by name for a user. -# info: This adds or deletes from the -# [dbtable[userprop]]/[dbtable[userproplite]] tables. One -# crappy thing about this interface is that it doesn't allow -# a batch of userprops to be updated at once, which is the -# common thing to do. -# args: dbarg?, uuserid, propname, value, memonly? -# des-uuserid: The userid of the user or a user hashref. -# des-propname: The name of the property. Or a hashref of propname keys and corresponding values. -# des-value: The value to set to the property. If undefined or the -# empty string, then property is deleted. -# des-memonly: if true, only writes to memcache, and not to database. -# </LJFUNC> -sub set_userprop -{ - &nodb; - my ($u, $propname, $value, $memonly) = @_; - $u = ref $u ? $u : LJ::load_userid($u); - my $userid = $u->{'userid'}+0; - - my $hash = ref $propname eq "HASH" ? $propname : { $propname => $value }; - - my %action; # $table -> {"replace"|"delete"} -> [ "($userid, $propid, $qvalue)" | propid ] - my %multihomed; # { $propid => $value } - - foreach $propname (keys %$hash) { - LJ::run_hook("setprop", prop => $propname, - u => $u, value => $value); - - my $p = LJ::get_prop("user", $propname) or - die "Invalid userprop $propname passed to LJ::set_userprop."; - if ($p->{multihomed}) { - # collect into array for later handling - $multihomed{$p->{id}} = $hash->{$propname}; - next; - } - my $table = $p->{'indexed'} ? "userprop" : "userproplite"; - if ($p->{datatype} eq 'blobchar') { - $table = 'userpropblob'; - } - elsif ($p->{'cldversion'} && $u->{'dversion'} >= $p->{'cldversion'}) { - $table = "userproplite2"; - } - unless ($memonly) { - my $db = $action{$table}->{'db'} ||= ( - $table !~ m{userprop(lite2|blob)} - ? LJ::get_db_writer() - : $u->writer ); - return 0 unless $db; - } - $value = $hash->{$propname}; - if (defined $value && $value) { - push @{$action{$table}->{"replace"}}, [ $p->{'id'}, $value ]; - } else { - push @{$action{$table}->{"delete"}}, $p->{'id'}; - } - } - - my $expire = time() + 3600*24; - foreach my $table (keys %action) { - my $db = $action{$table}->{'db'}; - if (my $list = $action{$table}->{"replace"}) { - if ($db) { - my $vals = join(',', map { "($userid,$_->[0]," . $db->quote($_->[1]) . ")" } @$list); - $db->do("REPLACE INTO $table (userid, upropid, value) VALUES $vals"); - } - LJ::MemCache::set([$userid,"uprop:$userid:$_->[0]"], $_->[1], $expire) foreach (@$list); - } - if (my $list = $action{$table}->{"delete"}) { - if ($db) { - my $in = join(',', @$list); - $db->do("DELETE FROM $table WHERE userid=$userid AND upropid IN ($in)"); - } - LJ::MemCache::set([$userid,"uprop:$userid:$_"], "", $expire) foreach (@$list); - } - } - - # if we had any multihomed props, set them here - if (%multihomed) { - my $dbh = LJ::get_db_writer(); - return 0 unless $dbh && $u->writer; - while (my ($propid, $pvalue) = each %multihomed) { - if (defined $pvalue && $pvalue) { - # replace data into master - $dbh->do("REPLACE INTO userprop VALUES (?, ?, ?)", - undef, $userid, $propid, $pvalue); - } else { - # delete data from master, but keep in cluster - $dbh->do("DELETE FROM userprop WHERE userid = ? AND upropid = ?", - undef, $userid, $propid); - } - - # fail out? - return 0 if $dbh->err; - - # put data in cluster - $pvalue ||= ''; - $u->do("REPLACE INTO userproplite2 VALUES (?, ?, ?)", - undef, $userid, $propid, $pvalue); - return 0 if $u->err; - - # set memcache - LJ::MemCache::set([$userid,"uprop:$userid:$propid"], $pvalue, $expire); - } - } - - return 1; -} - -# <LJFUNC> # name: LJ::get_shared_journals # des: Gets an array of shared journals a user has access to. # returns: An array of shared journals. Modified: trunk/cgi-bin/ljdb.pl =================================================================== --- trunk/cgi-bin/ljdb.pl 2010-11-29 04:54:45 UTC (rev 17809) +++ trunk/cgi-bin/ljdb.pl 2010-11-29 07:47:35 UTC (rev 17810) @@ -403,6 +403,8 @@ sub get_dbirole_dbh { my $dbh = $LJ::DBIRole->get_dbh( @_ ) or return undef; + $dbh->{'RaiseError'} = 1 if $LJ::IS_DEV_SERVER; + if ( $LJ::DB_LOG_HOST && $LJ::HAVE_DBI_PROFILE ) { $LJ::DB_REPORT_HANDLES{ $dbh->{Name} } = $dbh;