Committer: ailyin
LJSUP-7462 (refactor userprops)U trunk/bin/upgrading/update-db-general.pl U trunk/cgi-bin/LJ/MemCache.pm A trunk/cgi-bin/LJ/User/PropStorage/ A trunk/cgi-bin/LJ/User/PropStorage/DB.pm A trunk/cgi-bin/LJ/User/PropStorage/Global.pm A trunk/cgi-bin/LJ/User/PropStorage/GlobalIndexed.pm A trunk/cgi-bin/LJ/User/PropStorage/Packed.pm A trunk/cgi-bin/LJ/User/PropStorage/UserClusterBlob.pm A trunk/cgi-bin/LJ/User/PropStorage/UserClusterLite.pm A trunk/cgi-bin/LJ/User/PropStorage.pm U trunk/cgi-bin/LJ/User.pm U trunk/cgi-bin/ljdb.pl
Modified: trunk/bin/upgrading/update-db-general.pl
===================================================================
--- trunk/bin/upgrading/update-db-general.pl 2010-11-29 04:54:45 UTC (rev 17809)
+++ trunk/bin/upgrading/update-db-general.pl 2010-11-29 07:47:35 UTC (rev 17810)
@@ -653,9 +653,8 @@
CREATE TABLE user (
userid int(10) unsigned NOT NULL auto_increment,
user char(15) default NULL,
- caps SMALLINT UNSIGNED NOT NULL DEFAULT 0,
- email char(50) default NULL,
- password char(30) default NULL,
+ caps BIGINT UNSIGNED NOT NULL DEFAULT 0,
+ packed_props BIGINT UNSIGNED NOT NULL DEFAULT 0,
status char(1) NOT NULL default 'N',
statusvis char(1) NOT NULL default 'V',
statusvisdate datetime default NULL,
@@ -681,12 +680,21 @@
lang char(2) NOT NULL default 'EN',
PRIMARY KEY (userid),
UNIQUE KEY user (user),
- KEY (email),
KEY (status),
KEY (statusvis)
) PACK_KEYS=1
EOC
+if ( column_type('user', 'caps') =~ /smallint/i ) {
+ do_alter('user', qq{
+ ALTER TABLE user
+ DROP COLUMN email,
+ DROP COLUMN password,
+ MODIFY COLUMN caps BIGINT UNSIGNED NOT NULL DEFAULT 0,
+ ADD COLUMN packed_props BIGINT UNSIGNED NOT NULL DEFAULT 0
+ });
+}
+
register_tablecreate("userbio", <<'EOC');
CREATE TABLE userbio (
userid int(10) unsigned NOT NULL default '0',
Modified: trunk/cgi-bin/LJ/MemCache.pm
===================================================================
--- trunk/cgi-bin/LJ/MemCache.pm 2010-11-29 04:54:45 UTC (rev 17809)
+++ trunk/cgi-bin/LJ/MemCache.pm 2010-11-29 07:47:35 UTC (rev 17810)
@@ -32,11 +32,11 @@
%LJ::MEMCACHE_ARRAYFMT = (
'user' =>
- [qw[1 userid user caps clusterid dversion email password status statusvis statusvisdate
+ [qw[2 userid user caps clusterid dversion status statusvis statusvisdate
name bdate themeid moodthemeid opt_forcemoodtheme allow_infoshow allow_contactshow
allow_getljnews opt_showtalklinks opt_whocanreply opt_gettalkemail opt_htmlemail
opt_mangleemail useoverrides defaultpicid has_bio txtmsg_status is_system
- journaltype lang oldenc]],
+ journaltype lang oldenc packed_props]],
'fgrp' => [qw[1 userid groupnum groupname sortorder is_public]],
# version #101 because old userpic format in memcached was an arrayref of
# [width, height, ...] and widths could have been 1 before, although unlikely
Added: trunk/cgi-bin/LJ/User/PropStorage/DB.pm
===================================================================
--- trunk/cgi-bin/LJ/User/PropStorage/DB.pm (rev 0)
+++ trunk/cgi-bin/LJ/User/PropStorage/DB.pm 2010-11-29 07:47:35 UTC (rev 17810)
@@ -0,0 +1,91 @@
+package LJ::User::PropStorage::DB;
+use strict;
+use warnings;
+
+use base qw( LJ::User::PropStorage );
+
+sub can_handle { 0 }
+sub use_memcache { 'lite' }
+
+sub fetch_props_db {
+ my ( $class, $u, $dbh, $table, $props ) = @_;
+
+ my $userid = int $u->userid;
+
+ LJ::load_props('user');
+
+ my @propids;
+ foreach my $k (@$props) {
+ my $propinfo = LJ::get_prop('user', $k);
+ my $propid = $propinfo->{'id'};
+
+ push @propids, $propid;
+ }
+
+ my %propid_map = %{ $LJ::CACHE_PROPID{'user'} };
+
+ my %ret = map { $_ => undef } @$props;
+ my $sql;
+ if ( my $propids_in = join(',', @propids) ) {
+ $sql = qq{
+ SELECT upropid, value
+ FROM $table
+ WHERE userid=? AND upropid IN ($propids_in)
+ };
+ } else {
+ # well, it seems that they didn't pass us any props, so
+ # let's use a somewhat different SQL query to load everything
+ # from the given table
+ $sql = qq{
+ SELECT upropid, value
+ FROM $table
+ WHERE userid=?
+ };
+ }
+
+ my $res = $dbh->selectall_arrayref($sql, { 'Slice' => {} }, $userid);
+
+ foreach my $row (@$res) {
+ my $propname = $propid_map{ $row->{'upropid'} }->{'name'};
+ $ret{$propname} = $row->{'value'};
+ }
+
+ return \%ret;
+}
+
+sub store_props_db {
+ my ( $class, $u, $dbh, $table, $propmap ) = @_;
+
+ my $userid = int $u->userid;
+
+ my ( @values_sql, @subst, @propid_delete );
+ foreach my $k (keys %$propmap) {
+ my $propinfo = LJ::get_prop('user', $k);
+ my $propid = $propinfo->{'id'};
+
+ unless ( defined $propmap->{$k} && $propmap->{$k} ) {
+ push @propid_delete, $propid;
+ next;
+ }
+
+ push @values_sql, '(?, ?, ?)';
+ push @subst, $userid, $propid, $propmap->{$k};
+ }
+
+ if ( my $values_sql = join(',', @values_sql) ) {
+ $dbh->do(qq{
+ REPLACE INTO $table
+ (userid, upropid, value)
+ VALUES $values_sql
+ }, undef, @subst);
+ }
+
+ if ( my $props_delete_in = join(',', @propid_delete) ) {
+ $dbh->do(qq{
+ DELETE FROM $table
+ WHERE userid=? AND upropid IN ($props_delete_in)
+ }, undef, $userid);
+ }
+}
+
+1;
Added: trunk/cgi-bin/LJ/User/PropStorage/Global.pm
===================================================================
--- trunk/cgi-bin/LJ/User/PropStorage/Global.pm (rev 0)
+++ trunk/cgi-bin/LJ/User/PropStorage/Global.pm 2010-11-29 07:47:35 UTC (rev 17810)
@@ -0,0 +1,53 @@
+package LJ::User::PropStorage::Global;
+use strict;
+use warnings;
+
+use base qw( LJ::User::PropStorage::DB );
+
+sub memcache_key {
+ my ( $class, $u ) = @_;
+
+ return 'uprop2:global:' . $u->id;
+}
+
+sub can_handle {
+ my ( $class, $propname ) = @_;
+
+ my $propinfo = LJ::get_prop( 'user', $propname );
+ return 0 unless $propinfo;
+
+ return 0 if $propinfo->{'datatype'} =~ /^bit/;
+
+ return 0 if $propinfo->{'multihomed'};
+ return 0 if $propinfo->{'indexed'};
+
+ return 0 if $propinfo->{'cldversion'};
+ return 0 if $propinfo->{'datatype'} eq 'blobchar';
+
+ return 1;
+}
+
+sub get_props {
+ my ( $class, $u, $props, $opts ) = @_;
+
+ my $dbh = $opts->{'use_master'} ? LJ::get_db_writer()
+ : LJ::get_db_reader();
+
+ Carp::croak 'cannot get a database handle for global cluster'
+ unless $dbh;
+
+ return $class->fetch_props_db( $u, $dbh, 'userproplite', $props );
+}
+
+sub set_props {
+ my ( $class, $u, $propmap, $opts ) = @_;
+
+ my $dbh = LJ::get_db_writer();
+
+ Carp::croak 'cannot get a database handle for global cluster'
+ unless $dbh;
+
+ return $class->store_props_db( $u, $dbh, 'userproplite', $propmap );
+}
+
+1;
Added: trunk/cgi-bin/LJ/User/PropStorage/GlobalIndexed.pm
===================================================================
--- trunk/cgi-bin/LJ/User/PropStorage/GlobalIndexed.pm (rev 0)
+++ trunk/cgi-bin/LJ/User/PropStorage/GlobalIndexed.pm 2010-11-29 07:47:35 UTC (rev 17810)
@@ -0,0 +1,49 @@
+package LJ::User::PropStorage::GlobalIndexed;
+use strict;
+use warnings;
+
+use base qw(LJ::User::PropStorage::DB);
+
+sub memcache_key {
+ my ( $class, $u ) = @_;
+
+ return 'uprop2:indexed:' . $u->id;
+}
+
+sub can_handle {
+ my ( $class, $propname ) = @_;
+
+ my $propinfo = LJ::get_prop( 'user', $propname );
+ return 0 unless $propinfo;
+
+ return 0 if $propinfo->{'datatype'} =~ /^bit/;
+
+ return 0 unless $propinfo->{'multihomed'} || $propinfo->{'indexed'};
+
+ return 1;
+}
+
+sub get_props {
+ my ( $class, $u, $props, $opts ) = @_;
+
+ my $dbh = $opts->{'use_master'} ? LJ::get_db_writer()
+ : LJ::get_db_reader();
+
+ Carp::croak 'cannot get a database handle for global cluster'
+ unless $dbh;
+
+ return $class->fetch_props_db( $u, $dbh, 'userprop', $props );
+}
+
+sub set_props {
+ my ( $class, $u, $propmap, $opts ) = @_;
+
+ my $dbh = LJ::get_db_writer();
+
+ Carp::croak 'cannot get a database handle for global cluster'
+ unless $dbh;
+
+ return $class->store_props_db( $u, $dbh, 'userprop', $propmap );
+}
+
+1;
Added: trunk/cgi-bin/LJ/User/PropStorage/Packed.pm
===================================================================
--- trunk/cgi-bin/LJ/User/PropStorage/Packed.pm (rev 0)
+++ trunk/cgi-bin/LJ/User/PropStorage/Packed.pm 2010-11-29 07:47:35 UTC (rev 17810)
@@ -0,0 +1,65 @@
+package LJ::User::PropStorage::Packed;
+use strict;
+use warnings;
+
+use base qw(LJ::User::PropStorage);
+
+use Carp qw();
+
+sub use_memcache { return; }
+
+sub can_handle {
+ my ($class, $propname) = @_;
+
+ my $propinfo = LJ::get_prop( 'user', $propname );
+ return 0 unless $propinfo;
+
+ return 0 unless $propinfo->{'datatype'} =~ /^bit/;
+
+ return 1;
+}
+
+sub get_bit {
+ my ($class, $propname) = @_;
+
+ my $propinfo = LJ::get_prop( 'user', $propname );
+
+ Carp::croak "cannot get userprop info for $propname"
+ unless $propinfo;
+
+ if ($propinfo->{'datatype'} =~ /^bit(\d+)$/) {
+ return $1;
+ }
+
+ Carp::croak "$propname is not a packed userprop";
+}
+
+sub get_props {
+ my ($class, $u, $props) = @_;
+
+ my %ret;
+
+ foreach my $propname (@$props) {
+ my $bit = $class->get_bit($propname);
+ $ret{$propname} = ( $u->packed_props & (1 << $bit) ) ? 1 : 0;
+ }
+}
+
+sub set_props {
+ my ($class, $u, $propmap) = @_;
+
+ my $newprops = $u->packed_props;
+
+ foreach my $propname ( keys %$propmap ) {
+ my $bit = $class->get_bit($propname);
+ if ( $propmap->{$propname} ) {
+ $newprops |= (1 << $bit);
+ } else {
+ $newprops &= ~(1 << $bit);
+ }
+ }
+
+ $u->set_packed_props($newprops);
+}
+
+1;
Added: trunk/cgi-bin/LJ/User/PropStorage/UserClusterBlob.pm
===================================================================
--- trunk/cgi-bin/LJ/User/PropStorage/UserClusterBlob.pm (rev 0)
+++ trunk/cgi-bin/LJ/User/PropStorage/UserClusterBlob.pm 2010-11-29 07:47:35 UTC (rev 17810)
@@ -0,0 +1,57 @@
+package LJ::User::PropStorage::UserClusterBlob;
+use strict;
+use warnings;
+
+use base qw(LJ::User::PropStorage::DB);
+
+sub use_memcache { 'blob' }
+
+sub memcache_key {
+ my ( $class, $u, $propname ) = @_;
+
+ my $propinfo = LJ::get_prop( 'user', $propname );
+
+ return 'uprop2:blob:' . $u->id . ':' . $propinfo->{'id'};
+}
+
+sub can_handle {
+ my ( $class, $propname ) = @_;
+
+ my $propinfo = LJ::get_prop( 'user', $propname );
+ return 0 unless $propinfo;
+
+ return 0 if $propinfo->{'datatype'} =~ /^bit/;
+
+ return 0 if $propinfo->{'multihomed'};
+ return 0 if $propinfo->{'indexed'};
+
+ return 0 unless $propinfo->{'cldversion'};
+ return 0 unless $propinfo->{'datatype'} eq 'blobchar';
+
+ return 1;
+}
+
+sub get_props {
+ my ( $class, $u, $props, $opts ) = @_;
+
+ my $dbh = $opts->{'use_master'} ? LJ::get_cluster_master($u)
+ : LJ::get_cluster_reader($u);
+
+ Carp::croak 'cannot get a database handle for user #' . $u->userid
+ unless $dbh;
+
+ return $class->fetch_props_db( $u, $dbh, 'userpropblob', $props );
+}
+
+sub set_props {
+ my ( $class, $u, $propmap, $opts ) = @_;
+
+ my $dbh = LJ::get_cluster_master($u);
+
+ Carp::croak 'cannot get a database handle for user #' . $u->userid
+ unless $dbh;
+
+ return $class->store_props_db( $u, $dbh, 'userpropblob', $propmap );
+}
+
+1;
Added: trunk/cgi-bin/LJ/User/PropStorage/UserClusterLite.pm
===================================================================
--- trunk/cgi-bin/LJ/User/PropStorage/UserClusterLite.pm (rev 0)
+++ trunk/cgi-bin/LJ/User/PropStorage/UserClusterLite.pm 2010-11-29 07:47:35 UTC (rev 17810)
@@ -0,0 +1,53 @@
+package LJ::User::PropStorage::UserClusterLite;
+use strict;
+use warnings;
+
+use base qw(LJ::User::PropStorage::DB);
+
+sub memcache_key {
+ my ( $class, $u ) = @_;
+
+ return 'uprop2:lite:' . $u->id;
+}
+
+sub can_handle {
+ my ( $class, $propname ) = @_;
+
+ my $propinfo = LJ::get_prop( 'user', $propname );
+ return 0 unless $propinfo;
+
+ return 0 if $propinfo->{'datatype'} =~ /^bit/;
+
+ return 0 if $propinfo->{'multihomed'};
+ return 0 if $propinfo->{'indexed'};
+
+ return 0 unless $propinfo->{'cldversion'};
+ return 0 if $propinfo->{'datatype'} eq 'blobchar';
+
+ return 1;
+}
+
+sub get_props {
+ my ( $class, $u, $props, $opts ) = @_;
+
+ my $dbh = $opts->{'use_master'} ? LJ::get_cluster_master($u)
+ : LJ::get_cluster_reader($u);
+
+ Carp::croak 'cannot get a database handle for user #' . $u->userid
+ unless $dbh;
+
+ return $class->fetch_props_db( $u, $dbh, 'userproplite2', $props );
+}
+
+sub set_props {
+ my ( $class, $u, $propmap, $opts ) = @_;
+
+ my $dbh = LJ::get_cluster_master($u);
+
+ Carp::croak 'cannot get a database handle for user #' . $u->userid
+ unless $dbh;
+
+ return $class->store_props_db( $u, $dbh, 'userproplite2', $propmap );
+}
+
+1;
Added: trunk/cgi-bin/LJ/User/PropStorage.pm
===================================================================
--- trunk/cgi-bin/LJ/User/PropStorage.pm (rev 0)
+++ trunk/cgi-bin/LJ/User/PropStorage.pm 2010-11-29 07:47:35 UTC (rev 17810)
@@ -0,0 +1,142 @@
+package LJ::User::PropStorage;
+use strict;
+use warnings;
+
+use LJ::ModuleLoader;
+use Carp qw();
+
+# initialization code. do not touch this.
+my @SUBCLASSES = LJ::ModuleLoader->module_subclasses(__PACKAGE__);
+use LJ::User::PropStorage::DB;
+foreach my $class (@SUBCLASSES) {
+ eval "use $class";
+ Carp::confess "Error loading module '$class': $@" if $@;
+}
+
+sub get_handler {
+ my ($class, $propname) = @_;
+
+ foreach my $class (@SUBCLASSES) {
+ next unless $class->can_handle($propname);
+ return $class;
+ }
+
+ Carp::croak 'cannot get a handler for prop=' . $propname;
+}
+
+sub get_handler_multi {
+ my ($class, $props) = @_;
+
+ my %ret;
+
+ foreach my $prop (@$props) {
+ my $handler = $class->get_handler($prop);
+
+ $ret{$handler} = [] unless exists $ret{$handler};
+ push @{ $ret{$handler} }, $prop;
+ }
+
+ return \%ret;
+}
+
+sub get_props {
+ my ( $class, $u, $props, $opts ) = @_;
+
+ Carp::croak $class . 'does not have a "get" method defined';
+}
+
+sub set_props {
+ my ( $class, $u, $propmap, $opts ) = @_;
+
+ Carp::croak $class . 'does not have a "set" method defined';
+}
+
+sub can_handle {
+ my ( $class, $propname ) = @_;
+
+ Carp::croak $class . 'does not have a "can_handle" method defined';
+}
+
+sub use_memcache {
+ my ($class) = @_;
+
+ Carp::croak $class . 'does not have a "use_memcache" method defined';
+}
+
+sub memcache_key {
+ my ( $class, $u, $propname ) = @_;
+
+ Carp::croak $class . 'does not have a "memcache_key" method defined';
+}
+
+sub fetch_props_memcache {
+ my ( $class, $u, $props ) = @_;
+
+ my $userid = int $u->userid;
+
+ my ( %propid_map, @memkeys );
+ foreach my $k (@$props) {
+ my $propinfo = LJ::get_prop('user', $k);
+ my $propid = $propinfo->{'id'};
+ my $memkey = $class->memcache_key($u, $k);
+
+ $propid_map{$propid} = $k;
+ push @memkeys, [ $userid, $memkey ];
+ }
+
+ my $from_memcache = LJ::MemCache::get_multi(@memkeys);
+
+ my %ret;
+ foreach my $k (@$props) {
+ my $memkey = $class->memcache_key($u, $k);
+
+ next unless exists $from_memcache->{$memkey};
+
+ $ret{ $propid_map{$k} } = $from_memcache->{$memkey};
+ }
+
+ return \%ret;
+}
+
+sub store_props_memcache {
+ my ( $class, $u, $propmap ) = @_;
+
+ my $userid = int $u->userid;
+ my $expire = time + 3600 * 24;
+
+ foreach my $k (keys %$propmap) {
+ my $memkey = $class->memcache_key($u, $k);
+
+ LJ::MemCache::set( [ $userid, $memkey ],
+ $propmap->{$k} || '',
+ $expire );
+ }
+}
+
+sub pack_for_memcache {
+ my ( $class, $propmap ) = @_;
+
+ my %ret;
+ foreach my $propname (keys %$propmap) {
+ my $propinfo = LJ::get_prop( 'user', $propname );
+ my $propid = $propinfo->{'id'};
+
+ $ret{$propid} = $propmap->{$propname};
+ }
+
+ return \%ret;
+}
+
+sub unpack_from_memcache {
+ my ( $class, $packed ) = @_;
+
+ my %ret;
+ foreach my $propid (keys %$packed) {
+ my $propname = $LJ::CACHE_PROPID{'user'}->{$propid}->{'name'};
+ $ret{$propname} = $packed->{$propid};
+ }
+
+ return \%ret;
+}
+
+1;
Modified: trunk/cgi-bin/LJ/User.pm
===================================================================
--- trunk/cgi-bin/LJ/User.pm 2010-11-29 04:54:45 UTC (rev 17809)
+++ trunk/cgi-bin/LJ/User.pm 2010-11-29 07:47:35 UTC (rev 17810)
@@ -26,6 +26,7 @@
use LJ::JSON;
use HTTP::Date qw(str2time);
use LJ::TimeUtil;
+use LJ::User::PropStorage;
use Class::Autouse qw(
URI
@@ -1879,11 +1880,73 @@
# sets prop, and also updates $u's cached version
sub set_prop {
my ($u, $prop, $value) = @_;
- return 0 unless LJ::set_userprop($u, $prop, $value); # FIXME: use exceptions
- $u->{$prop} = $value;
- LJ::run_hooks("props_changed", $u, {$prop => $value});
+ my $propmap = ref $prop ? $prop : { $prop => $value };
+ # filter out props that do not change
+ foreach my $propname (keys %$propmap) {
+ # it's not loaded, so let's not check it
+ next unless exists $u->{$propname};
+
+ if ( (!$propmap->{$propname} && !$u->{$propname})
+ || $propmap->{$propname} eq $u->{$propname} )
+ {
+ delete $propmap->{$propname};
+ }
+ }
+
+ my @props_affected = keys %$propmap;
+ my $groups = LJ::User::PropStorage->get_handler_multi(\@props_affected);
+ my $memcache_available = @LJ::MEMCACHE_SERVERS;
+
+ my $memc_expire = time + 3600 * 24;
+
+ foreach my $handler (keys %$groups) {
+ my $propnames_handled = $groups->{$handler};
+ my %propmap_handled = map { $_ => $propmap->{$_} }
+ @$propnames_handled;
+
+ # first, actually save stuff to the database;
+ # then, delete it from memcache, depending on the memcache
+ # policy of the handler
+ $handler->set_props( $u, \%propmap_handled );
+
+ # if there is no memcache, or if the handler doesn't wish to use
+ # memcache, we don't need to deal with it, yay
+ if ( !$memcache_available || !defined $handler->use_memcache )
+ {
+ next;
+ }
+
+ # now let's find out what we're going to do with memcache
+ my $memcache_policy = $handler->use_memcache;
+
+ if ( $memcache_policy eq 'lite' ) {
+ # the handler loads everything from the corresponding
+ # table and uses only one memcache key to cache that
+
+ my $memkey = $handler->memcache_key($u);
+ LJ::MemCache::delete([ $u->userid, $memkey ]);
+ } elsif ( $memcache_policy eq 'blob' ) {
+ # the handler uses one memcache key for each prop,
+ # so let's delete them all
+
+ foreach my $propname (@$propnames_handled) {
+ my $memkey = $handler->memcache_key( $u, $propname );
+ LJ::MemCache::delete([ $u->userid, $memkey ]);
+ }
+ }
+ }
+
+ # now, actually reflect that we've changed the props in the
+ # user object
+ foreach my $propname (keys %$propmap) {
+ $u->{$propname} = $propmap->{$propname};
+ }
+
+ # and run the hooks, too
+ LJ::run_hooks( 'props_changed', $u, $propmap );
+
return $value;
}
@@ -5631,7 +5694,7 @@
my ($u, $url) = @_;
my $uid = $u->id;
- LJ::set_userprop($u, 'custom_usericon', $url);
+ $u->set_prop( 'custom_usericon' => $url );
}
sub _subscriptions_count {
@@ -5653,6 +5716,18 @@
return $count;
}
+sub packed_props {
+ my ($u) = @_;
+ return $u->{'packed_props'};
+}
+
+sub set_packed_props {
+ my ($u, $newprops) = @_;
+
+ LJ::update_user($u, { 'packed_props' => $newprops });
+ $u->{'packed_props'} = 1;
+}
+
sub reset_cache {
my $u = shift;
@@ -5912,8 +5987,7 @@
# des-opts: hashref of opts. set key 'cache' to use memcache.
# des-propname: the name of a property from the [dbtable[userproplist]] table.
# </LJFUNC>
-sub load_user_props
-{
+sub load_user_props {
&nodb;
my $u = shift;
@@ -5923,155 +5997,91 @@
my $opts = ref $_[0] ? shift : {};
my (@props) = @_;
- my ($sql, $sth);
- LJ::load_props("user");
+ my $extend_user_object = sub {
+ my ($propmap) = @_;
- ## user reference
- my $uid = $u->{'userid'}+0;
- $uid = LJ::get_userid($u->{'user'}) unless $uid;
+ foreach my $propname ( keys %$propmap ) {
+ $u->{$propname} = $propmap->{$propname};
+ }
+ };
- my $mem = {};
- my $use_master = 0;
- my $used_slave = 0; # set later if we ended up using a slave
+ my $groups = LJ::User::PropStorage->get_handler_multi(\@props);
+ my $memcache_available = @LJ::MEMCACHE_SERVERS;
+ my $use_master = $memcache_available || $opts->{'use_master'};
- if (@LJ::MEMCACHE_SERVERS) {
- my @keys;
- foreach (@props) {
- next if exists $u->{$_};
- my $p = LJ::get_prop("user", $_);
- die "Invalid userprop $_ passed to LJ::load_user_props." unless $p;
- push @keys, [$uid,"uprop:$uid:$p->{'id'}"];
- }
- $mem = LJ::MemCache::get_multi(@keys) || {};
- $use_master = 1;
- }
+ my $memc_expire = time() + 3600 * 24;
- $use_master = 1 if $opts->{'use_master'};
+ foreach my $handler (keys %$groups) {
+ # if there is no memcache, or if the handler doesn't wish to use
+ # memcache, hit the storage directly, update the user object,
+ # and get straight to the next handler
+ if ( !$memcache_available || !defined $handler->use_memcache )
+ {
+ my $propmap
+ = $handler->get_props( $u, $groups->{$handler},
+ { 'use_master' => $use_master } );
- my @needwrite; # [propid, propname] entries we need to save to memcache later
+ $extend_user_object->($propmap);
- my %loadfrom;
- my %multihomed; # ( $propid => 0/1 ) # 0 if we haven't loaded it, 1 if we have
- unless (@props) {
- # case 1: load all props for a given user.
- # multihomed props are stored on userprop and userproplite2, but since they
- # should always be in sync, it doesn't matter which gets loaded first, the
- # net results should be the same. see doc/server/lj.int.multihomed_userprops.html
- # for more information.
- $loadfrom{'userprop'} = 1;
- $loadfrom{'userproplite'} = 1;
- $loadfrom{'userproplite2'} = 1;
- $loadfrom{'userpropblob'} = 1;
- } else {
- # case 2: load only certain things
- foreach (@props) {
- next if exists $u->{$_};
- my $p = LJ::get_prop("user", $_);
- die "Invalid userprop $_ passed to LJ::load_user_props." unless $p;
- if (defined $mem->{"uprop:$uid:$p->{'id'}"}) {
- $u->{$_} = $mem->{"uprop:$uid:$p->{'id'}"};
- next;
- }
- push @needwrite, [ $p->{'id'}, $_ ];
- my $source = $p->{'indexed'} ? "userprop" : "userproplite";
- if ($p->{datatype} eq 'blobchar') {
- $source = "userpropblob"; # clustered blob
- }
- elsif ($p->{'cldversion'} && $u->{'dversion'} >= $p->{'cldversion'}) {
- $source = "userproplite2"; # clustered
- }
- elsif ($p->{multihomed}) {
- $multihomed{$p->{id}} = 0;
- $source = "userproplite2";
- }
- push @{$loadfrom{$source}}, $p->{'id'};
+ next;
}
- }
- foreach my $table (qw{userproplite userproplite2 userpropblob userprop}) {
- next unless exists $loadfrom{$table};
- my $db;
- if ($use_master) {
- $db = ($table =~ m{userprop(lite2|blob)}) ?
- LJ::get_cluster_master($u) :
- LJ::get_db_writer();
- }
- unless ($db) {
- $db = ($table =~ m{userprop(lite2|blob)}) ?
- LJ::get_cluster_reader($u) :
- LJ::get_db_reader();
- $used_slave = 1;
- }
- $sql = "SELECT upropid, value FROM $table WHERE userid=$uid";
- if (ref $loadfrom{$table}) {
- $sql .= " AND upropid IN (" . join(",", @{$loadfrom{$table}}) . ")";
- }
- $sth = $db->prepare($sql);
- $sth->execute;
- while (my ($id, $v) = $sth->fetchrow_array) {
- delete $multihomed{$id} if $table eq 'userproplite2';
- $u->{$LJ::CACHE_PROPID{'user'}->{$id}->{'name'}} = $v;
- }
+ # now let's find out what we're going to do with memcache
+ my $memcache_policy = $handler->use_memcache;
- # push back multihomed if necessary
- if ($table eq 'userproplite2') {
- push @{$loadfrom{userprop}}, $_ foreach keys %multihomed;
- }
- }
+ if ( $memcache_policy eq 'lite' ) {
+ # the handler loads everything from the corresponding
+ # table and uses only one memcache key to cache that
- # see if we failed to get anything above and need to hit the master.
- # this usually happens the first time a multihomed prop is hit. this
- # code will propagate that prop down to the cluster.
- if (%multihomed) {
+ my $memkey = $handler->memcache_key($u);
+ if ( my $packed = LJ::MemCache::get([ $u->userid, $memkey ]) ) {
+ my $propmap
+ = LJ::User::PropStorage->unpack_from_memcache($packed);
- # verify that we got the database handle before we try propagating data
- if ($u->writer) {
- my @values;
- foreach my $id (keys %multihomed) {
- my $pname = $LJ::CACHE_PROPID{user}{$id}{name};
- if (defined $u->{$pname} && $u->{$pname}) {
- push @values, "($uid, $id, " . $u->quote($u->{$pname}) . ")";
- } else {
- push @values, "($uid, $id, '')";
- }
+ $extend_user_object->($propmap);
+
+ # we've loaded everything handled by this handler,
+ # let's get to the next one
+ next;
}
- $u->do("REPLACE INTO userproplite2 VALUES " . join ',', @values);
- }
- }
- # Add defaults to user object.
+ # actually load everything from DB, cache it, and update
+ # the user object
+ my $propmap
+ = $handler->get_props( $u, [],
+ { 'use_master' => $use_master } );
- # defaults for S1 style IDs in config file are magic: really
- # uniq strings representing style IDs, so on first use, we need
- # to map them
- unless ($LJ::CACHED_S1IDMAP) {
+ my $packed = LJ::User::PropStorage->pack_for_memcache($propmap);
+ LJ::MemCache::set( [$u->userid, $memkey],
+ $packed, $memc_expire );
- my $pubsty = LJ::S1::get_public_styles();
- foreach (values %$pubsty) {
- my $k = "s1_$_->{'type'}_style";
- next unless $LJ::USERPROP_DEF{$k} eq "$_->{'type'}/$_->{'styledes'}";
+ $extend_user_object->($propmap);
+ } elsif ( $memcache_policy eq 'blob' ) {
+ # the handler uses one memcache key for each prop
+ # hit memcache for them first, then hit db and update
+ # memcache
- $LJ::USERPROP_DEF{$k} = $_->{'styleid'};
- }
+ my $handled_props = $groups->{$handler};
- $LJ::CACHED_S1IDMAP = 1;
- }
+ my $propmap_memc
+ = $handler->fetch_props_memcache( $u, $handled_props );
- # If this was called with no @props, then the function tried
- # to load all metadata. but we don't know what's missing, so
- # try to apply all defaults.
- unless (@props) { @props = keys %LJ::USERPROP_DEF; }
+ $extend_user_object->($propmap_memc);
- foreach my $prop (@props) {
- next if (defined $u->{$prop});
- $u->{$prop} = $LJ::USERPROP_DEF{$prop};
- }
+ my @load_from_db = grep { !exists $propmap_memc->{$_} }
+ @$handled_props;
- unless ($used_slave) {
- my $expire = time() + 3600*24;
- foreach my $wr (@needwrite) {
- my ($id, $name) = ($wr->[0], $wr->[1]);
- LJ::MemCache::set([$uid,"uprop:$uid:$id"], $u->{$name} || "", $expire);
+ # if we can avoid hitting the db, avoid it
+ next unless @load_from_db;
+
+ my $propmap_db
+ = $handler->get_props( $u, \@load_from_db,
+ { 'use_master' => $use_master } );
+
+ $extend_user_object->($propmap_db);
+
+ # now, update memcache
+ $handler->store_props_memcache( $u, $propmap_db );
}
}
}
@@ -6850,117 +6860,6 @@
}
# <LJFUNC>
-# name: LJ::set_userprop
-# des: Sets/deletes a userprop by name for a user.
-# info: This adds or deletes from the
-# [dbtable[userprop]]/[dbtable[userproplite]] tables. One
-# crappy thing about this interface is that it doesn't allow
-# a batch of userprops to be updated at once, which is the
-# common thing to do.
-# args: dbarg?, uuserid, propname, value, memonly?
-# des-uuserid: The userid of the user or a user hashref.
-# des-propname: The name of the property. Or a hashref of propname keys and corresponding values.
-# des-value: The value to set to the property. If undefined or the
-# empty string, then property is deleted.
-# des-memonly: if true, only writes to memcache, and not to database.
-# </LJFUNC>
-sub set_userprop
-{
- &nodb;
- my ($u, $propname, $value, $memonly) = @_;
- $u = ref $u ? $u : LJ::load_userid($u);
- my $userid = $u->{'userid'}+0;
-
- my $hash = ref $propname eq "HASH" ? $propname : { $propname => $value };
-
- my %action; # $table -> {"replace"|"delete"} -> [ "($userid, $propid, $qvalue)" | propid ]
- my %multihomed; # { $propid => $value }
-
- foreach $propname (keys %$hash) {
- LJ::run_hook("setprop", prop => $propname,
- u => $u, value => $value);
-
- my $p = LJ::get_prop("user", $propname) or
- die "Invalid userprop $propname passed to LJ::set_userprop.";
- if ($p->{multihomed}) {
- # collect into array for later handling
- $multihomed{$p->{id}} = $hash->{$propname};
- next;
- }
- my $table = $p->{'indexed'} ? "userprop" : "userproplite";
- if ($p->{datatype} eq 'blobchar') {
- $table = 'userpropblob';
- }
- elsif ($p->{'cldversion'} && $u->{'dversion'} >= $p->{'cldversion'}) {
- $table = "userproplite2";
- }
- unless ($memonly) {
- my $db = $action{$table}->{'db'} ||= (
- $table !~ m{userprop(lite2|blob)}
- ? LJ::get_db_writer()
- : $u->writer );
- return 0 unless $db;
- }
- $value = $hash->{$propname};
- if (defined $value && $value) {
- push @{$action{$table}->{"replace"}}, [ $p->{'id'}, $value ];
- } else {
- push @{$action{$table}->{"delete"}}, $p->{'id'};
- }
- }
-
- my $expire = time() + 3600*24;
- foreach my $table (keys %action) {
- my $db = $action{$table}->{'db'};
- if (my $list = $action{$table}->{"replace"}) {
- if ($db) {
- my $vals = join(',', map { "($userid,$_->[0]," . $db->quote($_->[1]) . ")" } @$list);
- $db->do("REPLACE INTO $table (userid, upropid, value) VALUES $vals");
- }
- LJ::MemCache::set([$userid,"uprop:$userid:$_->[0]"], $_->[1], $expire) foreach (@$list);
- }
- if (my $list = $action{$table}->{"delete"}) {
- if ($db) {
- my $in = join(',', @$list);
- $db->do("DELETE FROM $table WHERE userid=$userid AND upropid IN ($in)");
- }
- LJ::MemCache::set([$userid,"uprop:$userid:$_"], "", $expire) foreach (@$list);
- }
- }
-
- # if we had any multihomed props, set them here
- if (%multihomed) {
- my $dbh = LJ::get_db_writer();
- return 0 unless $dbh && $u->writer;
- while (my ($propid, $pvalue) = each %multihomed) {
- if (defined $pvalue && $pvalue) {
- # replace data into master
- $dbh->do("REPLACE INTO userprop VALUES (?, ?, ?)",
- undef, $userid, $propid, $pvalue);
- } else {
- # delete data from master, but keep in cluster
- $dbh->do("DELETE FROM userprop WHERE userid = ? AND upropid = ?",
- undef, $userid, $propid);
- }
-
- # fail out?
- return 0 if $dbh->err;
-
- # put data in cluster
- $pvalue ||= '';
- $u->do("REPLACE INTO userproplite2 VALUES (?, ?, ?)",
- undef, $userid, $propid, $pvalue);
- return 0 if $u->err;
-
- # set memcache
- LJ::MemCache::set([$userid,"uprop:$userid:$propid"], $pvalue, $expire);
- }
- }
-
- return 1;
-}
-
-# <LJFUNC>
# name: LJ::get_shared_journals
# des: Gets an array of shared journals a user has access to.
# returns: An array of shared journals.
Modified: trunk/cgi-bin/ljdb.pl
===================================================================
--- trunk/cgi-bin/ljdb.pl 2010-11-29 04:54:45 UTC (rev 17809)
+++ trunk/cgi-bin/ljdb.pl 2010-11-29 07:47:35 UTC (rev 17810)
@@ -403,6 +403,8 @@
sub get_dbirole_dbh {
my $dbh = $LJ::DBIRole->get_dbh( @_ ) or return undef;
+ $dbh->{'RaiseError'} = 1 if $LJ::IS_DEV_SERVER;
+
if ( $LJ::DB_LOG_HOST && $LJ::HAVE_DBI_PROFILE ) {
$LJ::DB_REPORT_HANDLES{ $dbh->{Name} } = $dbh;
