Committer: ailyin
LJSV-28 (scrapbook doesn't follow renames properly)A trunk/bin/test/ A trunk/bin/test/renames.pl U trunk/bin/upgrading/base-data.sql A trunk/bin/upgrading/convert-username-map.pl U trunk/lib/Apache/FotoBilder.pm U trunk/lib/FB/User.pm U trunk/lib/FotoBilder/Auth/LiveJournal.pm
Added: trunk/bin/test/renames.pl =================================================================== --- trunk/bin/test/renames.pl (rev 0) +++ trunk/bin/test/renames.pl 2010-11-10 11:38:10 UTC (rev 1435) @@ -0,0 +1,104 @@ +#!/usr/bin/perl +use strict; +use warnings; + +use lib "$ENV{'LJHOME'}/cgi-bin"; +use lib "$ENV{'FBHOME'}/lib"; +require 'ljlib.pl'; +require 'fotobilder.pl'; + +no warnings 'once'; + +$FB::AUTH_DOMAIN{1}->[1]->{'ljhost'} = $LJ::DOMAIN; + +my $verbose = 1; + +sub random_username { + my ($prefix) = @_; + + my @chars = split //, 'abcdefghijklmnopqrstuvwxyz'; + my $chars_count = scalar(@chars); + + my ($username, $u); + + do { + $username = $prefix; + + foreach (1..(15-length($prefix))) { + $username .= $chars[rand($chars_count)] ; + } + $u = LJ::load_user($username); + } while ($u); + + return $username; +} + +sub create_user { + my ($prefix, $noexpand) = @_; + + my ($username, $u); + + if ($noexpand) { + $username = $prefix; + } else { + $username = random_username($prefix); + } + + $u = LJ::User->create_personal( + 'user' => $username, + 'bdate' => '1980-01-01', + 'email' => 'jemmix+esntest@gmail.com', # this email doesn't actually work :p + 'password' => 'test', + 'get_ljnews' => 0, + 'underage' => 0, + 'ofage' => 1, + ); + + LJ::update_user($u, { 'status' => 'A' }); + $u->add_to_class('paid'); + $u->remove_from_class('plus'); + $u->set_password('test'); + + warn "created user " . $u->user if $verbose; + + get_fbu($u); + + return $u; +} + +sub rename_user { + my ($u, $newname, $nopreserve) = @_; + + my $u2 = LJ::load_user($newname); + if ($u2) { + my $u2newname = 'ex_' . substr($newname, 0, 10); + rename_user($u2, $u2newname, 1); + } + + my $oldname = $u->username; + + LJ::User::Rename::basic_rename( $oldname, $newname, + { 'nonotify' => 1, + 'token' => '[test]', + 'preserve_old_username' => !$nopreserve, + 'opt_redir' => 1, } ); + + warn "renamed $oldname => $newname" if $verbose; + + # it's weird that basic_rename doesn't seem to do that, but whatever + $u->{'user'} = $newname; +} + +sub get_fbu { + my ($u) = @_; + + my $username = LJ::isu($u) ? $u->username : $u; + return FB::load_user($username, 1, { 'create' => 1 }); +} + +my $ua = create_user('a_'); +my $ub = create_user('b_'); +my $uc = create_user('c_'); + +rename_user($ua, $ub->username); +rename_user($ub, $uc->username); Property changes on: trunk/bin/test/renames.pl ___________________________________________________________________ Added: svn:executable + * Modified: trunk/bin/upgrading/base-data.sql =================================================================== --- trunk/bin/upgrading/base-data.sql 2010-11-08 06:00:36 UTC (rev 1434) +++ trunk/bin/upgrading/base-data.sql 2010-11-10 11:38:10 UTC (rev 1435) @@ -22,3 +22,5 @@ UPDATE proplist SET des='upicid of gallery preview image',scope='general' WHERE propname='previewpicid'; INSERT IGNORE INTO proplist (propname, scope, des) VALUES ('styleid', 'general', 'S2 styleid to use'); UPDATE proplist SET des='S2 styleid to use',scope='general' WHERE propname='styleid'; +INSERT IGNORE INTO proplist (propname, scope, des) VALUES ('renamedto', 'general', 'A username to which this account was renamed, in case of statusvis="R"'); +UPDATE proplist SET des='A username to which this account was renamed, in case of statusvis="R"',scope='general' WHERE propname='renamedto'; Added: trunk/bin/upgrading/convert-username-map.pl =================================================================== --- trunk/bin/upgrading/convert-username-map.pl (rev 0) +++ trunk/bin/upgrading/convert-username-map.pl 2010-11-10 11:38:10 UTC (rev 1435) @@ -0,0 +1,150 @@ +#!/usr/bin/perl +use strict; +use warnings; + +use lib "$ENV{'LJHOME'}/cgi-bin"; +use lib "$ENV{'FBHOME'}/lib"; +require 'ljlib.pl'; +require 'fotobilder.pl'; + +no warnings 'once'; + +use Getopt::Long; + +Getopt::Long::Configure("bundling"); + +my $fix = 0; +my $verbose = 0; +my $help = 0; + +GetOptions( 'fix|f' => \$fix, + 'verbose|v' => \$verbose, + 'help|h|usage|u' => \$help ); + +my $dry = !$fix; + +if ($help) { + print "$0: find and (possibly) fix the users that do not have no \n"; + print "LJ userid <-> FB userid mapping in the useridlookup table. \n"; + print " \n"; + print "Options: \n"; + print " --fix, -f run to actually fix the table; the \n"; + print " script runs in the 'dry' mode \n"; + print " otherwise, without actually making \n"; + print " changes \n"; + print " \n"; + print " --verbose, -v verbose output, with information about \n"; + print " the progress and the actual (or \n"; + print " pretended) changes \n"; + print " \n"; + print " --help, -h, print this help message and then exit \n"; + print " --usage, -u immediately \n"; + exit; +} + +my $last_username = ''; +my $last_handled = 1; + +my $db = FB::get_db_writer(); +$db->{'RaiseError'} = 1; + +my $total_usernames = 0; +my $errors = 0; + +my @usernames; +my @error_usernames; + +while ($last_handled) { + my $res = $db->selectall_arrayref(qq{ + SELECT userid, kval + FROM useridlookup + WHERE kval > ? AND ktype='N' AND domainid=1 + ORDER BY kval + LIMIT 1000 + }, { 'Slice' => {} }, $last_username); + + last unless @$res; + + my %map; + + foreach my $row (@$res) { + $map{$row->{'userid'}} = $row->{'kval'}; + $last_username = $row->{'kval'}; + } + warn "$last_username\n" if $verbose; + $last_handled = scalar(@$res); + + my $in = join(',', keys %map); + my $userids_mapped = $db->selectcol_arrayref(qq{ + SELECT userid + FROM useridlookup + WHERE ktype='I' AND userid IN ($in) AND domainid=1 + }); + my %userids_mapped = map { $_ => 1 } @$userids_mapped; + + my @userids_unmapped = grep { !exists $userids_mapped{$_} } keys %map; + + $total_usernames += scalar(@userids_unmapped); + + foreach my $userid (@userids_unmapped) { + my $username = $map{$userid}; + + # we're communicating a lot with the fb database but seldom with + # the LJ's one, so the latter one may have disconnected, let's + # reset it + LJ::start_request(); + my $u = LJ::load_user($username); + + unless ($u) { + warn "no LJ user exists for $username, db corrupt?\n"; + $errors++; + push @error_usernames, $username; + next; + } + + if ($u->is_renamed) { + warn "$username is renamed, so it'll be fixed the next time " . + "it is accessed by web\n"; + } + + if ($u->is_expunged) { + warn "$username is expunged, whatever, skipping\n"; + } + + push @usernames, $username; + my $ljuid = $u->userid; + + if ($verbose) { + if ($dry) { + warn "would add a mapping between fbuid=$userid, " . + "username=$username and ljuid=$ljuid\n"; + } else { + warn "adding a mapping between fbuid=$userid, " . + "username=$username and ljuid=$ljuid\n"; + } + } + + next if $dry; + + $db->do(qq{ + INSERT INTO useridlookup + SET userid=?, ktype='I', domainid=1, kval=? + }, undef, $userid, $ljuid); + } +} + +if ($dry) { + warn "$total_usernames total users found, $errors errors\n"; +} else { + warn "$total_usernames total users fixed, $errors errors\n"; +} + +if ($dry) { + warn "DONE\n\n"; + + warn "users with errors:\n"; + warn "$_\n" foreach @error_usernames; + + warn "users to fix:\n"; + warn "$_\n" foreach @usernames; +} Property changes on: trunk/bin/upgrading/convert-username-map.pl ___________________________________________________________________ Added: svn:executable + * Modified: trunk/lib/Apache/FotoBilder.pm =================================================================== --- trunk/lib/Apache/FotoBilder.pm 2010-11-08 06:00:36 UTC (rev 1434) +++ trunk/lib/Apache/FotoBilder.pm 2010-11-10 11:38:10 UTC (rev 1435) @@ -79,6 +79,42 @@ LJ::Request->header_in('Host', $_); } + # reload libraries that might've changed + if ($LJ::IS_DEV_SERVER && !$LJ::DISABLED{'module_reload'}) { + my %to_reload; + while (my ($file, $mod) = each %LJ::LIB_MOD_TIME) { + my $cur_mod = (stat($file))[9]; + next if $cur_mod == $mod; + $to_reload{$file} = 1; + } + my @key_del; + foreach (my ($key, $file) = each %INC) { + push @key_del, $key if $to_reload{$file}; + } + delete $INC{$_} foreach @key_del; + + foreach my $file (keys %to_reload) { + print STDERR "Reloading $file...\n"; + LJ::clear_hooks($file); + + my %reloaded; + local $SIG{__WARN__} = sub { + if ($_[0] =~ m/^Subroutine (\S+) redefined at /) + { + warn @_ if ($reloaded{$1}++); + } else { + warn(@_); + } + }; + my $good = do $file; + if ($good) { + $LJ::LIB_MOD_TIME{$file} = (stat($file))[9]; + } else { + die "Failed to reload module [$file] (ret=$good) due to error: \$\!=$!, \$\@=$@"; + } + } + } + return LJ::Request::OK; } @@ -256,6 +292,14 @@ # probably for user: if ($topdir || $single_user) { + my $u = FB::load_user($topdir); + return LJ::Request::NOT_FOUND unless $u; + + if ($u->is_renamed) { + my $new_uri = $FB::SITEROOT . '/' . $u->prop('renamedto') . $rest; + return redir($new_uri); + } + if ($rest =~ m!^/pic/.+\.xml$!) { # picture XMLInfo page LJ::Request->handler('perl-script'); Modified: trunk/lib/FB/User.pm =================================================================== --- trunk/lib/FB/User.pm 2010-11-08 06:00:36 UTC (rev 1434) +++ trunk/lib/FB/User.pm 2010-11-10 11:38:10 UTC (rev 1435) @@ -940,6 +940,25 @@ return $userid; } +sub lookup_userid { + my ($class, $domainid, $ktype, $kval) = @_; + + my $db = @FB::MEMCACHE_SERVERS ? FB::get_db_writer() : FB::get_db_reader(); + + # don't use placeholders! If username is numeric, DBD::mysql + # won't quote it, and mysql won't case an integer to a varchar, + # and thus the query optimizer won't use all parts of the key. + # Lame. (last checked and valid as of Oct 2010) + my $qkval = $db->quote($kval); + + my ($uid) = $db->selectrow_array(qq{ + SELECT userid FROM useridlookup + WHERE domainid=? AND ktype=? AND kval=$qkval + }, undef, $domainid, $ktype); + + return $uid; +} + # class method # look up the userid for a user # args: $username, [$domainid], [$opts] @@ -979,26 +998,18 @@ return $id; }; - my $db = @FB::MEMCACHE_SERVERS ? FB::get_db_writer() : FB::get_db_reader(); - - # don't use placeholders! If username is numeric, DBD::mysql - # won't quote it, and mysql won't case an integer to a varchar, - # and thus the query optimizer won't use all parts of the key. - # Lame. (Nov 22nd, 2004 -- MySQL 4.0.20, DBD::mysql 1.2216-2) - my $quser = $db->quote($user); - - $uid = $db->selectrow_array("SELECT userid FROM useridlookup " . - "WHERE domainid=? AND ktype='N' AND kval=$quser", - undef, $domainid); - return $set->($uid) if $uid; - # virt install, create new account and populate disk usage. my $authmod = FB::domain_plugin($domainid); - if ($authmod && $opts->{'create'}) { - my $u = $authmod->load_user({ user => $user, vivify => 1 }); - return $set->($u->{userid}) if $u; + + unless ($authmod) { + $uid = $class->lookup_userid($domainid, 'N', $user); + return $set->($uid) if $uid; + return undef; } + my $u = $authmod->load_user({ user => $user, vivify => 1 }); + return $set->($u->{userid}) if $u; + # no hope now return undef; } @@ -1108,6 +1119,10 @@ return $u->get_prop('is_identity') ? 1 : 0; } +sub is_renamed { + my ($u) = @_; + return $u->{'statusvis'} eq 'R'; +} # a little place to stash persistant user info sub cache { @@ -1243,7 +1258,11 @@ my $u = $uid ? FB::load_userid($uid) : undef; # get authmod if we'll need it soon - my $authmod = ($opts->{'create'} || $opts->{'validate'}) ? FB::current_domain_plugin() : undef; + my $authmod; + if ( $opts->{'create'} || $opts->{'validate'} ) { + $authmod = $FB::DOMAINID_PLUGIN{$domainid} + || FB::current_domain_plugin(); + } # virt install, create new account and populate disk usage. if ($authmod && ! $u && $opts->{create}) { @@ -1326,7 +1345,7 @@ # don't use placeholders! If username is numeric, DBD::mysql # won't quote it, and mysql won't case an integer to a varchar, # and thus the query optimizer won't use all parts of the key. - # Lame. (Nov 22nd, 2004 -- MySQL 4.0.20, DBD::mysql 1.2216-2) + # Lame. (last checked and valid as of Oct 2010) my $qkval = $db->quote($duserid); $uid = $db->selectrow_array("SELECT userid FROM useridlookup " . Modified: trunk/lib/FotoBilder/Auth/LiveJournal.pm =================================================================== --- trunk/lib/FotoBilder/Auth/LiveJournal.pm 2010-11-08 06:00:36 UTC (rev 1434) +++ trunk/lib/FotoBilder/Auth/LiveJournal.pm 2010-11-10 11:38:10 UTC (rev 1435) @@ -56,7 +56,10 @@ # if the ljhost we're about to contact is localhost, then see if LiveJournal is already # living in the same process as us, if so we can skip the overhead of exchanging HTTP # traffic with ourselves - if ($self->{ljhost} eq 'localhost' || $self->{ljhost} eq '127.0.0.1') { + my $ljhost = $self->{ljhost}; + my $module_loaded = $INC{'Apache/LiveJournal/Interface/FotoBilder.pm'}; + if ( ($ljhost eq 'localhost' || $ljhost eq '127.0.0.1') && $module_loaded ) + { # note that the following is a LiveJournal function, NOT FotoBilder my $res = eval { Apache::LiveJournal::Interface::FotoBilder::run_method($mode, $req) }; if (ref $res eq 'HASH') { @@ -95,6 +98,13 @@ return ("lj_lastsync"); } +sub needs_sync { + my ($self, $u) = @_; + + my $lastsync = $u->prop('lj_lastsync'); + return !$lastsync || time - $lastsync > 3600; +} + # requires that props returned by validate_props already be loaded into $u # - gets called by pic.pm on every image load # - every 12 hours, we force a load_user to sync their @@ -107,7 +117,7 @@ # force a check if we've never done it or if it's been over 12 hours my $now = time(); - return 1 if $u->{lj_lastsync} && $now - $u->{lj_lastsync} < 3600*12; + return 1 unless $self->needs_sync($u); # validate all user's info by loading them from remote lj with fullsync $u = $self->load_user({ user => $u->{user}, fullsync => 1 }); @@ -258,6 +268,65 @@ return $u; } +sub lookup_domain_userid { + my ($self, $u) = @_; + + my $db = FB::get_db_writer(); + my ($userid) = $db->selectrow_array(qq{ + SELECT kval FROM useridlookup + WHERE userid=? AND domainid=? AND ktype='I' + }, undef, $u->{'userid'}, $self->{'domainid'}); + + return $userid; +} + +sub rename_fix { + my ($self, $u, $newusername) = @_; + + # is there a local user with the new username? if so, we will need to + # fix them first, recursively + my $newfbuid + = FB::User->lookup_userid($self->{'domainid'}, 'N', $newusername); + + if (defined $newfbuid) { + my $other_u = FB::load_userid($newfbuid); + my $other_domain_userid = $self->lookup_domain_userid($other_u); + my %rpc_res = $self->_do_rpc( "get_user_info", + { 'uid' => $other_domain_userid } ); + my $other_newusername = $rpc_res{'user'}; + + $self->rename_fix($other_u, $other_newusername); + } + + my $db = FB::get_db_writer(); + + $db->do(qq{ + DELETE FROM useridlookup + WHERE domainid=? AND ktype="N" AND userid=? + }, undef, $self->{'domainid'}, $u->{'userid'}); + + $db->do(qq{ + REPLACE INTO useridlookup + SET domainid=?, ktype="N", userid=?, kval=? + }, undef, $self->{'domainid'}, $u->{'userid'}, $newusername); + + FB::update_user($u, { 'usercs' => $newusername }); + $u->{'usercs'} = $newusername; + FB::add_u_user($u); + + # FIXME: need an api for setting useridlookup rows, these memcache keys + # will get confusing + + # delete useridlookup memcache keys for both old and new usernames + # old username => loc userid mapping: + FB::MemCache::delete([ $u->{user}, + "loc_uid_n:$self->{domainid}:$u->{user}" ]); + + # new username => loc userid mapping: + FB::MemCache::delete([ $newusername, + "loc_uid_n:$self->{domainid}:$newusername" ]); +} + # given a domain userid and domain username, return the associated local user, # creating one if necessary sub load_user @@ -265,6 +334,23 @@ my ($self, $opts) = @_; # domain userid, username return undef unless $self && ref $opts eq 'HASH'; + my $fbuid; + + if ($opts->{'uid'}) { + $fbuid = + FB::User->lookup_userid( $self->{'domainid'}, + 'I', $opts->{'uid'} ); + } + + if ($opts->{'user'}) { + $fbuid + = FB::User->lookup_userid( $self->{'domainid'}, + 'N', $opts->{'user'} ); + } + + my $u = FB::load_userid($fbuid); + return $u if $u && !$self->needs_sync($u); + # use passed res if we got one my %res = ref $opts->{res} ? %{$opts->{res}} : (); @@ -285,6 +371,7 @@ %res = $self->_do_rpc("get_user_info", \%req, @head); print STDERR "res=" , join('/', %res), "\n" if $FB::DEBUG; } + return undef unless $res{'user'} && $res{'userid'}; my $lj_user = $res{'user'}; @@ -292,96 +379,35 @@ my $lj_statusvis = $res{'statusvis'}; # first try loading by LJ userid - my $u = FB::load_user_domain_userid($self->{'domainid'}, $lj_userid); + $u = FB::load_user_domain_userid($self->{'domainid'}, $lj_userid); - my $db = FB::get_db_writer(); - # see if user renamed on LJ, update mappings here if ($u && $u->{'usercs'} ne $lj_user) { - - # 2 cases here: - # 1) there's been a simple rename, so we update useridlookup mappings and usercs for $u - # 2) something more complicated. we've got a new username but that already exists in - # useridlookup for a different userid on this domain. this is probably a swap, so - # we'll assume it is and switch the two's usercs and useridlookup rows - - # if case 2, move $u's info to $exist_u's before renaming $u to $lj_user - if (my $exist_u = FB::load_user($lj_user)) { - - # update usercs for $exist_u and useridlookup - FB::update_user($exist_u, { 'usercs' => $u->{'usercs'} }); - - # update user/usercs in memory for later in this request - # -- probably not necessary but definitely possible - $exist_u->{'usercs'} = $u->{'usercs'}; - FB::add_u_user($exist_u); - - # update useridlookup mapping - $db->do("REPLACE INTO useridlookup (domainid, ktype, kval, userid) VALUES (?,?,?,?)", - undef, $self->{'domainid'}, 'N', $exist_u->{'user'}, $exist_u->{'userid'}); - die $db->errstr if $db->err; - - # don't need to delete from memcache here because $lj_user's memcache row will - # be removed below - } - - # update to new username - 'N' mapping - FB::update_user($u, { 'usercs' => $lj_user }); - - # update user/usercs in memory for later in this request - $u->{'usercs'} = $lj_user; - FB::add_u_user($u); - - # update useridlookup mapping - $db->do("REPLACE INTO useridlookup (domainid, ktype, kval, userid) VALUES (?,?,?,?)", - undef, $self->{'domainid'}, 'N', $u->{'user'}, $u->{'userid'}); - die $db->errstr if $db->err; - - # FIXME: need an api for setting useridlookup rows, these memcache keys will get confusing - # delete useridlookup memcache keys for both old and new usernames - FB::MemCache::delete([$u->{user}, "loc_uid_n:$self->{domainid}:$u->{user}" ]); # old username => loc userid mapping - FB::MemCache::delete([$lj_user, "loc_uid_n:$self->{domainid}:$lj_user" ]); # new username => loc userid mapping + $self->rename_fix($u, $lj_user); } # if remote statusvis has changed, change locally as well - if ($u && $u->{statusvis} ne $lj_statusvis && $lj_statusvis =~ /[VXDS]/) { + if ($u && $u->{statusvis} ne $lj_statusvis && $lj_statusvis =~ /[VXDSR]/) { FB::update_user($u, { 'statusvis' => $lj_statusvis, 'raw' => 'statusvisdate=UNIX_TIMESTAMP()' }); $u->{statusvis} = $lj_statusvis; # update for this request } - # now try loading from username (old LJ integration worked this way) - unless ($u) { - $u = FB::load_user($lj_user, $self->{'domainid'}); - if ($u) { - # we found one by name. but is this really ours, or a remnant - # of an old user who had this name, renamed away, and hasn't - # logged back into fotobilder for the change to take effect? - my $uid = $u->{'userid'}; - if ($u->check_remoteuserid_uniqueness($self->{'domainid'}, $lj_userid)) { - # it's safe to use the $u we found, and let's give it a useridlookup - # mapping while we're here. - $db->do("REPLACE INTO useridlookup (domainid, ktype, kval, userid) " . - "VALUES (?,?,?,?)", undef, $self->{'domainid'}, 'I', $lj_userid, $uid); - FB::MemCache::delete([$lj_userid, "loc_uid_i:$self->{domainid}:$lj_userid"]); - FB::MemCache::delete([$uid, "dom_uid_i:$self->{domainid}:$uid" ]); - } else { - $u = undef; - } - } - } - # vivify_user if we don't have a $u yet - if (! $u && $opts->{'vivify'}) { - + if (! $u && ($opts->{'vivify'} || $lj_statusvis eq 'R')) { # don't create new accounts unless LJ signifies # they are allowed to have an fb account return undef unless $res{fb_account}; $u = $self->vivify_user($lj_user, $lj_userid); + + FB::update_user($u, { 'statusvis' => $lj_statusvis }); } return undef unless $u; + $u->set_prop( 'renamedto' => $res{'renamedto'} ) + if exists $res{'renamedto'}; + # update domain-specific caps in passed $u ref $self->update_dcaps($u, { 'res' => \%res });