Андрей (andy) wrote in changelog,
Андрей
andy
changelog

[ljcom] r9512: LJSUP-6780 (Twitter Digest): documentati...

Committer: ailyin
LJSUP-6780 (Twitter Digest): documentation, rate limits, fix a lot of stuff
U   trunk/bin/upgrading/proplists-local.dat
U   trunk/bin/worker/twitter-digest
U   trunk/cgi-bin/LJ/Client/Twitter/Tweet.pm
U   trunk/cgi-bin/LJ/Client/Twitter/User.pm
U   trunk/cgi-bin/LJ/Client/Twitter.pm
U   trunk/cgi-bin/LJ/Hooks/TwitterDigest.pm
U   trunk/cgi-bin/LJ/TwitterDigest.pm
Modified: trunk/bin/upgrading/proplists-local.dat
===================================================================
--- trunk/bin/upgrading/proplists-local.dat	2010-09-17 11:06:01 UTC (rev 9511)
+++ trunk/bin/upgrading/proplists-local.dat	2010-09-17 11:11:37 UTC (rev 9512)
@@ -492,3 +492,9 @@
 ratelist.invitefriend:
   des: Logged when a user sends a friend invite
 
+ratelist.twitter_api_request:
+  des: Logged for the 'system' user every time a site does a Twitter API request
+
+ratelist.twitter_digest:
+  des: Logged for the 'system' user every time the Twitter Digest system posts a digest to some journal
+

Modified: trunk/bin/worker/twitter-digest
===================================================================
--- trunk/bin/worker/twitter-digest	2010-09-17 11:06:01 UTC (rev 9511)
+++ trunk/bin/worker/twitter-digest	2010-09-17 11:11:37 UTC (rev 9512)
@@ -2,6 +2,8 @@
 use strict;
 use warnings;
 
+# see LJ::TwitterDigest for documentation
+
 use lib "$ENV{'LJHOME'}/cgi-bin";
 require 'ljlib.pl';
 

Modified: trunk/cgi-bin/LJ/Client/Twitter/Tweet.pm
===================================================================
--- trunk/cgi-bin/LJ/Client/Twitter/Tweet.pm	2010-09-17 11:06:01 UTC (rev 9511)
+++ trunk/cgi-bin/LJ/Client/Twitter/Tweet.pm	2010-09-17 11:11:37 UTC (rev 9512)
@@ -1,3 +1,78 @@
+=head1 NAME
+
+LJ::Client::Twitter::Tweet - an object interface to a Twitter tweet,
+returned by the Twitter API.
+
+=head1 SYNOPSIS
+
+ # get something from the API
+ my $tweets = LJ::Client::Twitter->call(
+     'api_method' => 'statuses/user_timeline',
+     'user' => $u,
+     'http_method' => 'GET',
+     'params' => { 'count'       => $LJ::TWITTER_RECENT_FEED_DEPTH,
+                   'include_rts' => 1, },
+ );
+ 
+ # construct an object (does some data conversions, because its internal
+ # format is so much more convenient to work with)
+ my $tw = LJ::Client::Twitter::Tweet->from_hash($tweets->[0]);
+ 
+ # use getters to retrieve data back
+ print $tw->text_formatted;
+
+=head1 METHODS
+
+=over 2
+
+=item *
+
+from_hash: a constructor; gets a hashref, presumably retrieved from the
+API, converts it and blesses.
+
+=item *
+
+$tw->post_time: the posting time of the tweet, a UNIX timestamp.
+
+=item *
+
+$tw->text_raw: the raw text of the tweet, as returned by Twitter.
+
+=item *
+
+$tw->text_formatted: the text of the tweet, with HTML tags linkifying
+https?:// and www. links, Twitter #hashtags, and Twitter @mentions.
+
+=item *
+
+$tw->user: the user who posted the tweet, an LJ::Client::Twitter::User.
+
+=item *
+
+$tw->id: the numeric identifier of this tweet, as returned by Twitter.
+
+=item *
+
+$tw->original_tweet: in case the tweet was "retweeted" from another
+Twitter, an LJ::Client::Twitter::Tweet object for the other tweet.
+Otherwise, $tw itself.
+
+=item *
+
+$tw->url: the URL to permalink to this tweet.
+
+=back
+
+=head1 SEE ALSO
+
+LJ::Client::Twitter
+
+=head1 AUTHOR
+
+Andrew Ilyin <andrey.ilyin@sup.com>
+
+=cut
+
 package LJ::Client::Twitter::Tweet;
 use strict;
 use warnings;

Modified: trunk/cgi-bin/LJ/Client/Twitter/User.pm
===================================================================
--- trunk/cgi-bin/LJ/Client/Twitter/User.pm	2010-09-17 11:06:01 UTC (rev 9511)
+++ trunk/cgi-bin/LJ/Client/Twitter/User.pm	2010-09-17 11:11:37 UTC (rev 9512)
@@ -1,3 +1,103 @@
+=head1 NAME
+
+LJ::Client::Twitter::User - an object interface to a Twitter user,
+returned by the Twitter API.
+
+=head1 SYNOPSIS
+
+ # get something from the API
+ my $tweets = LJ::Client::Twitter->call(
+     'api_method' => 'statuses/user_timeline',
+     'user' => $u,
+     'http_method' => 'GET',
+     'params' => { 'count'       => $LJ::TWITTER_RECENT_FEED_DEPTH,
+                   'include_rts' => 1, },
+ );
+ 
+ # construct an LJ::Client::Twitter::Tweet object
+ my $tw = LJ::Client::Twitter::Tweet->from_hash($tweets->[0]);
+ 
+ # and then get a user from it
+ my $twu = $tw->user;
+ 
+ # use getters to retrieve data back
+ print $twu->url;
+
+=head1 METHODS
+
+=over 2
+
+=item *
+
+from_hash: a constructor; gets a hashref, presumably retrieved from the
+API, converts it and blesses.
+
+=item *
+
+from_screen_name: another constructor; returns an object consisting of
+a screen name alone.
+
+=item *
+
+$twu->id: the numeric identifier of this user, as returned by Twitter.
+
+=item *
+
+$twu->url: the URL to permalink to this user's twueets.
+
+=item *
+
+$twu->screen_name: the Twitter screen name of this user.
+
+=item *
+
+$twu->url: the URL to permalink to this user's tweets.
+
+=item *
+
+$twu->homepage: the URL of this user's "homepage", or "website", if
+specified on Twitter.
+
+=item *
+
+$twu->lang: the language the user views Twitter in; format of this
+has not yet been investigated.
+
+=item *
+
+$twu->location: the location of the user specified on Twitter; format of
+this has not yet been investigated.
+
+=item *
+
+$twu->name: the user-specified name on Twitter.
+
+=item *
+
+$twu->bio: the user-specified "bio", or "description" on Twitter.
+
+=item *
+
+$twu->protected: a boolean value indicating that the user has chosen
+to protect their tweets, only showing them to trusted friends.
+
+=item *
+
+$twu->time_zone: the user-specified time zone on Twitter; format of
+this has not yet been investigated.
+
+=back
+
+=head1 SEE ALSO
+
+LJ::Client::Twitter
+
+=head1 AUTHOR
+
+Andrew Ilyin <andrey.ilyin@sup.com>
+
+=cut
+
 package LJ::Client::Twitter::User;
 use strict;
 use warnings;

Modified: trunk/cgi-bin/LJ/Client/Twitter.pm
===================================================================
--- trunk/cgi-bin/LJ/Client/Twitter.pm	2010-09-17 11:06:01 UTC (rev 9511)
+++ trunk/cgi-bin/LJ/Client/Twitter.pm	2010-09-17 11:11:37 UTC (rev 9512)
@@ -7,6 +7,7 @@
 use Net::OAuth::AccessTokenRequest;
 use Net::OAuth::ProtectedResourceRequest;
 use HTTP::Request::Common;
+use Carp qw();
 
 use LJ::Client::Twitter::Tweet;
 use LJ::Client::Twitter::User;
@@ -63,17 +64,31 @@
  # in the DB, so every so often you may want to clear that
  LJ::Client::Twitter->remove_request_token($req_token);
  LJ::Client::Twitter->clean_stale_request_tokens;
+ 
+ # convert a twitter API time to a UNIX timestamp in GMT
+ warn LJ::Client::Twitter->parse_time('Sat Jul 03 21:24:02 +0000 2010');
+ 
+ # get last tweets posted by a particular user; returns an
+ # arrayref with LJ::Client::Twitter::Tweet objects
+ warn Data::Dumper::Dumper(LJ::Client::Twitter->get_last_tweets($u));
 
 =head1 SEE ALSO
 
 Twitter API Documentation: http://dev.twitter.com/doc
 
-This module is primarily used for reposting, so
+This module is primarily used for reposting LJ => Twitter, so
 LJ::Setting::TwitterConnect,
 LJ::Hooks::ThirdPartyNotify,
 LJ::Worker::Repost::EntryToTwitter.pm,
 LJ::Worker::Repost::CommentToTwitter.pm
 
+Twitter Digest, that does reposting the other way around:
+LJ::TwitterDigest
+
+Related modules: LJ::Client::Twitter::User, LJ::Client::Twitter::Tweet
+
+Rate limits: twitter_digest
+
 =cut
 
 sub generate_nonce {
@@ -91,9 +106,22 @@
     );
 }
 
+sub _ratelog {
+    if (my $u_system = LJ::load_user('system')) {
+        unless ($u_system->rate_log('twitter_digest', 1)) {
+            # this way, we ensure it'll sit in the error log, even if
+            # a framework like LJ::Setting suppresses errors
+            warn "twitter API rate limit reached";
+            Carp::croak "twitter API rate limit reached";
+        }
+    }
+}
+
 sub request_request_token {
     my ($class) = @_;
 
+    _ratelog;
+
     my $request = Net::OAuth::RequestTokenRequest->new(
         $class->default_request_params,
         request_url => 'https://api.twitter.com/oauth/request_token',
@@ -102,7 +130,8 @@
 
     $request->sign;
 
-    my $ua = LJ::get_useragent( 'role' => 'twitter_auth' );
+    my $ua = LJ::get_useragent( 'role' => 'twitter_auth',
+                                'timeout' => $LJ::TWITTER_API_TIMEOUT, );
     my $res = $ua->post($request->to_url);
 
     unless ($res->is_success) {
@@ -129,6 +158,8 @@
 sub request_access_token {
     my ($class, $request_token, $verifier) = @_;
 
+    _ratelog;
+
     my $request = Net::OAuth::AccessTokenRequest->new(
         $class->default_request_params,
         token => $request_token->{'public'},
@@ -140,7 +171,8 @@
 
     $request->sign;
 
-    my $ua = LJ::get_useragent( 'role' => 'twitter_auth' );
+    my $ua = LJ::get_useragent( 'role' => 'twitter_auth',
+                                'timeout' => $LJ::TWITTER_API_TIMEOUT, );
     my $res = $ua->post($request->to_url);
 
     unless ($res->is_success) {
@@ -165,6 +197,8 @@
 sub call {
     my ($class, %opts) = @_;
 
+    _ratelog;
+
     my $api_method  = $opts{'api_method'};
     die 'API method not provided' unless $api_method;
 
@@ -225,7 +259,8 @@
 
     $request->sign;
 
-    my $ua = LJ::get_useragent( 'role' => 'twitter_auth' );
+    my $ua = LJ::get_useragent( 'role' => 'twitter_auth',
+                                'timeout' => $LJ::TWITTER_API_TIMEOUT, );
     my $res;
     if ($http_method eq 'GET') {
         $res = $ua->get($request->to_url);
@@ -340,7 +375,6 @@
 # (from Net::Twitter::API)
 #
 # - ailyin, Sep 15, 2010
-#TODO: POD
 sub parse_time {
     my ($class, $time) = @_;
 
@@ -380,7 +414,7 @@
         'api_method' => 'statuses/user_timeline',
         'user' => $u,
         'http_method' => 'GET',
-        'params' => { 'count'       => 200,
+        'params' => { 'count'       => $LJ::TWITTER_RECENT_FEED_DEPTH,
                       'include_rts' => 1, },
     );
 

Modified: trunk/cgi-bin/LJ/Hooks/TwitterDigest.pm
===================================================================
--- trunk/cgi-bin/LJ/Hooks/TwitterDigest.pm	2010-09-17 11:06:01 UTC (rev 9511)
+++ trunk/cgi-bin/LJ/Hooks/TwitterDigest.pm	2010-09-17 11:11:37 UTC (rev 9512)
@@ -1,28 +1,29 @@
-package LJ::Hooks::TwitterDigest;--use strict;-use warnings;--use LJ::TwitterDigest;--LJ::register_hook('props_changed', sub {-    my ($u, $changes) = @_;--    if ($changes->{'timezone'}) {-        if (    $u->prop('twitter_access_token')-             && LJ::TwitterDigest->turned_on_for_user($u) )-        {-            LJ::TwitterDigest->set_next_post_time($u);-        }-    }--    if (exists $changes->{'twitter_access_token'}) {-        if ( !$changes->{'twitter_access_token'} ) {-            LJ::TwitterDigest->disable_for_user($u);-        } elsif ( LJ::TwitterDigest->turned_on_for_user($u) ) {-            LJ::TwitterDigest->set_next_post_time($u);-        }-    }-});--1;
\ No newline at end of file
+# see LJ::TwitterDigest for documentation
+
+package LJ::Hooks::TwitterDigest;
+use strict;
+use warnings;
+
+use LJ::TwitterDigest;
+
+LJ::register_hook('props_changed', sub {
+    my ($u, $changes) = @_;
+
+    if ($changes->{'timezone'}) {
+        if (    $u->prop('twitter_access_token')
+             && LJ::TwitterDigest->turned_on_for_user($u) )
+        {
+            LJ::TwitterDigest->set_next_post_time($u);
+        }
+    }
+
+    if (exists $changes->{'twitter_access_token'}) {
+        if ( !$changes->{'twitter_access_token'} ) {
+            LJ::TwitterDigest->disable_for_user($u);
+        } elsif ( LJ::TwitterDigest->turned_on_for_user($u) ) {
+            LJ::TwitterDigest->set_next_post_time($u);
+        }
+    }
+});
+
+1;

Modified: trunk/cgi-bin/LJ/TwitterDigest.pm
===================================================================
--- trunk/cgi-bin/LJ/TwitterDigest.pm	2010-09-17 11:06:01 UTC (rev 9511)
+++ trunk/cgi-bin/LJ/TwitterDigest.pm	2010-09-17 11:11:37 UTC (rev 9512)
@@ -1,3 +1,105 @@
+=head1 NAME
+
+TwitterDigest - the module that deals with grabbing a daily tweets digest
+from twitter and posting it to one's journal.
+
+=head1 SYNOPSIS
+
+ use LJ::TwitterDigest;
+
+=head2 subroutines that deal with the user settings
+
+ # ensure that the feature is enabled for a particular user,
+ # and set the next time the digest will be posted on
+ LJ::TwitterDigest->set_next_post_time($u);
+ 
+ # turn the feature off for a particular user; this is a reaction
+ # to the user unchecking the checkbox on the settings page
+ LJ::TwitterDigest->turn_off_for_user($u);
+ 
+ # return a boolean value indicating that the user has turned the
+ # feature on; note that even if the feature is turned on, it can
+ # still not work because of the user not having a twitter
+ # access token recorded
+ LJ::TwitterDigest->turned_on_for_user($u);
+ 
+ # write it down that because of circumstances, the feature can
+ # no longer be active, but should circumstances change, the user
+ # wishes it reenabled; this may happen if the user chooses to
+ # disconnect from twitter
+ LJ::TwitterDigest->disable_for_user($u);
+
+=head2 subroutines that deal with the actual posting
+
+ # get a single user we need to post a digest for, locking it
+ # for 15 minutes so as to prevent another process doing the same
+ # retrieving the same user. if there is no user like that, returns
+ # undef
+ my $u = LJ::TwitterDigest->get_pending_user;
+ 
+ # actually post digest for the user
+ LJ::TwitterDigest->post_digest($u);
+
+=head1 USECASES, WORKFLOW
+
+=over 2
+
+=item *
+
+If a user checks the checkbox on the settings page and saves,
+while having a twitter access token stored, C<set_next_post_time>
+is called. This one is simple.
+
+=item *
+
+If a user unchecks the same checkbox, C<turn_off_for_user> is called.
+
+=item *
+
+If a user chooses to disconnect from twitter, a hook is activated,
+which in turns calls C<disable_for_user>.
+
+=item *
+
+If a user chooses to reconnect, a hook is activated, calls
+C<turned_on_for_user> first to check if the user wants to reactivate
+digest posting, and if so, C<set_next_post_time> is called to
+reenable and schedule it.
+
+=item *
+
+If a user changes their time zone, a hook is activated, checks
+that the user has a) connected to twitter, and b) activated the
+digest feature (C<turned_on_for_user>), and if both conditions are
+met, C<set_next_post_time> is called to reenable and schedule it.
+
+=item *
+
+The processing worker calls C<get_pending_user>, C<post_digest>, and
+C<set_next_post_time> in a loop. In case C<get_pending_user>, it
+waits for half an hour, because after that, users with a different
+timezone setting can appear.
+
+=back
+
+=head1 SEE ALSO
+
+JIRA: https://jira.sup.com/browse/LJSUP-6780
+
+Hooks: LJ::Hooks::TwitterDigest
+
+Worker: bin/worker/twitter-digest
+
+Twitter API: LJ::Client::Twitter
+
+Rate limits: twitter_digest
+
+=head1 AUTHOR
+
+Andrew Ilyin <andrey.ilyin@sup.com>
+
+=cut
+
 package LJ::TwitterDigest;
 use strict;
 use warnings;
@@ -109,6 +211,20 @@
 sub post_digest {
     my ($class, $u) = @_;
 
+    if (my $u_system = LJ::load_user('system')) {
+        unless ($u_system->rate_log('twitter_digest', 1)) {
+            # wait a little bit so as to not hit the limit repeatedly,
+            # because we're in a worker context right now
+            sleep 1;
+
+            # the worker will start working on a next user just fine,
+            # but this user will be locked for the next 15 minutes;
+            # unfortunate, yes, but the basic fault tolerance
+            # is here
+            die "twitter digest rate limit reached, skipping $u->{user}";
+        }
+    }
+
     # this way, ML knows which language to use
     LJ::set_remote($u);
 

Tags: andy, dat, ljcom, pm
Subscribe
  • Post a new comment

    Error

    Anonymous comments are disabled in this journal

    default userpic

    Your reply will be screened

    Your IP address will be recorded 

  • 0 comments