Refactoring and adding tests

This commit is contained in:
Michael Lazar
2017-09-13 23:35:40 -04:00
parent 5e82811918
commit 031b58b3b4
9 changed files with 2500 additions and 109 deletions

View File

@@ -17,7 +17,6 @@ from . import docs
from .config import TEMPLATES
from .exceptions import InvalidRefreshToken
from .packages.praw.errors import HTTPException, OAuthException
from .packages.praw.handlers import DefaultHandler
_logger = logging.getLogger(__name__)
@@ -228,104 +227,3 @@ class OAuthHelper(object):
def clear_oauth_data(self):
self.reddit.clear_authentication()
self.config.delete_refresh_token()
def fix_cache(func):
"""
This is a shim around PRAW's 30 second page cache that attempts
to address broken behavior that hasn't been fixed because PRAW 3
is deprecated.
"""
def wraps(self, _cache_key, _cache_ignore, *args, **kwargs):
if _cache_key:
# Pop the request's session cookies from the cache key.
# These appear to be unreliable and change with every
# request. Also, with the introduction of OAuth I don't think
# that cookies are being used to store anything that
# differentiates API requests anyways
url, items = _cache_key
_cache_key = (url, (items[0], items[1], items[3], items[4]))
if kwargs['request'].method != 'GET':
# Why were POST/PUT/DELETE requests ever being cached???
_cache_ignore = True
return func(self, _cache_key, _cache_ignore, *args, **kwargs)
return wraps
class OAuthRateLimiter(DefaultHandler):
"""Custom PRAW request handler for rate-limiting requests.
This is an alternative to PRAW 3's DefaultHandler that uses
Reddit's modern API guidelines to rate-limit requests based
on the X-Ratelimit-* headers returned from Reddit.
References:
https://github.com/reddit/reddit/wiki/API
https://github.com/praw-dev/prawcore/blob/master/prawcore/rate_limit.py
"""
next_request_timestamp = None
def delay(self):
"""
Pause before making the next HTTP request.
"""
if self.next_request_timestamp is None:
return
sleep_seconds = self.next_request_timestamp - time.time()
if sleep_seconds <= 0:
return
time.sleep(sleep_seconds)
def update(self, response_headers):
"""
Update the state of the rate limiter based on the response headers:
X-Ratelimit-Used: Approximate number of requests used this period
X-Ratelimit-Remaining: Approximate number of requests left to use
X-Ratelimit-Reset: Approximate number of seconds to end of period
PRAW 5's rate limiting logic is structured for making hundreds of
evenly-spaced API requests, which makes sense for running something
like a bot or crawler.
This handler's logic, on the other hand, is geared more towards
interactive usage. It allows for short, sporadic bursts of requests.
The assumption is that actual users browsing reddit shouldn't ever be
in danger of hitting the rate limit. If they do hit the limit, they
will be cutoff until the period resets.
"""
if 'x-ratelimit-remaining' not in response_headers:
# This could be because the API returned an error response, or it
# could be because we're using something like read-only credentials
# which Reddit doesn't appear to care about rate limiting.
return
used = float(response_headers['x-ratelimit-used'])
remaining = float(response_headers['x-ratelimit-remaining'])
seconds_to_reset = int(response_headers['x-ratelimit-reset'])
_logger.debug('Rate limit: %s used, %s remaining, %s reset',
used, remaining, seconds_to_reset)
if remaining <= 0:
self.next_request_timestamp = time.time() + seconds_to_reset
else:
self.next_request_timestamp = None
@fix_cache
@DefaultHandler.with_cache
def request(self, request, proxies, timeout, verify, **_):
settings = self.http.merge_environment_settings(
request.url, proxies, False, verify, None)
self.delay()
response = self.http.send(
request, timeout=timeout, allow_redirects=False, **settings)
self.update(response.headers)
return response