Skip to content
This repository was archived by the owner on Nov 16, 2023. It is now read-only.

Commit 99c2c64

Browse files
eadseads
authored
Merge pull request #276 from newsdev/0121-response-caching
add response caching - closes #121, closes #250, closes #273.
2 parents 6b05862 + d6cba1e commit 99c2c64

File tree

11 files changed

+143
-24
lines changed

11 files changed

+143
-24
lines changed

‎docs/configuration.rst‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,9 @@ The following environment variables may be set:
99
export API_VERSION='v2'
1010
export BASE_URL='http://api.ap.org/v2'
1111
export AP_API_KEY='<<YOURAPIKEY>>'
12-
export ELEX_DELEGATE_REPORT_ID_CACHE_FILE='/tmp/elex-cache'
1312
export ELEX_RECORDING='flat'
1413
export ELEX_RECORDING_DIR='/tmp/elex-recording'
14+
export ELEX_CACHE_DIRECTORY='/tmp/elex-cache'
1515

1616
API_VERSION
1717
===========
@@ -28,6 +28,11 @@ AP_API_KEY
2828

2929
Your API key. Must be set.
3030

31+
ELEX_CACHE_DIRECTORY
32+
====================
33+
34+
Path to the Elex cache directory. If not set, defaults to ``<tempdir>/elex-cache`` where ``<tempdir>`` is whatever Python's ``tempfile.gettempdir()`` returns.
35+
3136
ELEX_RECORDING, ELEX_RECORDING_DIR
3237
==================================
3338

‎elex/__init__.py‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,22 @@
11
import os
22
import pkg_resources
3+
import requests
4+
import tempfile
35

6+
from cachecontrol import CacheControl
7+
from cachecontrol.caches import FileCache
8+
from elex.cachecontrol_heuristics import EtagOnlyCache
49

510
__version__ = pkg_resources.get_distribution('elex').version
11+
_DEFAULT_CACHE_DIRECTORY = os.path.join(tempfile.gettempdir(), 'elex-cache')
12+
13+
API_KEY = os.environ.get('AP_API_KEY', None)
614
API_VERSION = os.environ.get('AP_API_VERSION', 'v2')
715
BASE_URL = os.environ.get('AP_API_BASE_URL', 'http://api.ap.org/{0}'.format(API_VERSION))
8-
API_KEY = os.environ.get('AP_API_KEY', None)
16+
CACHE_DIRECTORY = os.environ.get('ELEX_CACHE_DIRECTORY', _DEFAULT_CACHE_DIRECTORY)
17+
18+
session = requests.session()
19+
session.headers.update({'Accept-Encoding': 'gzip'})
20+
cache = CacheControl(session,
21+
cache=FileCache(CACHE_DIRECTORY),
22+
heuristic=EtagOnlyCache())

‎elex/api/__init__.py‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -14,13 +14,13 @@
1414
)
1515
__all__ = [
1616
'APElection',
17-
'Candidate',
1817
'BallotMeasure',
18+
'Candidate',
19+
'CandidateDelegateReport',
1920
'CandidateReportingUnit',
20-
'ReportingUnit',
21-
'Race',
22-
'Elections',
21+
'DelegateReport',
2322
'Election',
24-
'CandidateDelegateReport',
25-
'DelegateReport'
23+
'Elections',
24+
'Race',
25+
'ReportingUnit',
2626
]

‎elex/api/utils.py‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
import ujson as json
1111
import time
1212
import datetime
13-
import requests
13+
from elex import cache
1414
from elex.exceptions import APAPIKeyException
1515
from pymongo import MongoClient
1616

@@ -91,7 +91,14 @@ def api_request(path, **params):
9191
raise APAPIKeyException()
9292

9393
params['format'] = 'json'
94-
response = requests.get(elex.BASE_URL + path, params=params)
94+
95+
params = sorted(params.items()) # Sort for consistent caching
96+
97+
url = '{0}{1}'.format(elex.BASE_URL, path)
98+
99+
response = cache.get(url, params=params)
95100
response.raise_for_status()
101+
96102
write_recording(response.json())
103+
97104
return response

‎elex/cachecontrol_heuristics.py‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
from cachecontrol.heuristics import BaseHeuristic
2+
3+
4+
class EtagOnlyCache(BaseHeuristic):
5+
"""
6+
Strip max-age cache-control header if it exists alongside etag.
7+
"""
8+
def update_headers(self, response):
9+
headers = {}
10+
max_age = 'max-age' in response.headers.get('cache-control', '')
11+
etag = response.headers.get('etag', None)
12+
if max_age and etag:
13+
headers['cache-control'] = 'public'
14+
return headers

‎elex/cli/app.py‎

Lines changed: 42 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,11 @@
1-
from elex.api import Elections
2-
from elex.api import DelegateReport
3-
from elex import __version__ as VERSION
1+
from cement.core.controller import CementBaseController, expose
42
from cement.core.foundation import CementApp
5-
from elex.cli.hooks import add_election_hook
63
from cement.ext.ext_logging import LoggingLogHandler
7-
from cement.core.controller import CementBaseController, expose
4+
from elex.api import DelegateReport
5+
from elex.api import Elections
6+
from elex.cli.constants import BANNER, LOG_FORMAT
87
from elex.cli.decorators import require_date_argument, require_ap_api_key
9-
10-
LOG_FORMAT = '%(asctime)s (%(levelname)s) %(namespace)s (v{0}) : \
11-
%(message)s'.format(VERSION)
12-
BANNER = """
13-
NYT AP Elections version {0}
14-
""".format(VERSION)
8+
from elex.cli.hooks import add_election_hook, cachecontrol_logging_hook
159

1610

1711
class ElexBaseController(CementBaseController):
@@ -131,6 +125,12 @@ def races(self):
131125
self.app.log.debug(
132126
'Elex API URL: {0}'.format(self.app.election._response.url)
133127
)
128+
self.app.log.debug(
129+
'ELAPI cache hit: {0}'.format(self.app.election._response.from_cache)
130+
)
131+
if self.app.election._response.from_cache:
132+
self.app.exit_code = 64
133+
134134
self.app.render(data)
135135

136136
@expose(help="Get reporting units")
@@ -171,6 +171,12 @@ def reporting_units(self):
171171
self.app.log.debug(
172172
'Elex API URL: {0}'.format(self.app.election._response.url)
173173
)
174+
self.app.log.debug(
175+
'ELAPI cache hit: {0}'.format(self.app.election._response.from_cache)
176+
)
177+
if self.app.election._response.from_cache:
178+
self.app.exit_code = 64
179+
174180
self.app.render(data)
175181

176182
@expose(help="Get candidate reporting units (without results)")
@@ -233,6 +239,12 @@ def candidate_reporting_units(self):
233239
self.app.log.debug(
234240
'Elex API URL: {0}'.format(self.app.election._response.url)
235241
)
242+
self.app.log.debug(
243+
'ELAPI cache hit: {0}'.format(self.app.election._response.from_cache)
244+
)
245+
if self.app.election._response.from_cache:
246+
self.app.exit_code = 64
247+
236248
self.app.render(data)
237249

238250
@expose(help="Get candidates")
@@ -269,6 +281,12 @@ def candidates(self):
269281
self.app.log.debug(
270282
'Elex API URL: {0}'.format(self.app.election._response.url)
271283
)
284+
self.app.log.debug(
285+
'ELAPI cache hit: {0}'.format(self.app.election._response.from_cache)
286+
)
287+
if self.app.election._response.from_cache:
288+
self.app.exit_code = 64
289+
272290
self.app.render(data)
273291

274292
@expose(help="Get ballot measures")
@@ -305,6 +323,12 @@ def ballot_measures(self):
305323
self.app.log.debug(
306324
'Elex API URL: {0}'.format(self.app.election._response.url)
307325
)
326+
self.app.log.debug(
327+
'ELAPI cache hit: {0}'.format(self.app.election._response.from_cache)
328+
)
329+
if self.app.election._response.from_cache:
330+
self.app.exit_code = 64
331+
308332
self.app.render(data)
309333

310334
@expose(help="Get results")
@@ -343,6 +367,12 @@ def results(self):
343367
self.app.log.debug(
344368
'Elex API URL: {0}'.format(self.app.election._response.url)
345369
)
370+
self.app.log.debug(
371+
'ELAPI cache hit: {0}'.format(self.app.election._response.from_cache)
372+
)
373+
if self.app.election._response.from_cache:
374+
self.app.exit_code = 64
375+
346376
self.app.render(data)
347377

348378
@expose(help="Get list of available elections")
@@ -462,6 +492,7 @@ class Meta:
462492
label = 'elex'
463493
base_controller = ElexBaseController
464494
hooks = [
495+
('post_setup', cachecontrol_logging_hook),
465496
('post_argument_parsing', add_election_hook),
466497
]
467498
extensions = [

‎elex/cli/constants.py‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
from elex import __version__ as VERSION
2+
3+
LOG_FORMAT = '%(asctime)s (%(levelname)s) %(name)s : \
4+
%(message)s'
5+
6+
BANNER = """
7+
NYT AP Elections version {0}
8+
""".format(VERSION)

‎elex/cli/decorators.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
from elex import CACHE_DIRECTORY
12
from elex.cli.utils import parse_date
23
from elex.exceptions import APAPIKeyException
34
from functools import wraps
@@ -41,6 +42,7 @@ def require_ap_api_key(fn):
4142
"""
4243
@wraps(fn)
4344
def decorated(self):
45+
self.app.log.debug('Cache directory: {0}'.format(CACHE_DIRECTORY))
4446
try:
4547
return fn(self)
4648
except HTTPError as e:

‎elex/cli/hooks.py‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,12 @@
1+
import logging
12
from elex.api import Election
3+
from elex.cli.constants import LOG_FORMAT
24

35

46
def add_election_hook(app):
57
"""
68
Cache election API object reference after parsing args.
79
"""
8-
910
app.election = Election(
1011
testresults=app.pargs.test,
1112
liveresults=not app.pargs.not_live,
@@ -26,3 +27,17 @@ def add_election_hook(app):
2627

2728
if app.pargs.raceids:
2829
app.election.raceids = [x.strip() for x in app.pargs.raceids.split(',')]
30+
31+
32+
def cachecontrol_logging_hook(app):
33+
"""
34+
Reroute cachecontrol logger to use cement log handlers.
35+
"""
36+
from cachecontrol.controller import logger
37+
formatter = logging.Formatter(LOG_FORMAT)
38+
39+
for handler in app.log.backend.handlers:
40+
handler.setFormatter(formatter)
41+
logger.addHandler(handler)
42+
43+
logger.setLevel(logging.DEBUG)

‎requirements.txt‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
1+
CacheControl==0.11.7
12
cement==2.6.2
3+
lockfile==0.12.2
24
pymongo==2.8
3-
python-dateutil==2.2
5+
python-dateutil==2.5.3
46
requests==2.9.1
5-
ujson==1.35
7+
ujson==1.35

0 commit comments

Comments
 (0)