328 lines
15 KiB
Python
328 lines
15 KiB
Python
from StringIO import StringIO
|
|
from cgi import escape
|
|
import sys
|
|
import json
|
|
import os
|
|
import subprocess
|
|
from strip_chunked import decode
|
|
from mimetools import Message
|
|
from urlparse import urlparse
|
|
import random
|
|
import zlib
|
|
import string
|
|
|
|
class ReplayApp:
|
|
|
|
def _parse_push_strategy(self):
|
|
|
|
if self.push_strategy_file == "noop":
|
|
return
|
|
|
|
with open(self.push_strategy_file) as fd:
|
|
parsed_push_strategy = json.load(fd)
|
|
if len(parsed_push_strategy['push_configs']) > 0:
|
|
for push_strategy_for_host in parsed_push_strategy['push_configs']:
|
|
if 'push_host' in push_strategy_for_host.keys():
|
|
self.push_host.append(push_strategy_for_host['push_host'])
|
|
self.push_trigger_path.append(push_strategy_for_host['push_trigger'])
|
|
self.push_assets.append(push_strategy_for_host['push_resources'])
|
|
|
|
if 'hint_host' in push_strategy_for_host.keys():
|
|
self.hint_host.append(push_strategy_for_host['hint_host'])
|
|
self.hint_trigger_path.append(push_strategy_for_host['hint_trigger'])
|
|
self.hint_assets.append(push_strategy_for_host['hint_resources'])
|
|
self.hint_mimetype.append(push_strategy_for_host['hint_mimetype'])
|
|
|
|
mimetypes = []
|
|
for i,mimetype in enumerate(push_strategy_for_host['hint_mimetype']):
|
|
if 'image' in mimetype:
|
|
mimetypes.append('image')
|
|
elif 'css' in mimetype:
|
|
mimetypes.append('style')
|
|
elif 'javascript' in mimetype:
|
|
mimetypes.append('script')
|
|
elif 'font' in mimetype:
|
|
mimetypes.append('font')
|
|
else: #xhr
|
|
print "WARNING: unmatched mimetype",mimetype, push_strategy_for_host['hint_resources'][i]
|
|
mimetypes.append('')
|
|
|
|
self.hint_as_string.append(mimetypes)
|
|
|
|
def _query_replayserver(self, env, hostname, path):
|
|
passed_env = dict()
|
|
# remap for compat with replayserver
|
|
passed_env['SERVER_PROTOCOL'] = "HTTP/1.1"
|
|
passed_env['MAHIMAHI_CHDIR'] = env['WORKING_DIR']
|
|
passed_env['MAHIMAHI_RECORD_PATH'] = env['RECORDING_DIR']
|
|
passed_env['REQUEST_METHOD'] = 'GET' # we assume this is always get..
|
|
passed_env['HTTPS'] = "1"
|
|
passed_env['REQUEST_URI'] = path
|
|
passed_env['HTTP_HOST'] = hostname
|
|
|
|
if 'HTTP_RANGE' in env.keys():
|
|
passed_env['HTTP_RANGE'] = env['HTTP_RANGE']
|
|
|
|
p = subprocess.Popen([env['REPLAYSERVER_FN']], stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=passed_env)
|
|
(replay_stdout, _) = p.communicate()
|
|
return replay_stdout
|
|
|
|
def _init_push_responsecache(self):
|
|
env = os.environ
|
|
for i,host in enumerate(self.push_host):
|
|
self.push_cache.append([])
|
|
for j,asset_name in enumerate(self.push_assets[i]):
|
|
parsed = urlparse(asset_name)
|
|
stripped = asset_name[len('https://'+parsed.netloc):]
|
|
asset_name = stripped
|
|
passed_host = parsed.netloc
|
|
|
|
# print "Caching... ", asset_name, parsed
|
|
# load responses into cache
|
|
self.push_cache[i].append(self._query_replayserver(env,passed_host,asset_name))
|
|
|
|
|
|
def _init_custom_scheduler(self):
|
|
#check if this instance of the replay app is running a custom scheduler
|
|
env = os.environ
|
|
if env['NGINX_CUSTOM_SCHEDULER'] == "1" and self.push_strategy_file != "noop":
|
|
self.has_custom_strategy = True
|
|
self.custom_strategy_host_trigger = env['NGINX_CUSTOM_TRIGGER_HOST']
|
|
self.custom_strategy_path_trigger = env['NGINX_CUSTOM_TRIGGER_PATH']
|
|
self.custom_strategy_offset_html = int(env['NGINX_CUSTOM_OFFSET'])
|
|
|
|
custom_strategy_html_content_file = os.path.join(os.path.dirname(self.push_strategy_file), env['NGINX_CUSTOM_CONTENT_FILE'])
|
|
|
|
print "Custom Replay Scheduler active for "+env['NGINX_CUSTOM_TRIGGER_HOST']
|
|
|
|
replay_response = self._query_replayserver(env,self.custom_strategy_host_trigger,self.custom_strategy_path_trigger)
|
|
self.custom_strategy_response = replay_response
|
|
|
|
c = zlib.compressobj()
|
|
|
|
with open(custom_strategy_html_content_file,'r') as fd:
|
|
decompressed_index = fd.read()
|
|
|
|
print decompressed_index[:self.custom_strategy_offset_html]
|
|
|
|
#print decompressed_index
|
|
self.custom_html_content_A = c.compress(decompressed_index[:self.custom_strategy_offset_html]) + c.flush(zlib.Z_SYNC_FLUSH)
|
|
self.custom_html_content_B = c.compress(decompressed_index[self.custom_strategy_offset_html:]) + c.flush(zlib.Z_FINISH)
|
|
|
|
self.CRITICAL_CSS_PATH = '/push__critical.css'
|
|
critical_css_file = os.path.join(os.path.dirname(self.push_strategy_file), env['NGINX_CRITICAL_CSS_FILE'])
|
|
print "Reading", critical_css_file
|
|
c = zlib.compressobj()
|
|
with open(critical_css_file,'r') as fd:
|
|
self.CRITICAL_CSS_CONTENT = c.compress(fd.read()) + c.flush(zlib.Z_FINISH)
|
|
print "Done Reading"
|
|
|
|
|
|
def __init__(self):
|
|
self.has_custom_strategy = False
|
|
self.custom_strategy_host_trigger = ''
|
|
self.custom_strategy_path_trigger = ''
|
|
self.custom_html_content_A = ''
|
|
self.custom_html_content_B = ''
|
|
self.custom_strategy_offset_html = 0
|
|
|
|
self.push_strategy_file = os.environ['PUSH_STRATEGY_FILE']
|
|
self.push_host = []
|
|
self.push_trigger_path = []
|
|
self.push_assets = []
|
|
self.push_cache = []
|
|
|
|
self.hint_host = []
|
|
self.hint_trigger_path = []
|
|
self.hint_assets = []
|
|
self.hint_as_string = []
|
|
self.hint_mimetype = []
|
|
|
|
self._parse_push_strategy()
|
|
self._init_push_responsecache()
|
|
self._init_custom_scheduler()
|
|
|
|
|
|
|
|
|
|
def __call__(self, environ, start_response):
|
|
#print "call! " + p['REQUEST_URI']
|
|
#print environ
|
|
hdrlist = []
|
|
env = dict(environ)
|
|
if env['HTTP_HOST'].endswith('.'):
|
|
env['HTTP_HOST'] = env['HTTP_HOST'][:-1]
|
|
|
|
|
|
#FastPath for Push Garbage:
|
|
if env['REQUEST_METHOD'] == "GET" and env['REQUEST_URI'].startswith('/__GARBAGE__'):
|
|
amount_kb = int(env['REQUEST_URI'].split('__')[2])
|
|
N = amount_kb * 1024
|
|
response = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(N))
|
|
start_response('200 OK', [('Content-Type', 'text/plain')])
|
|
yield response
|
|
|
|
#FastPath for Push Critical CSS:
|
|
elif env['REQUEST_METHOD'] == "GET" and env['HTTP_HOST'] == self.custom_strategy_host_trigger and env['REQUEST_URI'].startswith(self.CRITICAL_CSS_PATH):
|
|
start_response("200 OK",
|
|
[
|
|
('Content-Type', 'text/css'),
|
|
('Content-Encoding', 'deflate'),
|
|
])
|
|
yield self.CRITICAL_CSS_CONTENT
|
|
|
|
else:
|
|
|
|
cached_response = None
|
|
is_push = False
|
|
if env['REQUEST_METHOD'] == "GET":
|
|
for u,host in enumerate(self.push_host):
|
|
if host == env['HTTP_HOST']:
|
|
for v,push_resource in enumerate(self.push_assets[u]):
|
|
parsed = urlparse(push_resource)
|
|
stripped = push_resource[len('https://'+parsed.netloc):]
|
|
push_resource = stripped
|
|
if push_resource == env['REQUEST_URI']:
|
|
#print "pushing from cache..."
|
|
is_push = True
|
|
cached_response = self.push_cache[u][v]
|
|
|
|
#print env['HTTP_HOST'], env['REQUEST_URI']
|
|
|
|
if cached_response is None:
|
|
# print environ['REQUEST_URI'], "Not cached!"
|
|
passed_env = dict()
|
|
|
|
# remap for compat with replayserver
|
|
passed_env['MAHIMAHI_CHDIR'] = env['WORKING_DIR']
|
|
passed_env['MAHIMAHI_RECORD_PATH'] = env['RECORDING_DIR']
|
|
passed_env['REQUEST_METHOD'] = env['REQUEST_METHOD']
|
|
passed_env['REQUEST_URI'] = env['REQUEST_URI']
|
|
|
|
# env['SERVER_PROTOCOL'], is currently a hack to find the corresponding
|
|
# h1 traces
|
|
passed_env['SERVER_PROTOCOL'] = "HTTP/1.1"
|
|
passed_env['HTTP_HOST'] = env['HTTP_HOST']
|
|
|
|
if 'HTTP_RANGE' in env.keys():
|
|
passed_env['HTTP_RANGE'] = env['HTTP_RANGE']
|
|
|
|
if env['wsgi.url_scheme'] == 'https':
|
|
passed_env['HTTPS'] = "1"
|
|
|
|
# shell=True,
|
|
p = subprocess.Popen(
|
|
[env['REPLAYSERVER_FN']], stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=passed_env)
|
|
|
|
(cached_response, replay_stderr) = p.communicate()
|
|
|
|
custom_interleaving = False
|
|
# Check if we must deliver using a custom strategy:
|
|
if env['REQUEST_METHOD'] == "GET" and env['HTTP_HOST'] == self.custom_strategy_host_trigger and env['REQUEST_URI'] == self.custom_strategy_path_trigger:
|
|
custom_interleaving = True
|
|
#print "Size: " + str(len(self.custom_html_content_A))
|
|
|
|
if not custom_interleaving:
|
|
response_header, response_body = cached_response.split('\r\n\r\n', 1)
|
|
else:
|
|
response_header, _ = self.custom_strategy_response.split('\r\n\r\n', 1)
|
|
|
|
|
|
|
|
status_line, headers_alone = response_header.split('\r\n', 1)
|
|
splitted_status = status_line.split(' ')
|
|
|
|
# response_code = status_line[1]
|
|
|
|
status_cleaned = ' '.join(splitted_status[1:])
|
|
#print status_cleaned
|
|
|
|
headers = Message(StringIO(headers_alone))
|
|
|
|
#print env['REQUEST_METHOD'], env['REQUEST_URI']
|
|
|
|
is_chunked = False
|
|
corsfound = False
|
|
hdrlist = []
|
|
|
|
for key in headers.keys():
|
|
if key == "transfer-encoding" and 'chunked' in headers[key]:
|
|
is_chunked = True
|
|
else:
|
|
if custom_interleaving and key.lower() == 'content-encoding':
|
|
hdrlist.append((key.strip(), 'deflate'))
|
|
elif custom_interleaving and key.lower() == 'content-length':
|
|
continue
|
|
|
|
else:
|
|
#Filter out any link header the site might emit.
|
|
if key not in ['expires', 'date', 'last-modified','link', 'alt-svc','content-security-policy']:
|
|
hdrlist.append((key.strip(), headers[key]))
|
|
if key.lower() == 'access-control-allow-origin':
|
|
corsfound = True
|
|
|
|
|
|
if not corsfound:
|
|
# this is required for font hinting...
|
|
# note that we will not overwrite cors headers b/c
|
|
# in some xhr situations * is not sufficient
|
|
hdrlist.append(('Access-Control-Allow-Origin','*'))
|
|
|
|
|
|
if not is_push and env['SERVER_PROTOCOL'] == "HTTP/2":
|
|
for i, push_host_strategy in enumerate(self.push_host):
|
|
if passed_env['HTTP_HOST'] == push_host_strategy:
|
|
if passed_env['REQUEST_URI'] == self.push_trigger_path[i]:
|
|
linkstr = ''
|
|
# TODO (bewo): is there any limitation?
|
|
for asset in self.push_assets[i]:
|
|
if linkstr != '':
|
|
linkstr += ','
|
|
# print asset;
|
|
#print asset
|
|
linkstr += '<' + asset + '>; rel=preload'
|
|
hdrlist.append(('x-extrapush', str(linkstr)))
|
|
print 'WILL PUSH: ' ,len(self.push_assets[i]) #//, ('x-extrapush', str(linkstr))
|
|
break
|
|
|
|
if not is_push:
|
|
for i, hint_host_strategy in enumerate(self.hint_host):
|
|
if passed_env['HTTP_HOST'] == hint_host_strategy:
|
|
if passed_env['REQUEST_URI'] == self.hint_trigger_path[i]:
|
|
linkstr = ''
|
|
for j, asset in enumerate(self.hint_assets[i]):
|
|
if linkstr != '':
|
|
linkstr += ','
|
|
as_string = self.hint_as_string[i][j]
|
|
if as_string != '':
|
|
as_string='; as='+as_string+''
|
|
if self.hint_as_string[i][j] == "font":
|
|
as_string += ";crossorigin"
|
|
linkstr_to_append = '<'+asset + '>; rel=preload'+as_string+';type="'+self.hint_mimetype[i][j]+'"'
|
|
linkstr += linkstr_to_append
|
|
hdrlist.append(('link', str(linkstr)))
|
|
# print 'WILL HINT: ' ,len(self.hint_assets[i]) #//, ('x-extrapush', str(linkstr))
|
|
break
|
|
|
|
# print "start response! " + environ['HTTP_HOST'] + " - " + environ['REQUEST_URI']
|
|
|
|
if not custom_interleaving:
|
|
if is_chunked:
|
|
# print "will decode chunked"
|
|
start_response(status_cleaned, hdrlist)
|
|
# print "chunked"
|
|
for chunk in decode(StringIO(response_body)):
|
|
yield str(chunk)
|
|
else:
|
|
start_response(status_cleaned, hdrlist)
|
|
yield response_body
|
|
else:
|
|
#note that we already decode the chunk
|
|
start_response(status_cleaned, hdrlist)
|
|
yield self.custom_html_content_A
|
|
yield self.custom_html_content_B
|
|
|
|
|
|
#start_response('200 OK', [('Content-Type', 'text/html')])
|
|
# yield replay_stdout
|