diff --git a/.gitignore b/.gitignore index d590fa9..9582357 100644 --- a/.gitignore +++ b/.gitignore @@ -8,5 +8,9 @@ *.swp settings.py build/* +gbench/parse +gbench/t +gbench/string +gbench/query .DS_Store __pycache__ diff --git a/README.md b/README.md index 59b5ced..b83f915 100644 --- a/README.md +++ b/README.md @@ -4,29 +4,39 @@ Async Python 3.5+ web server written in C # Benchmarks ``` -Hello pipelined 4,152,858 Requests/second -Hello 633,097 Requests/second -404 654,053 Requests/second -Cookies 422,728 Requests/second -Form parsing 328,780 Requests/second -Parse JSON 224,872 Requests/second -Templates 257,753 Requests/second -Sessions: - memcached 163,833 Requests/second - mrcache 283,359 Requests/second -MrWorkServer 338,891 Requests/second -File Upload 132,242 Requests/second + Pipelined + Hello (cached) 8,534,332 Requests/second + Hello 6,834,994 Requests/second + More hdrs 6,193,307 Requests/second + Sessions 4,396,364 Requests/second + File Upload 3,510,289 Requests/second + mrpacker 2,052,674 Requests/second + Form 1,182,228 Requests/second + + One by one + Hello 707,667 Requests/second + Hello hdrs 728,639 Requests/second + Cookies 588,212 Requests/second + many args 691,910 Requests/second + 404 natural 763,643 Requests/second + 404 580,424 Requests/second + Form parsing 338,553 Requests/second + mrpacker 533,242 Requests/second + Sessions 325,354 Requests/second + File Upload 292,331 Requests/second + get ip 503,454 Requests/second + ``` Versus sanic a pure python async server ``` -Hello World 64,366 Requests/second -Cookies 50,867 Requests/second -404 9,256 Requests/second -forms 27,104 Requests/second +Hello World 22,366 Requests/second +Cookies 20,867 Requests/second +404 8,256 Requests/second +forms 11,104 Requests/second sessions 4,053 Requests/second -File upload 21,457 Requests/second +File upload 1,457 Requests/second ``` Hello World Example diff --git a/bench/sanic/readme b/bench/sanic/readme index 0c84d7f..224ae28 100644 --- a/bench/sanic/readme +++ b/bench/sanic/readme @@ -1,5 +1,4 @@ -pip install sanic -pip install sanic_session +pip install sanic sanic_session wrk -t 4 -c 32 -d 2 http://localhost:8080/ diff --git a/bench/sanic/session.py b/bench/sanic/session.py new file mode 100644 index 0000000..6105421 --- /dev/null +++ b/bench/sanic/session.py @@ -0,0 +1,30 @@ + +import aiomcache +import uvloop + +from sanic import Sanic +from sanic.response import text +from sanic_session import Session, MemcacheSessionInterface + +app = Sanic("app") + +# create a memcache client +client = aiomcache.Client("127.0.0.1", 11211) + +# pass the memcache client into the session +session = Session(app, interface=MemcacheSessionInterface(client)) + +@app.route("/") +async def test(request): + # interact with the session like a normal dict + if not request.ctx.session.get('foo'): + request.ctx.session['foo'] = 0 + + request.ctx.session['foo'] += 1 + + response = text("YAY") + + return response + +if __name__ == "__main__": + app.run(host="0.0.0.0", port=8000, debug=True) diff --git a/bench/sanic/tst.py b/bench/sanic/tst.py index d07e503..f8ae93a 100644 --- a/bench/sanic/tst.py +++ b/bench/sanic/tst.py @@ -1,13 +1,15 @@ import sanic -from sanic.response import json app = sanic.Sanic("my-hello-world-app") @app.route('/') async def test(request): - return sanic.text("Hello World") + return sanic.text("Hello World") + +@app.route("/s") +async def sess(request): + return sanic.text("session") if __name__ == '__main__': app.run(port=8080) - diff --git a/curl.sh b/curl.sh index 38065e6..a9c7f15 100755 --- a/curl.sh +++ b/curl.sh @@ -1,5 +1,3 @@ - -for run in {1..1000} -do - curl -d "param1=value1¶m2=value2" -X POST http://localhost:8080/ -H "Content-Type: application/x-www-form-urlencoded" +for n in {1..10}; do + curl -d "param1=value1¶m2=value2" -X POST http://localhost:8080/form -H "Content-Type: application/x-www-form-urlencoded" done diff --git a/dotests.py b/dotests.py index f5fb599..e6bb305 100644 --- a/dotests.py +++ b/dotests.py @@ -1,6 +1,13 @@ readme = """ - pip install psutil requests msgpack mrasyncmc tenjin mrpacker + pip install psutil requests msgpack mrasyncmc tenjin mrpacker mrworkserver + mrcache and mrworkserver must be running for the benchmarks + mrcache: + git clone https://github.com/MarkReedZ/mrcache.git + cd mrcache; ./bld; ./mrcache + mrworkserver + python workserver.py + """ @@ -11,12 +18,24 @@ import tests +import argparse +import sys +import asyncio +import os +from asyncio.subprocess import PIPE, STDOUT +import statistics + +import uvloop +import psutil +import atexit + # TODO -# Check for memcached being up and add the session key so we hit and load the json 43709dd361cc443e976b05714581a7fb -# memcached -l 127.0.0.1 -p 11211 -d -m 50 +# Check for mrworkserver and mrcache being up and add the session key so we hit and load the json 43709dd361cc443e976b05714581a7fb +# mrcache -m 64 -i 16 +# python mrworkserver/tst.py -if 1: +async def run_tests(): package = tests for importer, modname, ispkg in pkgutil.iter_modules(package.__path__): if modname.startswith("test"): @@ -36,53 +55,33 @@ if f[0] == 'teardown': f[1]() -print("Benchmarks") -import argparse -import sys -import asyncio -import os -from asyncio.subprocess import PIPE, STDOUT -import statistics - -import uvloop -import psutil -import atexit #from misc import cpu -def run_wrk(loop, endpoint=None, lua=None, options=None): +async def run_wrk(loop, endpoint=None, lua=None, options=None): rps = 0 try: endpoint = endpoint or 'http://localhost:8080/' if lua: if options != None: - wrk_fut = asyncio.create_subprocess_exec( 'wrk', '-t', '4', '-c', '32', '-d', '2', '-s', lua, *options, endpoint, stdout=PIPE, stderr=STDOUT) + proc = await asyncio.create_subprocess_exec( 'wrk', '-t', '4', '-c', '32', '-d', '2', '-s', lua, *options, endpoint, stdout=PIPE, stderr=STDOUT) else: - wrk_fut = asyncio.create_subprocess_exec( 'wrk', '-t', '4', '-c', '32', '-d', '2', '-s', lua, endpoint, stdout=PIPE, stderr=STDOUT) + proc = await asyncio.create_subprocess_exec( 'wrk', '-t', '4', '-c', '32', '-d', '2', '-s', lua, endpoint, stdout=PIPE, stderr=STDOUT) else: if options != None: - wrk_fut = asyncio.create_subprocess_exec( 'wrk', '-t', '4', '-c', '32', '-d', '2', *options, endpoint, stdout=PIPE, stderr=STDOUT) + proc = await asyncio.create_subprocess_exec( 'wrk', '-t', '4', '-c', '32', '-d', '2', *options, endpoint, stdout=PIPE, stderr=STDOUT) else: - wrk_fut = asyncio.create_subprocess_exec( 'wrk', '-t', '4', '-c', '32', '-d', '2', endpoint, stdout=PIPE, stderr=STDOUT) + proc = await asyncio.create_subprocess_exec( 'wrk', '-t', '4', '-c', '32', '-d', '2', endpoint, stdout=PIPE, stderr=STDOUT) - wrk = loop.run_until_complete(wrk_fut) + stdout, stderr = await proc.communicate() rps = 0 - lines = [] - while 1: - line = loop.run_until_complete(wrk.stdout.readline()) - if line: - line = line.decode('utf-8') - lines.append(line) - if line.startswith('Requests/sec:'): - rps = float(line.split()[-1]) - else: - break + lines = stdout.decode('utf-8').split("\n") + for line in lines: + if line.startswith('Requests/sec:'): + rps = float(line.split()[-1]) - retcode = loop.run_until_complete(wrk.wait()) - if retcode != 0: - print('\r\n'.join(lines)) except Exception as e: print(e) @@ -90,88 +89,76 @@ def run_wrk(loop, endpoint=None, lua=None, options=None): return rps -noisy = ['atom', 'chrome', 'firefox', 'dropbox', 'opera', 'spotify', 'gnome-documents'] +async def run_benchmarks(): + proc = await asyncio.create_subprocess_exec( 'python', 'tests/s_bench.py', stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE ) + process = psutil.Process(proc.pid) -def silence(): - for proc in psutil.process_iter(): - if proc.name() in noisy: - proc.suspend() + await asyncio.sleep(1) - def resume(): - for proc in psutil.process_iter(): - if proc.name() in noisy: - proc.resume() - atexit.register(resume) + if proc.returncode != None: + print("tests/s_bench.py failed to start:") + print(await proc.stdout.read()) + print(await proc.stderr.read()) + exit() -silence() + print("Benchmarks") -loop = uvloop.new_event_loop() + try: + + more_headers = ('-H','User-Agent: Mozilla/5.0 (X11; Linux x86_64) Gecko/20130501 Firefox/30.0 AppleWebKit/600.00 Chrome/30.0.0000.0 Trident/10.0 Safari/600.00', + '-H','Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', + '-H','Accept-Language: en-US,en;q=0.5', + '-H','Cookie: mrsession=43709dd361cc443e976b05714581a7fb; foo=fdsfdasdfasdfdsfasdfsdfsdfasdfas; short=fazc;', + '-H','Connection: keep-alive') + opts = ('-H','Cookie: mrsession=43709dd361cc443e976b05714581a7fb; foo=fdsfdasdfasdfdsfasdfsdfsdfasdfas; short=fazc;') + + print(" Pipelined") + print (" Hello ", await run_wrk(loop, 'http://localhost:8080/',lua='tests/lua/pipeline.lua'), "Requests/second" ) + print (" More hdrs ", await run_wrk(loop, 'http://localhost:8080/',options=more_headers,lua='tests/lua/pipeline.lua'), "Requests/second" ) + print (" Sessions ", await run_wrk(loop, 'http://localhost:8080/s',lua='tests/lua/q-session.lua'), "Requests/second" ) + print (" File Upload ", await run_wrk(loop, 'http://localhost:8080/upload',lua='tests/lua/q-upload.lua'), "Requests/second" ) + print (" mrpacker ", await run_wrk(loop, 'http://localhost:8080/mrpacker',lua='tests/lua/q-mrp.lua'), "Requests/second" ) + print (" Form ", await run_wrk(loop, 'http://localhost:8080/form',lua='tests/lua/q-form.lua'), "Requests/second" ) + if 1: + + print("") + print(" One by one") + print (" Hello ", await run_wrk(loop, 'http://localhost:8080/'), "Requests/second" ) + print (" Hello hdrs ", await run_wrk(loop, 'http://localhost:8080/', options=more_headers), "Requests/second" ) + print (" Cookies ", await run_wrk(loop, 'http://localhost:8080/printCookies', options=opts), "Requests/second" ) + print (" many args ", await run_wrk(loop, 'http://localhost:8080/sixargs/one/two/three/four/five/six'), "Requests/second" ) + print (" 404 natural ", await run_wrk(loop, 'http://localhost:8080/dfads404/'), "Requests/second" ) + print (" 404 ", await run_wrk(loop, 'http://localhost:8080/404/'), "Requests/second" ) + print (" Form parsing ", await run_wrk(loop, 'http://localhost:8080/form',lua='tests/lua/form.lua'), "Requests/second" ) + #print (" Templates ", await run_wrk(loop, 'http://localhost:8080/template'), "Requests/second" ) + print (" mrpacker ", await run_wrk(loop,'http://localhost:8080/mrpacker',lua='tests/lua/mrpacker.lua'), "Requests/second" ) + print (" Sessions ", await run_wrk(loop, 'http://localhost:8080/s', options=opts), "Requests/second" ) + print (" File Upload ", await run_wrk(loop,'http://localhost:8080/upload',lua='tests/lua/upload.lua'), "Requests/second" ) + # Disabled in s_bench.py print ("Sessions (py) ", run_wrk(loop, 'http://localhost:8080/pys', options=opts), "Requests/second" ) + #print (" Session login ", await run_wrk(loop, 'http://localhost:8080/login'), "Requests/second" ) + #print (" json post ", await run_wrk(loop,'http://localhost:8080/json',lua='tests/lua/json.lua'), "Requests/second" ) + #print (" mrpacker py ", await run_wrk(loop,'http://localhost:8080/mrpackerpy',lua='tests/lua/mrpacker.lua'), "Requests/second" ) + #print (" msgpack py ", await run_wrk(loop,'http://localhost:8080/msgpack',lua='tests/lua/msgpack.lua'), "Requests/second" ) + + + opts = ('-H','XX-Real-IP: 1.2.3.4') + print (" get ip ", await run_wrk(loop,'http://localhost:8080/getip',options=opts), "Requests/second" ) + #print ("many num args ", await run_wrk(loop, 'http://localhost:8080/sixargs/155/2001/29999/25/29999543/93243242394'), "Requests/second" ) + #print ("404 ", await run_wrk(loop, 'http://localhost:8080/404/'), "Requests/second" ) + -asyncio.set_event_loop(loop) + except KeyboardInterrupt: + pass + finally: + proc.terminate() + await proc.wait() -server_fut = asyncio.create_subprocess_exec( 'python', 'tests/s_bench.py', stdout=asyncio.subprocess.PIPE ) -server = loop.run_until_complete(server_fut) -process = psutil.Process(server.pid) - -time.sleep(1) -try: - - more_headers = ('-H','User-Agent: Mozilla/5.0 (X11; Linux x86_64) Gecko/20130501 Firefox/30.0 AppleWebKit/600.00 Chrome/30.0.0000.0 Trident/10.0 Safari/600.00', - '-H','Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', - '-H','Accept-Language: en-US,en;q=0.5', - '-H','Connection: keep-alive') - opts = ('-H','Cookie: mrsession=43709dd361cc443e976b05714581a7fb; foo=fdsfdasdfasdfdsfasdfsdfsdfasdfas; short=fazc;') - print ("Hello ", run_wrk(loop, 'http://localhost:8080/'), "Requests/second" ) - print ("Sessions ", run_wrk(loop, 'http://localhost:8080/s', options=opts), "Requests/second" ) - if 0: - print ("Hello pipelined", run_wrk(loop, 'http://localhost:8080/',lua='tests/lua/pipeline.lua'), "Requests/second" ) - print ("More hdrs pipelined", run_wrk(loop, 'http://localhost:8080/',options=more_headers,lua='tests/lua/pipeline.lua'), "Requests/second" ) - print ("Hello ", run_wrk(loop, 'http://localhost:8080/'), "Requests/second" ) - print ("Hello hdrs ", run_wrk(loop, 'http://localhost:8080/', options=more_headers), "Requests/second" ) - - #print ("Cookies ", run_wrk(loop, 'http://localhost:8080/printCookies', options=opts), "Requests/second" ) - #print ("many args ", run_wrk(loop, 'http://localhost:8080/sixargs/one/two/three/four/five/six'), "Requests/second" ) - #print ("404 natural ", run_wrk(loop, 'http://localhost:8080/dfads404/'), "Requests/second" ) - #print ("404 ", run_wrk(loop, 'http://localhost:8080/404/'), "Requests/second" ) - #print ("Form parsing ", run_wrk(loop, 'http://localhost:8080/form',lua='tests/lua/form.lua'), "Requests/second" ) - #print ("Templates ", run_wrk(loop, 'http://localhost:8080/template'), "Requests/second" ) - #print ("mrpacker ", run_wrk(loop,'http://localhost:8080/mrpacker',lua='tests/lua/mrpacker.lua'), "Requests/second" ) - #print ("Sessions ", run_wrk(loop, 'http://localhost:8080/s', options=opts), "Requests/second" ) - # Disabled in s_bench.py print ("Sessions (py) ", run_wrk(loop, 'http://localhost:8080/pys', options=opts), "Requests/second" ) - #print ("Session login ", run_wrk(loop, 'http://localhost:8080/login'), "Requests/second" ) - #print ("json post ", run_wrk(loop,'http://localhost:8080/json',lua='tests/lua/json.lua'), "Requests/second" ) - #print ("mrpacker py ", run_wrk(loop,'http://localhost:8080/mrpackerpy',lua='tests/lua/mrpacker.lua'), "Requests/second" ) - #print ("msgpack py ", run_wrk(loop,'http://localhost:8080/msgpack',lua='tests/lua/msgpack.lua'), "Requests/second" ) +async def main(): + print("main") + await run_tests() + await run_benchmarks() - - opts = ('-H','XX-Real-IP: 1.2.3.4') - #print ("get ip ", run_wrk(loop,'http://localhost:8080/getip',options=opts), "Requests/second" ) - #print ("many num args ", run_wrk(loop, 'http://localhost:8080/sixargs/155/2001/29999/25/29999543/93243242394'), "Requests/second" ) - #print ("404 ", run_wrk(loop, 'http://localhost:8080/404/'), "Requests/second" ) - - # Grab the stdout for debug - if 0: - lines = [] - x = 0 - while 1: - x += 1 - print(x) - #if x > 19842: break - if x > 21605: break - line = loop.run_until_complete(server.stdout.readline()) - if line: - line = line.decode('utf-8') - lines.append(line) - else: - break - print ( len(lines) ) - o = open( "wrkout", "wb" ) - o.write( (''.join(lines)).encode("utf-8") ) - o.close() - -except KeyboardInterrupt: - pass -finally: - server.terminate() - loop.run_until_complete(server.wait()) +loop = uvloop.new_event_loop() +asyncio.set_event_loop(loop) +asyncio.run( main() ) diff --git a/dotests.py.old b/dotests.py.old new file mode 100644 index 0000000..81a5e3b --- /dev/null +++ b/dotests.py.old @@ -0,0 +1,189 @@ + +readme = """ + pip install psutil requests msgpack mrasyncmc tenjin mrpacker + mrcache and mrworkserver must be running for the benchmarks +""" + + +import pkgutil, time +import inspect +import types +import importlib + +import tests + +# TODO +# Check for mrworkserver and mrcache being up and add the session key so we hit and load the json 43709dd361cc443e976b05714581a7fb +# mrcache -m 64 -i 16 +# python mrworkserver/tst.py + + + +if 1: + package = tests + for importer, modname, ispkg in pkgutil.iter_modules(package.__path__): + if modname.startswith("test"): + m = importlib.import_module('tests.'+modname) + functions = inspect.getmembers(m, inspect.isfunction) + for f in functions: + if f[0] == 'setup': + if f[1](): + exit() + for f in functions: + if f[0].startswith("test_"): + try: + f[1]() + except Exception as e: + print(e) + for f in functions: + if f[0] == 'teardown': + f[1]() + +print("Benchmarks") + +import argparse +import sys +import asyncio +import os +from asyncio.subprocess import PIPE, STDOUT +import statistics + +import uvloop +import psutil +import atexit + +#from misc import cpu + + +def run_wrk(loop, endpoint=None, lua=None, options=None): + rps = 0 + try: + endpoint = endpoint or 'http://localhost:8080/' + if lua: + if options != None: + wrk_fut = asyncio.create_subprocess_exec( 'wrk', '-t', '4', '-c', '32', '-d', '2', '-s', lua, *options, endpoint, stdout=PIPE, stderr=STDOUT) + else: + wrk_fut = asyncio.create_subprocess_exec( 'wrk', '-t', '4', '-c', '32', '-d', '2', '-s', lua, endpoint, stdout=PIPE, stderr=STDOUT) + else: + if options != None: + wrk_fut = asyncio.create_subprocess_exec( 'wrk', '-t', '4', '-c', '32', '-d', '2', *options, endpoint, stdout=PIPE, stderr=STDOUT) + else: + wrk_fut = asyncio.create_subprocess_exec( 'wrk', '-t', '4', '-c', '32', '-d', '2', endpoint, stdout=PIPE, stderr=STDOUT) + + wrk = loop.run_until_complete(wrk_fut) + rps = 0 + lines = [] + while 1: + line = loop.run_until_complete(wrk.stdout.readline()) + if line: + line = line.decode('utf-8') + lines.append(line) + if line.startswith('Requests/sec:'): + rps = float(line.split()[-1]) + else: + break + + retcode = loop.run_until_complete(wrk.wait()) + if retcode != 0: + print('\r\n'.join(lines)) + except Exception as e: + print(e) + + + return rps + + +noisy = ['atom', 'chrome', 'firefox', 'dropbox', 'opera', 'spotify', 'gnome-documents'] + +def silence(): + for proc in psutil.process_iter(): + if proc.name() in noisy: + proc.suspend() + + def resume(): + for proc in psutil.process_iter(): + if proc.name() in noisy: + proc.resume() + atexit.register(resume) + +silence() + +loop = uvloop.new_event_loop() + +asyncio.set_event_loop(loop) + +server_fut = asyncio.create_subprocess_exec( 'python', 'tests/s_bench.py', stdout=asyncio.subprocess.PIPE ) +proc = loop.run_until_complete(server_fut) +process = psutil.Process(proc.pid) + +# Can't get this to work when the server fails to start +#retcode = loop.run_until_complete(proc.wait()) +#print(retcode) +#exit() +#print(dir(proc)) +#time.sleep(5) +#print( proc.pid in psutil.pids()) +#print(proc.returncode) +#if proc.returncode != None: + #print( "YAY") + #exit() +try: + + more_headers = ('-H','User-Agent: Mozilla/5.0 (X11; Linux x86_64) Gecko/20130501 Firefox/30.0 AppleWebKit/600.00 Chrome/30.0.0000.0 Trident/10.0 Safari/600.00', + '-H','Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', + '-H','Accept-Language: en-US,en;q=0.5', + '-H','Connection: keep-alive') + opts = ('-H','Cookie: mrsession=43709dd361cc443e976b05714581a7fb; foo=fdsfdasdfasdfdsfasdfsdfsdfasdfas; short=fazc;') + if 1: + print ("Hello pipelined", run_wrk(loop, 'http://localhost:8080/',lua='tests/lua/pipeline.lua'), "Requests/second" ) + print ("More hdrs pipelined", run_wrk(loop, 'http://localhost:8080/',options=more_headers,lua='tests/lua/pipeline.lua'), "Requests/second" ) + #print ("Hello ", run_wrk(loop, 'http://localhost:8080/'), "Requests/second" ) + #print ("Hello hdrs ", run_wrk(loop, 'http://localhost:8080/', options=more_headers), "Requests/second" ) + + #print ("Cookies ", run_wrk(loop, 'http://localhost:8080/printCookies', options=opts), "Requests/second" ) + #print ("many args ", run_wrk(loop, 'http://localhost:8080/sixargs/one/two/three/four/five/six'), "Requests/second" ) + #print ("404 natural ", run_wrk(loop, 'http://localhost:8080/dfads404/'), "Requests/second" ) + #print ("404 ", run_wrk(loop, 'http://localhost:8080/404/'), "Requests/second" ) + #print ("Form parsing ", run_wrk(loop, 'http://localhost:8080/form',lua='tests/lua/form.lua'), "Requests/second" ) + #print ("Templates ", run_wrk(loop, 'http://localhost:8080/template'), "Requests/second" ) + #print ("mrpacker ", run_wrk(loop,'http://localhost:8080/mrpacker',lua='tests/lua/mrpacker.lua'), "Requests/second" ) + #print ("Sessions ", run_wrk(loop, 'http://localhost:8080/s', options=opts), "Requests/second" ) + # Disabled in s_bench.py print ("Sessions (py) ", run_wrk(loop, 'http://localhost:8080/pys', options=opts), "Requests/second" ) + #print ("Session login ", run_wrk(loop, 'http://localhost:8080/login'), "Requests/second" ) + #print ("json post ", run_wrk(loop,'http://localhost:8080/json',lua='tests/lua/json.lua'), "Requests/second" ) + #print ("mrpacker py ", run_wrk(loop,'http://localhost:8080/mrpackerpy',lua='tests/lua/mrpacker.lua'), "Requests/second" ) + #print ("msgpack py ", run_wrk(loop,'http://localhost:8080/msgpack',lua='tests/lua/msgpack.lua'), "Requests/second" ) + + + opts = ('-H','XX-Real-IP: 1.2.3.4') + #print ("get ip ", run_wrk(loop,'http://localhost:8080/getip',options=opts), "Requests/second" ) + print ("many num args ", run_wrk(loop, 'http://localhost:8080/sixargs/155/2001/29999/25/29999543/93243242394'), "Requests/second" ) + #print ("404 ", run_wrk(loop, 'http://localhost:8080/404/'), "Requests/second" ) + + # Grab the stdout for debug + if 0: + lines = [] + x = 0 + while 1: + x += 1 + print(x) + #if x > 19842: break + if x > 21605: break + line = loop.run_until_complete(proc.stdout.readline()) + if line: + line = line.decode('utf-8') + lines.append(line) + else: + break + print ( len(lines) ) + print(lines) + #o = open( "wrkout", "wb" ) + #o.write( (''.join(lines)).encode("utf-8") ) + #o.close() + +except KeyboardInterrupt: + pass +finally: + proc.terminate() + loop.run_until_complete(proc.wait()) + diff --git a/examples/10_multiplefiles.py b/examples/10_multiplefiles.py index aa34a68..6ee9ac0 100644 --- a/examples/10_multiplefiles.py +++ b/examples/10_multiplefiles.py @@ -9,6 +9,6 @@ def hello(request): app.run(cores=2) # curl -i --raw 'http://localhost:8080/' -# /tst was added in multiplefiles.py +# /tst was added in multiplefiles.py # curl -i --raw 'http://localhost:8080/tst' diff --git a/examples/9_mrworkserver.py b/examples/9_mrworkserver.py index ba6a9b6..af65bbd 100755 --- a/examples/9_mrworkserver.py +++ b/examples/9_mrworkserver.py @@ -1,4 +1,6 @@ +# TODO Not sure we support json here at the moment. Need to change to mrpacker or support both +# # What is this? # # Json is posted to mrhttp and forwarded to a mrworkserver cluster using C code. This is much faster than doing it using python @@ -11,6 +13,10 @@ # wait. By gathering messages in mrworkserver I'm able to process many related messages # together to speed up msg handling # +# Note - /sq/ will use the user id in the session key to decide which server in the cluster gets the message. +# /q/{} will use the path arg to choose the cluster server - so /q/{thread_id} with 2 servers +# would have the even ids go to server 0 and odd to server 1 +# # Setup the memcached and mrq servers first: # # memcached -l 127.0.0.1 -p 11211 -d -m 50 @@ -20,6 +26,7 @@ # python tst.py 7101 # # curl -i --raw http://localhost:8080/q/0/0/ -X POST -d '{"username":"xyz"}' +# curl -H "Content-Type: application/mrpacker" --data-binary @tests/lua/test.mrp http://localhost:8080/q2/0 # # To fetch a user's session and pass it to mrworkserver: # @@ -35,7 +42,10 @@ app = mrhttp.Application() app.config["memcache"] = [ ("127.0.0.1", 11211) ] -app.config["mrq"] = [("127.0.0.1",7100),("127.0.0.1",7101)] + +# We setup 2 clusters of 1 server each +app.config["mrq"] = [("127.0.0.1",7100)] +app.config["mrq2"] = [("127.0.0.1",7101)] @app.route('/q/{}/{}/',options=['mrq']) def queue(r, s, t): @@ -58,6 +68,13 @@ def session_queue(r): if r.servers_down: return "Servers not available, try again later" return 'Hello World!' + +@app.route('/q2/{}',options=['mrq2']) +def queue2(r, some_id): + if r.servers_down: + return "Servers not available, try again later" + return 'Hello World!' + app.run(cores=1) diff --git a/gbench/bld b/gbench/bld new file mode 100755 index 0000000..72ba4a5 --- /dev/null +++ b/gbench/bld @@ -0,0 +1,7 @@ +# +#g++ t.cpp -O3 -msse4.2 -mavx2 -std=c++11 -lbenchmark -lpthread -o t +#g++ tst.cpp -O3 -msse4.2 -mavx2 -std=c++11 -lbenchmark -lpthread -o tst +g++ parse.cpp -O3 -msse4.2 -mavx2 -std=c++11 -lbenchmark -lpthread -o parse +#g++ string.cpp -O0 -msse4.2 -mavx2 -std=c++11 -lbenchmark -lpthread -o string +#g++ query.cpp -O0 -msse4.2 -mavx2 -std=c++11 -lbenchmark -lpthread -o query + diff --git a/gbench/parse.cpp b/gbench/parse.cpp index d95e618..acbea6a 100644 --- a/gbench/parse.cpp +++ b/gbench/parse.cpp @@ -1,8 +1,8 @@ - #include #include #include +#include #include #ifdef __AVX2__ #include @@ -18,8 +18,26 @@ #define unlikely(x) (x) #endif +#ifdef _MSC_VER +#define ALIGNED(n) _declspec(align(n)) +#else +#define ALIGNED(n) __attribute__((aligned(n))) +#endif + + #define IS_PRINTABLE_ASCII(c) ((unsigned char)(c)-040u < 0137u) +#define CHAR4_TO_INT(a, b, c, d) \ + (unsigned int)((d << 24) | (c << 16) | (b << 8) | a) + + +#define CHECK_END() \ + if (buf == buf_end) { \ + *ret = -2; \ + return NULL; \ + } + + #define CHECK_EOF() \ if (buf == buf_end) { \ *ret = -2; \ @@ -36,6 +54,7 @@ CHECK_EOF(); \ EXPECT_CHAR_NO_CHECK(ch); + // Table for converting to lower case #define TOLC(c) __lct[(unsigned char)c] static const unsigned char __lct[] __attribute__((aligned(64))) = { @@ -133,6 +152,33 @@ static const char *findchar(const char *buf, const char *buf_end, const char *ra } return buf; } +static const char *adv_token(const char *buf, int *ret) { + const char *tok_start = buf; + const char *buf_end = buf+512; + static const char ranges2[] = "\000\042\177\177"; + int found2; + buf = findchar(buf, buf+512, ranges2, sizeof(ranges2) - 1, &found2); + if (!found2) { + CHECK_END(); + } else if ( unlikely(*buf != ' ' )) { + *ret = -1; + return NULL; + } + while (1) { + if (*buf == ' ') { + return buf; + } else if (unlikely(!IS_PRINTABLE_ASCII(*buf))) { + if ((unsigned char)*buf < '\040' || *buf == '\177') { + *ret = -1; + return NULL; + } + } + ++buf; + CHECK_END(); + } + *ret = buf - tok_start; + return tok_start; +} static const char *get_token_to_eol(const char *buf, const char *buf_end, int *ret) { @@ -169,6 +215,164 @@ static const char *get_token_to_eol(const char *buf, const char *buf_end, int *r return buf; } +static inline int getSession( const char *buf, size_t buflen ) { + const char *end = buf + buflen; + const char *last = buf; + const char *ses; + int len; + + static char ALIGNED(16) ranges1[] = "==" ";;"; + int found; + int state = 0; + do { + last = buf; + buf = findchar(buf, end, ranges1, sizeof(ranges1) - 1, &found); + if ( found ) { + if ( *buf == '=' ) { + //printf( " fnd >%.*s<\n", buf-last, last ); + if ( state == 0 ) { + // Save out the mrsession id + if ( buf-last == 9 && ( *((unsigned int *)(last)) == CHAR4_TO_INT('m', 'r', 's','e') ) ) { + state = 1; + } + buf+=1; + } + } + else if ( *buf == ';' ) { + //printf( " fnd >%.*s<\n", buf-last, last ); + if (state == 1 ) { + ses = last; + len = buf-last; + return len; + } + state = 0; + buf+=1; + while ( *buf == 32 ) buf++; + } + } + } while( found ); + if (state) { + ses = last; + len = buf-last; + return len; + } + return -1; +} +static const char *my_get_eol128(const char *buf) { + //__m128i* pSrc1 = (__m128i *)string; // init pointer to start of string + __m128i m0 = _mm_set1_epi8(13); // vector of 16 `\0` characters + + while (1) + { + __m128i v0 = _mm_loadu_si128((const __m128i *)buf); + __m128i v1 = _mm_cmpeq_epi8(v0, m0); // compare all 16 chars + unsigned int vmask = _mm_movemask_epi8(v1); // get 16 comparison result bits + if (vmask != 0) { + buf += TZCNT(vmask) + 2; + break; // we found a `\0`, break out of loop + } + buf += 16; //pSrc1++; // next 16 characters... + } + return buf; +} + + //64bits 256bits bytes 8 * 32 +static const char *my_get_eol(const char *buf, const char *buf_end) { + + __m256i m13 = _mm256_set1_epi8(13); + while (1) + { + __m256i v0 = _mm256_loadu_si256((const __m256i *)buf); + __m256i v1 = _mm256_cmpeq_epi8(v0, m13); + unsigned long vmask = _mm256_movemask_epi8(v1); + if (vmask != 0) { + buf += TZCNT(vmask) + 2; + if ( buf > buf_end ) return NULL; + break; + } + buf += 32; //pSrc1++; + if ( buf > buf_end ) return NULL; + } + return buf; +} + __m256i m32 = _mm256_set1_epi8(32); +static const char *get_to_space(const char *buf, int *len) { + const char *orig = buf; + while (1) + { + __m256i v0 = _mm256_loadu_si256((const __m256i *)buf); + __m256i v1 = _mm256_cmpeq_epi8(v0, m32); + unsigned long vmask = _mm256_movemask_epi8(v1); + if (vmask != 0) { + buf += TZCNT(vmask) + 1; + break; + } + buf += 32; + } + *len = buf-orig-1; + return buf; +} + +__m256i m59 = _mm256_set1_epi8(59); +__m256i m61 = _mm256_set1_epi8(61); +static int getSession_avx2( const char* buf, const char* buf_end ) { + unsigned int msk; + int i=0,tz; // 32B index + int cnt = 0; + unsigned int shifted; + const char *sbuf = buf; + const char *obuf = buf; + int name_or_value = 0; + int found = 0; + + do { + const char *block_start = obuf+32*i; + __m256i b0 = _mm256_loadu_si256((const __m256i *) block_start); + msk = _mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b0, m59), _mm256_cmpeq_epi8(b0, m61) ) ); + while (1) { + + //if ( buf >= buf_end ) { goto sesdone; } + shifted = buf-block_start; + if ( shifted >= 32 ) break; + tz = TZCNT((msk >> shifted)); + if ( tz < 32 ) { + buf += tz; + //printf( " fnd >%.*s<\n", buf-sbuf, sbuf ); + if ( buf >= buf_end ) { goto sesdone; } + if ( name_or_value == 1 ) { + if ( *buf == '=' ) { buf += 1; continue; } // = in value field + if ( found ) { + //printf( " done >%.*s<\n", buf-sbuf, sbuf ); + return buf-sbuf; + } + buf+=1; + name_or_value = 0; + } else { + if ( buf-sbuf == 9 && ( *((unsigned int *)(sbuf)) == CHAR4_TO_INT('m', 'r', 's','e') ) ) { + found = 1; + } + name_or_value = 1; + } + buf += 1; + sbuf = buf; + } else { + buf += 32 - shifted; + break; + } + + } + i+=1; + if ( buf >= buf_end ) { goto sesdone; } + } while ( buf-obuf < buf_end-obuf ); + +sesdone: + if ( found ) { + //printf( " sesdone >%.*s<\n", buf-sbuf, sbuf ); + return buf-sbuf; + } + return 0; +} + static const char *parse_headers(const char *buf, const char *buf_end, int *ret) { @@ -227,7 +431,8 @@ static const char *parse_headers(const char *buf, const char *buf_end, int *ret) //if ( buf[0] == 'a' && buf[13] == 'r' ) { //"application/mrpacker" //mrr->flags = 2; //} - buf = get_token_to_eol(buf, buf_end, ret); + //buf = get_token_to_eol(buf, buf_end, ret); + buf = my_get_eol(buf, buf_end); goto skipvalue; } if ( buf[13] == ':' ) { // Cache-Control: @@ -247,7 +452,8 @@ static const char *parse_headers(const char *buf, const char *buf_end, int *ret) //headers[*num_headers].name_len = 16; buf += 18; //mrr->ip = buf; - buf = get_token_to_eol(buf, buf_end, ret); + //buf = get_token_to_eol(buf, buf_end, ret); + buf = my_get_eol(buf, buf_end); //mrr->ip_len = headers[*num_headers].value_len; goto skipvalue; } @@ -274,7 +480,8 @@ static const char *parse_headers(const char *buf, const char *buf_end, int *ret) //headers[*num_headers].name_len = 9; buf += 11; //mrr->ip = buf; - buf = get_token_to_eol(buf, buf_end, ret); + //buf = get_token_to_eol(buf, buf_end, ret); + buf = my_get_eol(buf, buf_end); //mrr->ip_len = headers[*num_headers].value_len; goto skipvalue; } @@ -441,7 +648,8 @@ static const char *parse_headers(const char *buf, const char *buf_end, int *ret) //headers[*num_headers].name_len = 0; } hvalue: - if ((buf = get_token_to_eol(buf, buf_end, ret)) == NULL) { + //if ((buf = get_token_to_eol(buf, buf_end, ret)) == NULL) { + if ((buf = my_get_eol(buf, buf_end)) == NULL) { return NULL; } skipvalue: @@ -589,202 +797,79 @@ static void find_ranges(const char* buf, const char* buf_end, unsigned long *ran static const char* parse_headers_avx2(const char* buf, const char* buf_end, int* ret) { - int num_headers = 0; - int max_headers = 20; - /* Bitmap for the first type of tokens */ - unsigned long rr0[2] = {0}; - /* Bitmap for the second type of tokens */ - unsigned long rr1[2] = {0}; - /* Pointer to the start of the currently parsed block of 128 bytes */ - const char* prep_start = NULL; - int found; - int n_headers = num_headers; - - for (; ; ++n_headers) { - CHECK_EOF(); - if (*buf == '\015') { - ++buf; - EXPECT_CHAR('\012'); - break; - } else if (*buf == '\012') { - ++buf; - break; - } - if (n_headers == max_headers) { - *ret = -1; - num_headers = n_headers; - return NULL; - } + // 128 bit token mask + unsigned long bm[8] = {0}; + // Pointer to the start of the currently parsed block of 128 bytes + const char* prep_start = buf; + const char *p = buf; - if (! (n_headers != 0 && (*buf == ' ' || *buf == '\t')) && !(*buf >= 65 && * buf <= 90)) { - if (! token_char_map[(unsigned char)*buf]) { - *ret = -1; - num_headers = n_headers; - return NULL; - } - //headers[n_headers].name = buf; - - /* Attempt to find a match in the index */ - found = 0; - do { - unsigned long distance = buf - prep_start; - /* Check if the bitmaps are still valid. An assumption I make is that - buf > 128 (i.e. the os will never allocate memory at address 0-128 */ - if(unlikely(distance >= 128)) { /* Bitmaps are too old, make new ones */ - prep_start = buf; - distance = 0; - find_ranges(buf, buf_end, rr0, rr1); - } else if(distance >= 64) { /* In the second half of the bitmap */ - unsigned long index = rr0[1] >> (distance - 64); /* Correct offset of the bitmap */ - unsigned long find = TZCNT(index); /* Fine next set bit */ - if((find < 64)) { /* Yey, we found a token */ - buf += find; - found = 1; - break; - } - buf = prep_start + 128; /* No token was found in the current bitmap */ - continue; - } - unsigned long index = rr0[0] >> (distance); /* In the first half of the bitmap */ - unsigned long find = TZCNT(index); /* Find next set bit */ - if((find < 64)){ /* Token found */ - buf += find; - found = 1; - break; - } /* Token not found, look at second half of bitmap */ - index = rr0[1]; - find = TZCNT(index); - if((find < 64)){ - buf += 64+find - distance; - found = 1; - break; - } - - buf = prep_start + 128; - } while (buf < buf_end); - - if(!found) - if(buf >= buf_end) { - *ret = -2; - //*num_headers = n_headers; - return NULL; - } - //headers[n_headers].name_len = buf - headers[n_headers].name; - ++buf; - CHECK_EOF(); - while( (*buf == ' ' || *buf == '\t') ) { - ++buf; - CHECK_EOF(); - } - } else { - //headers[n_headers].name = NULL; - //headers[n_headers].name_len = 0; + // Load the \r and : mask into rr13 and rr58 + // Load 512 bytes at a time into the bit mask bm[8] + // Load 32 bytes from the buffer into each register and compare against the mask registers + + __m256i b0, b1, b2, b3; + const __m256i rr13 = _mm256_set1_epi8(13); + const __m256i rr58 = _mm256_set1_epi8(58); + int state = 0; + + // Process 512b per loop + while(1) { + int i = 0; + // Load 512b into bm[0-7] + while ( i < 8 ) { + b0 = _mm256_loadu_si256((__m256i *)(buf + (64*i)+ 0)); + b1 = _mm256_loadu_si256((__m256i *)(buf + (64*i)+32)); + b2 = _mm256_loadu_si256((__m256i *)(buf + (64*i)+64)); + b3 = _mm256_loadu_si256((__m256i *)(buf + (64*i)+96)); + bm[i++] = _mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(rr13, b0),_mm256_cmpeq_epi8(rr58, b0)) ) | + ((unsigned long)_mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(rr13, b1),_mm256_cmpeq_epi8(rr58, b1)) ) << 32); + bm[i++] = _mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(rr13, b2),_mm256_cmpeq_epi8(rr58, b2)) ) | + ((unsigned long)_mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(rr13, b3),_mm256_cmpeq_epi8(rr58, b3)) ) << 32); } - const char* token_start = buf; - found = 0; + // Each bit in the mask is either a : or a \r + int off = 0; + int shft = 0; + int bmOff = 0; + unsigned long bitmap, tz; + int slen = 0; //DELME do { - /* Too far */ - unsigned long distance = buf - prep_start; /* Same algorithm as above */ - if(unlikely(distance >= 128)) { - prep_start = buf; - distance = 0; - find_ranges(buf, buf_end, rr0, rr1); - } else if(distance >= 64) { - unsigned long index = rr1[1] >> (distance - 64); - unsigned long find = TZCNT(index); - if((find < 64)) { - buf += find; - found = 1; - break; + bitmap = bm[ bmOff ] >> shft; + tz = TZCNT(bitmap); + if ( tz < 64 ) { // tz is 64 if not found + p += tz; + //printf( " fnd >%.*s<\n", p-buf, buf ); + if ( state == 0 ) { // : + state = 1; + p += 2; buf = p; + } else { // \r + state = 0; + p += 2; buf = p; + if ( *p == '\r' ) goto wedone; } - buf = prep_start + 128; - continue; - } - unsigned long index = rr1[0] >> (distance); - unsigned long find = TZCNT(index); - if((find < 64)){ - buf += find; - found = 1; - break; - } - index = rr1[1]; - find = TZCNT(index); - if((find < 64)){ - buf += 64+find - distance; - found = 1; - break; - } - - buf = prep_start + 128; - } while (buf < buf_end); - - if(!found) - if(buf >= buf_end) { - *ret = -2; - num_headers = n_headers; - return NULL; + } else { + p += 64 - shft; + //printf("DELMEZ %.*s\n", 3, p) ; } - - unsigned short two_char = *(unsigned short*)buf; - - if( likely(two_char == 0x0a0d) ) { - //headers[n_headers].value_len = buf - token_start; - buf += 2; - } else if (unlikely(two_char & 0x0a == 0x0a)) { - //headers[n_headers].value_len = buf - token_start; - ++buf; - } else { - *ret = -1; - num_headers = n_headers; - return NULL; - } - //headers[n_headers].value = token_start; + off = p-prep_start; + //printf("DELME off=%d\n",off); + shft = off&0x3F; + bmOff = off/64; + } while ( bmOff < 8 ) ; + prep_start += 512; + buf = prep_start; } - num_headers = n_headers; - return buf; -} - - -static const char *my_get_eol128(const char *buf) { - //__m128i* pSrc1 = (__m128i *)string; // init pointer to start of string - __m128i m0 = _mm_set1_epi8(13); // vector of 16 `\0` characters - while (1) - { - __m128i v0 = _mm_loadu_si128((const __m128i *)buf); - __m128i v1 = _mm_cmpeq_epi8(v0, m0); // compare all 16 chars - unsigned int vmask = _mm_movemask_epi8(v1); // get 16 comparison result bits - if (vmask != 0) { - buf += TZCNT(vmask) + 2; - break; // we found a `\0`, break out of loop - } - buf += 16; //pSrc1++; // next 16 characters... - } +wedone: +// Host: server\r\n +// User-Agent: Mozilla/5.0 (X11; Linux x86_64) Gecko/20130501 Firefox/30.0 AppleWebKit/600.00 Chrome/30.0.0000.0 Trident/10.0 Safari/600.00\r\n + //printf("%.*s\n", p - prep_start, prep_start); return buf; } - //64bits 256bits bytes 8 * 32 -__m256i m13 = _mm256_set1_epi8(13); -static const char *my_get_eol(const char *buf) { - while (1) - { - __m256i v0 = _mm256_loadu_si256((const __m256i *)buf); - __m256i v1 = _mm256_cmpeq_epi8(v0, m13); // - unsigned long vmask = _mm256_movemask_epi8(v1); - if (vmask != 0) { - buf += TZCNT(vmask) + 2; - break; - } - buf += 32; //pSrc1++; - } - return buf; -} - -//__m256i m13 = _mm256_set1_epi8(13); __m256i m58 = _mm256_set1_epi8(58); // 0x1313131313131313... // 0x32333435363713 // abcdef\r // 32 bit number 0x40 @@ -794,6 +879,7 @@ static void parse_mine( const char* buf ) { //__m256i b0,b1,b2,b3,b4,b5,b6,b7,b8,b9,b10,b11,b12,b13,b14,b15; __m256i b0,b1,b2,b3,b4,b5,b6,b7; + __m256i m13 = _mm256_set1_epi8(13); b0 = _mm256_loadu_si256((const __m256i *) (buf + 32*0)); // buf[0] b1 = _mm256_loadu_si256((const __m256i *) (buf + 32*1)); // buf[32] @@ -889,6 +975,21 @@ static void parse_mine3( const char* buf ) { //__m256i b0,b1,b2,b3,b4,b5,b6,b7,b8,b9,b10,b11,b12,b13,b14,b15; __m256i b0,b1,b2,b3,b4,b5,b6,b7; + __m256i m13 = _mm256_set1_epi8(13); + + const char *obuf = buf; + const char *sbuf = buf; + + int i; // msk[i] + int t; + unsigned int s = 0; + int name_or_value = 0; + + const char *block_start = obuf; + +new512: + i = 0; + buf = obuf; b0 = _mm256_loadu_si256((const __m256i *) (buf + 32*0)); // buf[0] b1 = _mm256_loadu_si256((const __m256i *) (buf + 32*1)); // buf[32] @@ -899,15 +1000,14 @@ static void parse_mine3( const char* buf ) { b6 = _mm256_loadu_si256((const __m256i *) (buf + 32*6)); b7 = _mm256_loadu_si256((const __m256i *) (buf + 32*7)); // 256 bytes - msk[0] = _mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b0, m13), _mm256_cmpeq_epi8(b0, m58) ) ) | - ((unsigned long long) _mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b1, m13), _mm256_cmpeq_epi8(b1, m58) ) ) << 32); - msk[1] = _mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b2, m13), _mm256_cmpeq_epi8(b2, m58) ) ) | - ((unsigned long long) _mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b3, m13), _mm256_cmpeq_epi8(b3, m58) ) ) << 32); - msk[2] = _mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b4, m13), _mm256_cmpeq_epi8(b4, m58) ) ) | - ((unsigned long long) _mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b5, m13), _mm256_cmpeq_epi8(b5, m58) ) ) << 32); - msk[3] = _mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b6, m13), _mm256_cmpeq_epi8(b6, m58) ) ) | - ((unsigned long long) _mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b7, m13), _mm256_cmpeq_epi8(b7, m58) ) ) << 32); - + msk[0] = (unsigned int)_mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b0, m13), _mm256_cmpeq_epi8(b0, m58) ) ) | + ((unsigned long) _mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b1, m13), _mm256_cmpeq_epi8(b1, m58) ) ) << 32); + msk[1] = (unsigned int)_mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b2, m13), _mm256_cmpeq_epi8(b2, m58) ) ) | + ((unsigned long) _mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b3, m13), _mm256_cmpeq_epi8(b3, m58) ) ) << 32); + msk[2] = (unsigned int)_mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b4, m13), _mm256_cmpeq_epi8(b4, m58) ) ) | + ((unsigned long) _mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b5, m13), _mm256_cmpeq_epi8(b5, m58) ) ) << 32); + msk[3] = (unsigned int)_mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b6, m13), _mm256_cmpeq_epi8(b6, m58) ) ) | + ((unsigned long) _mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b7, m13), _mm256_cmpeq_epi8(b7, m58) ) ) << 32); b0 = _mm256_loadu_si256((const __m256i *) (buf + 32*8)); b1 = _mm256_loadu_si256((const __m256i *) (buf + 32*9)); @@ -918,28 +1018,36 @@ static void parse_mine3( const char* buf ) { b6 = _mm256_loadu_si256((const __m256i *) (buf + 32*14)); b7 = _mm256_loadu_si256((const __m256i *) (buf + 32*15)); - msk[4] = _mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b0, m13), _mm256_cmpeq_epi8(b0, m58) ) ) ^ - ((unsigned long) _mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b1, m13), _mm256_cmpeq_epi8(b1, m58) ) ) << 32); - msk[5] = _mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b2, m13), _mm256_cmpeq_epi8(b2, m58) ) ) ^ - ((unsigned long) _mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b3, m13), _mm256_cmpeq_epi8(b3, m58) ) ) << 32); - msk[6] = _mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b4, m13), _mm256_cmpeq_epi8(b4, m58) ) ) ^ - ((unsigned long) _mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b5, m13), _mm256_cmpeq_epi8(b5, m58) ) ) << 32); - msk[7] = _mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b6, m13), _mm256_cmpeq_epi8(b6, m58) ) ) ^ - ((unsigned long) _mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b7, m13), _mm256_cmpeq_epi8(b7, m58) ) ) << 32); + msk[4] = (unsigned int)_mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b0, m13), _mm256_cmpeq_epi8(b0, m58) ) ) ^ + ((unsigned long) _mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b1, m13), _mm256_cmpeq_epi8(b1, m58) ) ) << 32); + msk[5] = (unsigned int)_mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b2, m13), _mm256_cmpeq_epi8(b2, m58) ) ) ^ + ((unsigned long) _mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b3, m13), _mm256_cmpeq_epi8(b3, m58) ) ) << 32); + msk[6] = (unsigned int)_mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b4, m13), _mm256_cmpeq_epi8(b4, m58) ) ) ^ + ((unsigned long) _mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b5, m13), _mm256_cmpeq_epi8(b5, m58) ) ) << 32); + msk[7] = (unsigned int)_mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b6, m13), _mm256_cmpeq_epi8(b6, m58) ) ) ^ + ((unsigned long) _mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b7, m13), _mm256_cmpeq_epi8(b7, m58) ) ) << 32); + - const char *obuf = buf; - const char *sbuf = buf; // "Host: server\r\n" - int i = 0; // msk[i] - int t; do { - const char *block_start = obuf+64*i; + block_start = obuf+64*i; while(1) { - t = TZCNT((msk[i]>>(buf-block_start))); + s = buf-block_start; + t = TZCNT((msk[i]>>s)); + //printf("DELME mski %016llx shift %d\n", msk[i], s ); + //printf("DELME shft %016llx\n", msk[i]>>s ); if ( t < 64 ) { - buf += t+2; + buf += t; + if ( name_or_value == 1 ) { + if ( *buf == ':' ) { buf += 1; continue; } // : in value field + name_or_value = 0; + } else { + name_or_value = 1; + } + //printf( " fnd >%.*s<\n", buf-sbuf, sbuf ); + buf += 2; if ( *buf == '\r' ) break; // \r\n\r\n marks the end sbuf = buf; if ( (buf-block_start)> 64 ) break; // TODO? } else { @@ -948,42 +1056,61 @@ static void parse_mine3( const char* buf ) { } } - - i+=1; + + i+=1; + if ( buf[0] == '\r' ) goto done; } while ( i < 8 && buf[0] != '\r' ); - - + + obuf += 512; + goto new512; +done: + i += 1; } -static void parse_mine2( const char* buf ) { +static void parse_mine2( const char* buf, const char *buf_end ) { unsigned long msk; - int i=0,t; // 32B index - int cnt = 0; + int i=0, tz; // 32B index + unsigned int shifted; const char *sbuf = buf; const char *obuf = buf; + int name_or_value = 0; + __m256i m13 = _mm256_set1_epi8(13); + do { - int shifted = 0; - const char *block_start = obuf+32*i; + const char *block_start = obuf+64*i; __m256i b0 = _mm256_loadu_si256((const __m256i *) block_start); - msk = _mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b0, m13), _mm256_cmpeq_epi8(b0, m58) ) ); + __m256i b1 = _mm256_loadu_si256((const __m256i *) (block_start+32)); + msk = (unsigned int) _mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b0, m13), _mm256_cmpeq_epi8(b0, m58) ) ) | + ((unsigned long) _mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b1, m13), _mm256_cmpeq_epi8(b1, m58) ) ) << 32); + //const char *block_start = obuf+32*i; + //__m256i b0 = _mm256_loadu_si256((const __m256i *) block_start); + //msk = _mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b0, m13), _mm256_cmpeq_epi8(b0, m58) ) ); while (1) { shifted = buf-block_start; - t = TZCNT((msk >> shifted)); - if ( t < 32 ) { - if ( t == 0 ) break; - buf += t+2; + if ( shifted >= 64 ) break; + tz = TZCNT((msk >> shifted)); + if ( tz < 64 ) { + buf += tz; + + if ( name_or_value == 1 ) { + if ( *buf == ':' ) { buf += 1; continue; } // : in value field + name_or_value = 0; + } else { + name_or_value = 1; + } + //printf( " fnd >%.*s<\n", buf-sbuf, sbuf ); + buf += 2; if ( *buf == '\r' ) return; // \r\n\r\n marks the end sbuf = buf; } else { - buf = block_start + 32; + buf += 64 - shifted; break; } } - i+=1; - cnt += 1; - } while ( cnt < 20 && buf[0] != '\r' ); + } while ( buf < buf_end); + //} while ( *buf != '\r' ); } static void parse_sse4( const char* buf ) { @@ -995,16 +1122,30 @@ static void parse_sse4( const char* buf ) { static void parse_mysse4( const char* buf ) { int ret = 0; while ( ret == 0 && buf != NULL && buf[0] != '\r' ) { - buf = my_get_eol( buf ); + buf = my_get_eol( buf, buf+512 ); } } +static char buf[8096] = "Host: server\r\n" +"User-Agent: Mozilla/5.0 (X11; Linux x86_64) Gecko/20130501 Firefox/30.0 AppleWebKit/600.00 Chrome/30.0.0000.0 Trident/10.0 Safari/600.00\r\n" +"Cookie: uid=12345678901234567890; __utma=1.1234567890.1234567890.1234567890.1234567890.12; wd=2560x1600\r\n" +"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,* /*;q=0.8\r\n" +"Accept-Language: en-US,en;q=0.5\r\n" +"Connection: keep-alive\r\n\r\n"; +static char buf2[8096] = "Host: localhost:8080\r\nUser-Agent: python-requests/2.31.0\r\nAccept-Encoding: gzip, deflate\r\nAccept: * /*\r\nConnection: keep-alive\r\nCookie: foo=b=ar\r\nContent-Length: 0\r\n\r\n"; +static char path[8096] = "/foo/bar/bazfdasfffffffffffffffffffffffffffffffffffffffdfffffffffffffffffffffffffffffffffffffffffffffffffff "; + +//static char cbuf[8096] = "uid=123456781234567890; mrsession=1234567890.1234567890.12; wd=2560x1600"; +//static int clen= strlen("uid=123456781234567890; mrsession=1234567890.1234567890.12; wd=2560x1600"); +static char cbuf[8096] = "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa; mrsession=1234567890.1234567890.12; xxxxxxxxxxxxxxxxxxxxxxxxx=zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz"; +static int clen= strlen("zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa; mrsession=1234567890.1234567890.12; xxxxxxxxxxxxxxxxxxxxxxxxx=zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz"); +static const char *cend = cbuf + clen; static void BM_SlowParse(benchmark::State& state) { // Perform setup here std::string text = "Host: server\n" "User-Agent: Mozilla/5.0 (X11; Linux x86_64) Gecko/20130501 Firefox/30.0 AppleWebKit/600.00 Chrome/30.0.0000.0 Trident/10.0 Safari/600.00\n" "Cookie: uid=12345678901234567890; __utma=1.1234567890.1234567890.1234567890.1234567890.12; wd=2560x1600\n" -"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\n" +"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,* /*;q=0.8\n" "Accept-Language: en-US,en;q=0.5\n" "Connection: keep-alive\n"; @@ -1015,13 +1156,6 @@ static void BM_SlowParse(benchmark::State& state) { } static void BM_sse4_get_eol(benchmark::State& state) { // Perform setup here - char buf[8096] = "Host: server\r\n" -"User-Agent: Mozilla/5.0 (X11; Linux x86_64) Gecko/20130501 Firefox/30.0 AppleWebKit/600.00 Chrome/30.0.0000.0 Trident/10.0 Safari/600.00\r\n" -"Cookie: uid=12345678901234567890; __utma=1.1234567890.1234567890.1234567890.1234567890.12; wd=2560x1600\r\n" -"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\r\n" -"Accept-Language: en-US,en;q=0.5\r\n" -"Connection: keep-alive\r\n\r\n\r\n\r\n\r\n"; - for (auto _ : state) { // This code gets timed parse_sse4(buf); @@ -1029,13 +1163,6 @@ static void BM_sse4_get_eol(benchmark::State& state) { } static void BM_my_get_eol(benchmark::State& state) { // Perform setup here - char buf[8096] = "Host: server\r\n" -"User-Agent: Mozilla/5.0 (X11; Linux x86_64) Gecko/20130501 Firefox/30.0 AppleWebKit/600.00 Chrome/30.0.0000.0 Trident/10.0 Safari/600.00\r\n" -"Cookie: uid=12345678901234567890; __utma=1.1234567890.1234567890.1234567890.1234567890.12; wd=2560x1600\r\n" -"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\r\n" -"Accept-Language: en-US,en;q=0.5\r\n" -"Connection: keep-alive\r\n\r\n\r\n\r\n\r\n"; - for (auto _ : state) { // This code gets timed parse_mysse4(buf); @@ -1043,99 +1170,116 @@ static void BM_my_get_eol(benchmark::State& state) { } static void BM_my_header_parse(benchmark::State& state) { - // Perform setup here - char buf[8096] = "Host: server\r\n" -"User-Agent: Mozilla/5.0 (X11; Linux x86_64) Gecko/20130501 Firefox/30.0 AppleWebKit/600.00 Chrome/30.0.0000.0 Trident/10.0 Safari/600.00\r\n" -"Cookie: uid=12345678901234567890; __utma=1.1234567890.1234567890.1234567890.1234567890.12; wd=2560x1600\r\n" -"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\r\n" -"Accept-Language: en-US,en;q=0.5\r\n" -"Connection: keep-alive\r\n\r\nzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz " -" " -" "; - for (auto _ : state) { - // This code gets timed parse_mine(buf); } } static void BM_my2_header_parse(benchmark::State& state) { - // Perform setup here - char buf[8096] = "Host: server\r\n" -"User-Agent: Mozilla/5.0 (X11; Linux x86_64) Gecko/20130501 Firefox/30.0 AppleWebKit/600.00 Chrome/30.0.0000.0 Trident/10.0 Safari/600.00\r\n" -"Cookie: uid=12345678901234567890; __utma=1.1234567890.1234567890.1234567890.1234567890.12; wd=2560x1600\r\n" -"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\r\n" -"Accept-Language: en-US,en;q=0.5\r\n" -"Connection: keep-alive\r\n\r\nzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz " -" " -" "; - for (auto _ : state) { - // This code gets timed - parse_mine2(buf); + parse_mine2(buf,buf+2048); } } static void BM_my3_header_parse(benchmark::State& state) { - // Perform setup here - char buf[8096] = "Host: server\r\n" -"User-Agent: Mozilla/5.0 (X11; Linux x86_64) Gecko/20130501 Firefox/30.0 AppleWebKit/600.00 Chrome/30.0.0000.0 Trident/10.0 Safari/600.00\r\n" -"Cookie: uid=12345678901234567890; __utma=1.1234567890.1234567890.1234567890.1234567890.12; wd=2560x1600\r\n" -"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\r\n" -"Accept-Language: en-US,en;q=0.5\r\n" -"Connection: keep-alive\r\n\r\nzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz " -" " -" "; - for (auto _ : state) { - // This code gets timed parse_mine3(buf); } } static void BM_old_header_parse(benchmark::State& state) { - // Perform setup here - char buf[8096] = "Host: server\r\n" -"User-Agent: Mozilla/5.0 (X11; Linux x86_64) Gecko/20130501 Firefox/30.0 AppleWebKit/600.00 Chrome/30.0.0000.0 Trident/10.0 Safari/600.00\r\n" -"Cookie: uid=12345678901234567890; __utma=1.1234567890.1234567890.1234567890.1234567890.12; wd=2560x1600\r\n" -"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\r\n" -"Accept-Language: en-US,en;q=0.5\r\n" -"Connection: keep-alive\r\n\r\nzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz " -" " -" "; int ret = 0; for (auto _ : state) { - // This code gets timed - parse_headers(buf,buf+512,&ret); + parse_headers(buf,buf+2048,&ret); } } static void BM_avx2_header_parse(benchmark::State& state) { - // Perform setup here - char buf[8096] = "Host: server\r\n" -"User-Agent: Mozilla/5.0 (X11; Linux x86_64) Gecko/20130501 Firefox/30.0 AppleWebKit/600.00 Chrome/30.0.0000.0 Trident/10.0 Safari/600.00\r\n" -"Cookie: uid=12345678901234567890; __utma=1.1234567890.1234567890.1234567890.1234567890.12; wd=2560x1600\r\n" -"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\r\n" -"Accept-Language: en-US,en;q=0.5\r\n" -"Connection: keep-alive\r\n\r\nzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz " -" " -" "; int ret = 0; for (auto _ : state) { - // This code gets timed - parse_headers_avx2(buf,buf+512,&ret); + parse_headers_avx2(buf,buf+2048,&ret); } } +static void BM_adv_token(benchmark::State& state) { + int ret = 0; + int path_len = 0; + for (auto _ : state) { + adv_token(path, &path_len); + } +} +static void BM_adv_token_avx2(benchmark::State& state) { + int ret = 0; + int path_len = 0; + for (auto _ : state) { + get_to_space(path, &path_len); + } +} +static void BM_get_session(benchmark::State& state) { + int ret = 0; + for (auto _ : state) { + getSession(cbuf, clen); + } +} +static void BM_get_session_avx2(benchmark::State& state) { + int ret = 0; + for (auto _ : state) { + getSession_avx2(cbuf, cend); + } +} + + + + //BENCHMARK(BM_SlowParse); -//BENCHMARK(BM_sse4_get_eol); -//BENCHMARK(BM_my_get_eol); -BENCHMARK(BM_my3_header_parse); -BENCHMARK(BM_my2_header_parse); +BENCHMARK(BM_sse4_get_eol); +BENCHMARK(BM_my_get_eol); +//BENCHMARK(BM_my3_header_parse); +//BENCHMARK(BM_my2_header_parse); //BENCHMARK(BM_my_header_parse); -BENCHMARK(BM_old_header_parse); -BENCHMARK(BM_avx2_header_parse); +//BENCHMARK(BM_old_header_parse); +//BENCHMARK(BM_avx2_header_parse); +//BENCHMARK(BM_adv_token); +//BENCHMARK(BM_adv_token_avx2); +//BENCHMARK(BM_get_session); +//BENCHMARK(BM_get_session_avx2); BENCHMARK_MAIN(); +/* + +int main() { + char buf[8096] = "Host: server\r\n" +"User-Agent: Mozilla/5.0 (X11; Linux x86_64) Gecko/20130501 Firefox/30.0 AppleWebKit/600.00 Chrome/30.0.0000.0 Trident/10.0 Safari/600.00\r\n" +"Cookie: uid=12345678901234567890; __utma=1.1234567890.1234567890.1234567890.1234567890.12; wd=2560x1600\r\n" +"Cookie: uid=12345678901234567890; __utma=1.1234567890.1234567890.1234567890.1234567890.12; wd=2560x1600\r\n" +"Cookie: uid=12345678901234567890; __utma=1.1234567890.1234567890.1234567890.1234567890.12; wd=2560x1600\r\n" +"Cookie: uid=12345678901234567890; __utma=1.1234567890.1234567890.1234567890.1234567890.12; wd=2560x1600\r\n" +"Cookie: uid=12345678901234567890; __utma=1.1234567890.1234567890.1234567890.1234567890.12; wd=2560x1600\r\n" +"Cookie: uid=12345678901234567890; __utma=1.1234567890.1234567890.1234567890.1234567890.12; wd=2560x1600\r\n" +"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,* /*;q=0.8\r\n" +"Accept-Language: en-US,en;q=0.5\r\n" +"Connection: keep-alive\r\n\r\n"; + //strcpy(buf,"Host: localhost:8080\r\nUser-Agent: curl/7.68.0\r\nAccept: * /*\r\n\r\n"); + //strcpy(buf,"Host: localhost:8080\r\nUser-Agent: python-requests/2.31.0\r\nAccept-Encoding: gzip, deflate\r\nAccept: * /*\r\nConnection: keep-alive\r\nCookie: foo=b=ar\r\nContent-Length: 0\r\n\r\n"); + + char cbuf[8096] = "uid=123456781234567890; mrsession=1234567890.1234567890.12; wd=2560x1600"; + int len = strlen("uid=123456781234567890; mrsession=1234567890.1234567890.12; wd=2560x1600"); + const char *cend = cbuf + len; + + getSession_avx2(cbuf,cend); + + int ret = 0; + //parse_headers_avx2(buf,buf+512,&ret); + //parse_headers(buf,buf+2048,&ret); + //parse_mine3(buf); + //printf(" ret=%d\n",ret); + + //unsigned long long l = 0x80008020ull; + //unsigned int s = 7; + //printf(" WTF %08x\n", l >> s ); + +} + +*/ diff --git a/gbench/query.cpp b/gbench/query.cpp new file mode 100644 index 0000000..8ece864 --- /dev/null +++ b/gbench/query.cpp @@ -0,0 +1,757 @@ + +#include +#include +#include +#include +#include +#ifdef __AVX2__ +#include +#endif + +#include + +#if __GNUC__ >= 3 +#define likely(x) __builtin_expect(!!(x), 1) +#define unlikely(x) __builtin_expect(!!(x), 0) +#else +#define likely(x) (x) +#define unlikely(x) (x) +#endif + +#ifdef _MSC_VER +#define ALIGNED(n) _declspec(align(n)) +#else +#define ALIGNED(n) __attribute__((aligned(n))) +#endif + + +#define IS_PRINTABLE_ASCII(c) ((unsigned char)(c)-040u < 0137u) + +#define hex_to_dec(x) \ + ((x <= '9' ? 0 : 9) + (x & 0x0f)) +#define is_hex(x) ((x >= '0' && x <= '9') || (x >= 'A' && x <= 'F')) + + +#define CHAR4_TO_INT(a, b, c, d) \ + (unsigned int)((d << 24) | (c << 16) | (b << 8) | a) + + +#define CHECK_END() \ + if (buf == buf_end) { \ + *ret = -2; \ + return NULL; \ + } + + +#define CHECK_EOF() \ + if (buf == buf_end) { \ + *ret = -2; \ + return NULL; \ + } + +#define EXPECT_CHAR_NO_CHECK(ch) \ + if (*buf++ != ch) { \ + *ret = -1; \ + return NULL; \ + } + +#define EXPECT_CHAR(ch) \ + CHECK_EOF(); \ + EXPECT_CHAR_NO_CHECK(ch); + + +// Table for converting to lower case +#define TOLC(c) __lct[(unsigned char)c] +static const unsigned char __lct[] __attribute__((aligned(64))) = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, + 0x40, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, + 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, + 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, + 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, + 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, + 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff +}; + + +static const char *token_char_map = "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0" + "\0\1\0\1\1\1\1\1\0\0\1\1\0\1\1\0\1\1\1\1\1\1\1\1\1\1\0\0\0\0\0\0" + "\0\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\0\0\0\1\1" + "\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\0\1\0\1\0" + "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0" + "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0" + "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0" + "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"; + +static unsigned long TZCNT(unsigned long long in) { + unsigned long res; + asm("tzcnt %1, %0\n\t" : "=r"(res) : "r"(in)); + return res; +} + + +// Search for a range of characters and return a pointer to the location or buf_end if none are found +char *findchar(char *buf, char *buf_end, char *ranges, size_t ranges_size, int *found) +{ + *found = 0; + __m128i ranges16 = _mm_loadu_si128((const __m128i *)ranges); + if (likely(buf_end - buf >= 16)) { + + size_t left = (buf_end - buf) & ~15; + do { + __m128i b16 = _mm_loadu_si128((const __m128i *)buf); + int r = _mm_cmpestri(ranges16, ranges_size, b16, 16, _SIDD_LEAST_SIGNIFICANT | _SIDD_CMP_RANGES | _SIDD_UBYTE_OPS); + if (unlikely(r != 16)) { + buf += r; + *found = 1; + return buf; + } + buf += 16; + left -= 16; + } while (likely(left != 0)); + + } + + size_t left = buf_end - buf; + if ( left != 0 ) { + static char sbuf[16] = {0}; + memcpy( sbuf, buf, left ); + __m128i b16 = _mm_loadu_si128((const __m128i *)sbuf); + size_t r = _mm_cmpestri(ranges16, ranges_size, b16, 16, _SIDD_LEAST_SIGNIFICANT | _SIDD_CMP_RANGES | _SIDD_UBYTE_OPS); + if (unlikely(r != 16) && r < left) { + buf += r; + *found = 1; + return buf; + } else { + buf = buf_end; + } + } + + *found = 0; + return buf; +} +// 1 2 3 4 5 6 7 +// 1234567890123456789012345678901234567890123456789012345678901234567890 +// /spanish/objetos%20voladores%20no%20identificados?foo=bar +// /spanish/objetos%20voladores/ +static inline size_t sse_decode(char* path, ssize_t length, int *qs_len) { + //DBG printf("sse_decode >%.*s<\n", (int)length, path); + if (length == 0) return length; + char *pat = path; + static char ranges1[] = "%%" "??"; + char *end = path + length; + int found; + + // We only findchar once - benchmark one or more % encodings with continuing to use findchar ( Spanish / Chinese ) + do { + //printf("sse_decode >%.*s<\n", (int)length, path); + pat = findchar(pat, end, ranges1, sizeof(ranges1) - 1, &found); + if ( found ) { + if(*pat == '%' && is_hex(*(pat + 1)) && is_hex(*(pat + 2))) { + *pat = (hex_to_dec(*(pat + 1)) << 4) + hex_to_dec(*(pat + 2)); + pat+=3; + length -= 2; + } else { + *qs_len = end-pat; + length -= end-pat; + break; + } + } + } while (0); + + if( !found || *pat == '?') return length; + char *write = pat; + if ( found ) write -= 2; + char *read = pat; + for (;read < end;) { + if (read[0] == '?') { + length -= end-read; + *qs_len = end-pat; + break; + } + if ( read[0] == '%' ) { + if( is_hex(read[1]) && is_hex(read[2]) ) { + *write = (hex_to_dec(read[1]) << 4) + hex_to_dec(read[2]); + write+=1; + read += 3; + length-=2; + } else { + if (read > write) { + write[0] = read[0]; + write[1] = read[1]; + } + read += 2; + write += 2; + } + + } else { + if (read > write) { + write[0] = read[0]; + } + read++; + write++; + } + } + //printf("sse_decode len %d path >%.*s<\n", (int)length, (int)length, path); + //printf(" qs %d\n",*qs_len); + + return length; +} + +__m256i m37 = _mm256_set1_epi8(37); // % +__m256i m63 = _mm256_set1_epi8(63); // ? +// 1 2 3 4 5 6 7 +// 1234567890123456789012345678901234567890123456789012345678901234567890 +// /print/%E4%B8%8D%E5%8F%AF%E5%86%8D%E7%94%9F%E8%B5%84%E6%BA%90/?test"; +// /spanish/objetos%20voladores%20no%20identificados?foo=bar +// /spanish/objetos%20voladores/ +static inline int path_decode(char* buf, int len, int *qs_len) { + unsigned int msk; + int i=0,tz; // 32B index + unsigned int shifted; + char *sbuf = buf; + char *obuf = buf; + char *buf_end = buf+len; + char *wbuf; + int found = 0; + + do { + const char *block_start = obuf+32*i; + __m256i b0 = _mm256_loadu_si256((const __m256i *) block_start); + msk = _mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b0, m37), _mm256_cmpeq_epi8(b0, m63) ) ); + while (1) { + + //if ( buf >= buf_end ) { goto decdone; } + shifted = buf-block_start; + if ( shifted >= 32 ) break; + tz = TZCNT((msk >> shifted)); + if ( tz < 32 ) { + buf += tz; + //printf( " fnd >%.*s<\n", (int)(buf-sbuf), sbuf ); + if ( buf >= buf_end ) { goto decdone; } + if ( *buf == '?' ) { + len -= buf_end-buf; + *qs_len = buf_end-buf-1; + //printf("path_decode len %d path >%.*s<\n", (int)len, (int)len, obuf); + goto decdone; + } + if ( *buf == '%' ) { + if ( found ) { + //printf( " copy >%.*s<\n", (int)(buf-sbuf), sbuf ); + memcpy( wbuf, sbuf, buf-sbuf ); + //printf( " to >%.*s<\n", (int)(buf-sbuf), wbuf ); + wbuf += buf-sbuf; + *wbuf = (hex_to_dec(buf[1]) << 4) + hex_to_dec(buf[2]); + wbuf++; + } else { + found = 1; + *buf = (hex_to_dec(buf[1]) << 4) + hex_to_dec(buf[2]); + wbuf = buf+1; + } + len -= 2; + buf += 3; + } + sbuf = buf; + } else { + buf += 32 - shifted; + break; + } + + } + i+=1; + if ( buf >= buf_end ) { goto decdone; } + } while ( buf < buf_end ); // Why doesn't this work + +decdone: + if ( found ) { + //printf( " copy >%.*s<\n", (int)(buf-sbuf), sbuf ); + memcpy( wbuf, sbuf, buf-sbuf ); + //printf( " to >%.*s<\n", (int)(buf-sbuf), wbuf ); + } + //printf( " fnd >%.*s<\n", (int)(buf-sbuf), sbuf ); + //printf("path_decode len %d path >%.*s<\n", (int)len, (int)len, obuf); + //printf(" qs %d\n",*qs_len); + return len; +} +static inline int path_decode2(char* buf, int len, int *qs_len) { + unsigned long long msk; + int i=0,tz; // 32B index + unsigned int shifted; + char *sbuf = buf; + char *obuf = buf; + char *buf_end = buf+len; + char *wbuf; + int found = 0; + + do { + const char *block_start = obuf+64*i; + __m256i b0 = _mm256_loadu_si256((const __m256i *) block_start); + __m256i b1 = _mm256_loadu_si256((const __m256i *) (block_start+32)); + msk = (unsigned int) _mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b0, m37), _mm256_cmpeq_epi8(b0, m63) ) ) | + ((unsigned long) _mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b1, m37), _mm256_cmpeq_epi8(b1, m63) ) ) << 32); + while (1) { + + //if ( buf >= buf_end ) { goto decdone; } + shifted = buf-block_start; + if ( shifted >= 64 ) break; + tz = TZCNT((msk >> shifted)); + if ( tz < 64 ) { + buf += tz; + //printf( " fnd >%.*s<\n", (int)(buf-sbuf), sbuf ); + if ( buf >= buf_end ) { goto decdone; } + if ( *buf == '?' ) { + len -= buf_end-buf; + *qs_len = buf_end-buf-1; + //printf("path_decode len %d path >%.*s<\n", (int)len, (int)len, obuf); + goto decdone; + } + if ( *buf == '%' ) { + if ( found ) { + //printf( " copy >%.*s<\n", (int)(buf-sbuf), sbuf ); + memcpy( wbuf, sbuf, buf-sbuf ); + //printf( " to >%.*s<\n", (int)(buf-sbuf), wbuf ); + wbuf += buf-sbuf; + *wbuf = (hex_to_dec(buf[1]) << 4) + hex_to_dec(buf[2]); + wbuf++; + } else { + found = 1; + *buf = (hex_to_dec(buf[1]) << 4) + hex_to_dec(buf[2]); + wbuf = buf+1; + } + len -= 2; + buf += 3; + } + sbuf = buf; + } else { + buf += 64 - shifted; + break; + } + + } + i+=1; + if ( buf >= buf_end ) { goto decdone; } + } while ( buf < buf_end ); // Why doesn't this work + +decdone: + if ( found ) { + //printf( " copy >%.*s<\n", (int)(buf-sbuf), sbuf ); + memcpy( wbuf, sbuf, buf-sbuf ); + //printf( " to >%.*s<\n", (int)(buf-sbuf), wbuf ); + } + //printf( " fnd >%.*s<\n", (int)(buf-sbuf), sbuf ); + //printf("path_decode len %d path >%.*s<\n", (int)len, (int)len, obuf); + //printf(" qs %d\n",*qs_len); + return len; +} +static inline int path_decode3(char* buf, int len, int *qs_len) { + if ( len > 32 ) return path_decode2(buf,len,qs_len); + else return path_decode(buf,len,qs_len); +} + +static inline void parse_query_args_old( char *buf, size_t buflen ) { + char *end = buf + buflen; + char *last = buf; + //PyObject* args = PyDict_New(); + + if ( buflen == 0 ) return; + + + //PyObject* key = NULL; PyObject* value = NULL; + + static char ALIGNED(16) ranges1[] = "==" "&&"; + int found; + int state = 0; + int grab_session = 0; + int ignore_me = 0; + size_t len; + // foo=bar&key=23%28 + do { + buf = findchar(buf, end, ranges1, sizeof(ranges1) - 1, &found); + if ( found ) { + if ( *buf == '=' ) { + len = sse_decode( last, buf-last, &ignore_me ); + //key = PyUnicode_FromStringAndSize(last, len); //TODO error + //printf( " key >%.*s<\n", (int)(buf-last), last); + state = 1; + buf+=1; + last = buf; + } + else if ( *buf == '&' ) { + //if ( state == 0 ) key = PyUnicode_FromString(""); + + len = sse_decode( last, buf-last, &ignore_me ); + //value = PyUnicode_FromStringAndSize(last, len); + //printf( " val >%.*s<\n", (int)(buf-last), last); + state = 0; + //PyDict_SetItem(args, key, value); // == -1) goto loop_error; + //Py_XDECREF(key); + //Py_XDECREF(value); + buf+=1; + while ( *buf == 32 ) buf++; + last = buf; + } + else { + printf(" ERR found not = or ; %.*s\n", 5, buf ); + } + } + } while( found ); + + if ( buf == end ) { + //if ( state == 0 ) key = PyUnicode_FromString(""); + if ( buf == end && *(buf-1) == ' ' ) { + len = path_decode( last, buf-last-1, &ignore_me ); + //value = PyUnicode_FromStringAndSize(last, len); //TODO error + //printf( " val >%.*s<\n", (int)(buf-last-1), last); + } else { + len = path_decode( last, buf-last, &ignore_me ); + //value = PyUnicode_FromStringAndSize(last, len); //TODO error + //printf( " val >%.*s<\n", (int)(buf-last), last); + } + state = 0; + //PyDict_SetItem(args, key, value); // == -1) goto loop_error; + //Py_XDECREF(key); + //Py_XDECREF(value); + } + + return; +} +static inline void parse_query_args( char *buf, size_t buflen ) { + char *end; + char *last = buf; + size_t len; + int ignore_me = 0; + //PyObject* args = PyDict_New(); + + if ( buflen == 0 ) return; + + len = path_decode2( buf, buflen, &ignore_me ); + //printf( " decoded >%.*s<\n", (int)(len), buf); + end = buf + len; + + //PyObject* key = NULL; PyObject* value = NULL; + + static char ALIGNED(16) ranges1[] = "==" "&&"; + int found; + int state = 0; + int grab_session = 0; + // foo=bar&key=23%28 + do { + buf = findchar(buf, end, ranges1, sizeof(ranges1) - 1, &found); + if ( found ) { + if ( *buf == '=' ) { + //len = path_decode( last, buf-last, &ignore_me ); + //key = PyUnicode_FromStringAndSize(last, len); //TODO error + //printf( " key >%.*s<\n", (int)(buf-last), last); + state = 1; + buf+=1; + last = buf; + } + else if ( *buf == '&' ) { + //if ( state == 0 ) key = PyUnicode_FromString(""); + + //len = path_decode( last, buf-last, &ignore_me ); + //value = PyUnicode_FromStringAndSize(last, len); + //printf( " val >%.*s<\n", (int)(buf-last), last); + state = 0; + //PyDict_SetItem(args, key, value); // == -1) goto loop_error; + //Py_XDECREF(key); + //Py_XDECREF(value); + buf+=1; + while ( *buf == 32 ) buf++; + last = buf; + } + else { + printf(" ERR found not = or ; %.*s\n", 5, buf ); + } + } + } while( found ); + + if ( buf == end ) { + //if ( state == 0 ) key = PyUnicode_FromString(""); + if ( buf == end && *(buf-1) == ' ' ) { + //len = path_decode( last, buf-last-1, &ignore_me ); + //value = PyUnicode_FromStringAndSize(last, len); //TODO error + //printf( " val >%.*s<\n", (int)(buf-last-1), last); + } else { + //len = path_decode( last, buf-last, &ignore_me ); + //value = PyUnicode_FromStringAndSize(last, len); //TODO error + //printf( " val >%.*s<\n", (int)(buf-last), last); + } + state = 0; + //PyDict_SetItem(args, key, value); // == -1) goto loop_error; + //Py_XDECREF(key); + //Py_XDECREF(value); + } + + return; +} +__m256i m38 = _mm256_set1_epi8(38); // & +__m256i m61 = _mm256_set1_epi8(61); // = +static inline void parse_query_args2( char *buf, size_t len ) { + unsigned long long msk; + int i=0,tz; // 32B index + unsigned int shifted; + char *sbuf = buf; + char *obuf = buf; + int state = 0; + int ignore_me = 0; + + if ( len == 0 ) return; + char *buf_end = buf+len; + + do { + const char *block_start = obuf+64*i; + __m256i b0 = _mm256_loadu_si256((const __m256i *) block_start); + __m256i b1 = _mm256_loadu_si256((const __m256i *) (block_start+32)); + msk = (unsigned int) _mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b0, m38), _mm256_cmpeq_epi8(b0, m61) ) ) | + ((unsigned long) _mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b1, m38), _mm256_cmpeq_epi8(b1, m61) ) ) << 32); + while (1) { + + //if ( buf >= buf_end ) { goto decdone; } + shifted = buf-block_start; + if ( shifted >= 64 ) break; + tz = TZCNT((msk >> shifted)); + if ( tz < 64 ) { + buf += tz; + if ( buf >= buf_end ) { goto pdone; } + if ( *buf == '=' ) { + //printf( " key >%.*s<\n", (int)(buf-sbuf), sbuf ); + len = path_decode2( sbuf, buf-sbuf, &ignore_me ); + state = 1; + buf += 1; + } + if ( *buf == '&' ) { + //printf( " val >%.*s<\n", (int)(buf-sbuf), sbuf); + len = path_decode2( sbuf, buf-sbuf, &ignore_me ); + state = 0; + buf+=1; + } + sbuf = buf; + } else { + buf += 64 - shifted; + break; + } + + } + i+=1; + if ( buf >= buf_end ) { goto pdone; } + } while ( buf < buf_end ); // Why doesn't this work + +pdone: + len = path_decode2( sbuf, buf-sbuf, &ignore_me ); + //printf( " done >%.*s<\n", (int)(buf-sbuf), sbuf ); + return; +} + + + + +static char buf[8096] = "/foo/bar/bazfdas"; +static int blen = strlen("/foo/bar/bazfdas"); +static char buf2[8096] = "/foo/bar/bazfdasfffffffffffffffffffffffffffffffffffffffdffffffffffffffffffffffffffffffffffffffffffffffffff1?foo=bar"; +static int blen2 = strlen("/foo/bar/bazfdasfffffffffffffffffffffffffffffffffffffffdffffffffffffffffffffffffffffffffffffffffffffffffff1?foo=bar"); +//static char buf[8096] = "/spanish/objetos%20voladoresentificados"; +//static int blen = strlen("/spanish/objetos%20voladoresentificados"); +static char buf3[8096] = "/spanish/objetos%20voladores%20no%20identificados?foo=bar"; +static int blen3 = strlen("/spanish/objetos%20voladores%20no%20identificados?foo=bar"); + +static char buf4[8096] = "/print/%E4%B8%8D%E5%8F%AF%E5%86%8D%E7%94%9F%E8%B5%84%E6%BA%90/?test"; +static int blen4 = strlen("/print/%E4%B8%8D%E5%8F%AF%E5%86%8D%E7%94%9F%E8%B5%84%E6%BA%90/?test"); + +static char qbuf[8096] = "p1=v1¶m2=value2"; +static int qlen = strlen("p1=v1¶m2=value2"); +static char qbuf2[8096] = "key%201=%C2%BFPeroc%C3%B3mopuedesdecir%20esto%3F¶m2=val2"; +static int qlen2 = strlen("key%201=%C2%BFPeroc%C3%B3mopuedesdecir%20esto%3F¶m2=val2"); +static char qbuf3[8096] = "key%201=%C2%BFPeroc%C3%B3mopuedesdecir%20esto%3F¶m2222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222220=val2&ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1"; +static int qlen3 = strlen("key%201=%C2%BFPeroc%C3%B3mopuedesdecir%20esto%3F¶m2222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222220=val2&ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1"); + + +static void BM_sse_decode(benchmark::State& state) { + int qslen = 0; + for (auto _ : state) { + sse_decode(buf, blen, &qslen); + } +} +static void BM_path_decode(benchmark::State& state) { + int qslen = 0; + for (auto _ : state) { + path_decode(buf, blen, &qslen); + } +} +static void BM_path_decode2(benchmark::State& state) { + int qslen = 0; + for (auto _ : state) { + path_decode2(buf, blen, &qslen); + } +} +static void BM_path_decode3(benchmark::State& state) { + int qslen = 0; + for (auto _ : state) { + path_decode3(buf, blen, &qslen); + } +} +static void BM_sse_decode_long(benchmark::State& state) { + int qslen = 0; + for (auto _ : state) { + sse_decode(buf2, blen2, &qslen); + } +} +static void BM_path_decode_long(benchmark::State& state) { + int qslen = 0; + for (auto _ : state) { + path_decode(buf2, blen2, &qslen); + } +} +static void BM_path_decode2_long(benchmark::State& state) { + int qslen = 0; + for (auto _ : state) { + path_decode2(buf2, blen2, &qslen); + } +} +static void BM_path_decode3_long(benchmark::State& state) { + int qslen = 0; + for (auto _ : state) { + path_decode3(buf2, blen2, &qslen); + } +} +static void BM_sse_decode_complex(benchmark::State& state) { + int qslen = 0; + for (auto _ : state) { + sse_decode(buf3, blen3, &qslen); + } +} +static void BM_path_decode_complex(benchmark::State& state) { + int qslen = 0; + for (auto _ : state) { + path_decode(buf3, blen3, &qslen); + } +} +static void BM_path_decode2_complex(benchmark::State& state) { + int qslen = 0; + for (auto _ : state) { + path_decode2(buf3, blen3, &qslen); + } +} +static void BM_path_decode3_complex(benchmark::State& state) { + int qslen = 0; + for (auto _ : state) { + path_decode3(buf3, blen3, &qslen); + } +} +static void BM_sse_decode_chinese(benchmark::State& state) { + int qslen = 0; + for (auto _ : state) { + sse_decode(buf4, blen4, &qslen); + } +} +static void BM_path_decode_chinese(benchmark::State& state) { + int qslen = 0; + for (auto _ : state) { + path_decode(buf4, blen4, &qslen); + } +} +static void BM_path_decode2_chinese(benchmark::State& state) { + int qslen = 0; + for (auto _ : state) { + path_decode2(buf4, blen4, &qslen); + } +} +static void BM_query_args_old(benchmark::State& state) { + int qslen = 0; + for (auto _ : state) { + parse_query_args_old(qbuf, qlen); + } +} +static void BM_query_args(benchmark::State& state) { + int qslen = 0; + for (auto _ : state) { + parse_query_args(qbuf, qlen); + } +} +static void BM_query_args2(benchmark::State& state) { + int qslen = 0; + for (auto _ : state) { + parse_query_args2(qbuf, qlen); + } +} +static void BM_query_args_decodes_old(benchmark::State& state) { + int qslen = 0; + for (auto _ : state) { + parse_query_args_old(qbuf3, qlen3); + } +} +static void BM_query_args_decodes(benchmark::State& state) { + int qslen = 0; + for (auto _ : state) { + parse_query_args(qbuf3, qlen3); + } +} +static void BM_query_args_decodes2(benchmark::State& state) { + int qslen = 0; + for (auto _ : state) { + parse_query_args2(qbuf3, qlen3); + } +} + +BENCHMARK(BM_query_args_old); +BENCHMARK(BM_query_args); +BENCHMARK(BM_query_args2); +BENCHMARK(BM_query_args_decodes_old); +BENCHMARK(BM_query_args_decodes); +BENCHMARK(BM_query_args_decodes2); +/* +BENCHMARK(BM_sse_decode); +BENCHMARK(BM_path_decode); +BENCHMARK(BM_path_decode2); +BENCHMARK(BM_path_decode3); +BENCHMARK(BM_sse_decode_long); +BENCHMARK(BM_path_decode_long); +BENCHMARK(BM_path_decode2_long); +BENCHMARK(BM_path_decode3_long); +BENCHMARK(BM_sse_decode_complex); +BENCHMARK(BM_path_decode_complex); +BENCHMARK(BM_path_decode2_complex); +BENCHMARK(BM_path_decode3_complex); +BENCHMARK(BM_sse_decode_chinese); +BENCHMARK(BM_path_decode_chinese); +BENCHMARK(BM_path_decode2_chinese); +*/ +BENCHMARK_MAIN(); +/* +int main() { + + int qslen = 0; + //printf("%.*s\n",blen4,buf4); + //path_decode2(buf4, blen4, &qslen); + //sse_decode(buf, blen, &qslen); + + parse_query_args2(qbuf3, qlen3); + +} + + +*/ diff --git a/gbench/readme b/gbench/readme index ccffad2..03d21d5 100644 --- a/gbench/readme +++ b/gbench/readme @@ -1,7 +1,7 @@ Benches g++ tst.cpp -std=c++11 -lbenchmark -lpthread -o tst - g++ parse.cpp -std=c++11 -lbenchmark -lpthread -o parse + g++ parse.cpp -msse4.2 -mavx2 -std=c++11 -lbenchmark -lpthread -o parse Test code first: g++ t.cpp -msse4.2 -mavx2 -std=c++11 -lpthread -o t @@ -16,3 +16,8 @@ Install gbench cmake -E chdir "build" cmake -DBENCHMARK_DOWNLOAD_DEPENDENCIES=on -DCMAKE_BUILD_TYPE=Release ../ cmake --build "build" --config Release sudo cmake --build "build" --config Release --target install + + +sudo cpupower frequency-set --governor performance +./mybench +sudo cpupower frequency-set --governor powersave diff --git a/gbench/run b/gbench/run new file mode 100755 index 0000000..1c16508 --- /dev/null +++ b/gbench/run @@ -0,0 +1,4 @@ + +sudo sh -c "echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor" +./$1 +sudo sh -c "echo powersave | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor" diff --git a/gbench/string.cpp b/gbench/string.cpp index 8e7818e..cd65715 100644 --- a/gbench/string.cpp +++ b/gbench/string.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #ifdef __AVX2__ #include @@ -19,28 +20,94 @@ #endif static inline bool _isdigit(char c) { return c >= '0' && c <= '9'; } +static inline bool _isdigit2(unsigned char c) { return (c & 0xF0) == 0x30; } +#define IS_DIGIT(c) ((c&0xF0) == 0x30) +#define IS_DIGIT2(c) (c >= '0' && c <= '9') -static void _strtol( char* buf ) { +static long _strtol( char* buf ) { char * endptr = buf+4; - long n = strtol(buf, &endptr, 10); + return strtol(buf, &endptr, 10); } -static void my_strtol( char* s ) { +static long my_strtol( char* s, int maxlen ) { long l; + int n = 0; + benchmark::DoNotOptimize(n); while (_isdigit(*s)) { l = (l * 10) + (*s++ - '0'); + n+=1; + if ( n > maxlen ) return l; } + return l; } +static long my_strtol2( char* s ) { + long l; + while (_isdigit2(*s)) { + l = (l * 10) + (*s++ - 0x30); + } + return l; +} +static long my_strtol3( char* s ) { + long l; + while (IS_DIGIT(*s)) { + l = (l * 10) + (*s++ - '0'); + } + return l; +} +static long my_strcmp( char* s ) { + //if ( s[0] == 'C' && s[11] == 'e' ) return 0; + if ( s[0] == 'C' ) return 0; + return 1; +} + + static void BM_strtol(benchmark::State& state) { - char buf[8096] = "1234 "; - for (auto _ : state) { _strtol(buf); } + char buf[8096] = "123z4 "; + for (auto _ : state) { long x = _strtol(buf); } } static void BM_my_strtol(benchmark::State& state) { - char buf[8096] = "1234 "; - for (auto _ : state) { my_strtol(buf); } + char buf[8096] = "123z4 "; + for (auto _ : state) { long x = my_strtol(buf,4); } +} +static void BM_my_strtol2(benchmark::State& state) { + char buf[8096] = "123z4 "; + for (auto _ : state) { long x = my_strtol2(buf); } +} +static void BM_my_strtol3(benchmark::State& state) { + char buf[8096] = "123z4 "; + for (auto _ : state) { long x = my_strtol3(buf); } +} +static void BM_strcmp(benchmark::State& state) { + char buf[8096] = "Content-Type"; + for (auto _ : state) { + long x; + benchmark::DoNotOptimize(x); + x = strcmp(buf, "Cntent-Type"); + } +} +static void BM_my_strcmp(benchmark::State& state) { + char buf[8096] = "Content-Type"; + for (auto _ : state) { + long x; + benchmark::DoNotOptimize(x); + x = my_strcmp(buf); + } } BENCHMARK(BM_strtol); BENCHMARK(BM_my_strtol); +BENCHMARK(BM_my_strtol2); +BENCHMARK(BM_my_strtol3); +BENCHMARK(BM_strcmp); +BENCHMARK(BM_my_strcmp); BENCHMARK_MAIN(); +/* +int main() { + char buf[8096] = "123z4 "; + //strcpy(buf,"942312"); + + printf(" my strtol %d\n", my_strtol(buf, 4)); + +} +*/ diff --git a/gbench/t.cpp b/gbench/t.cpp index 479459e..c335b1d 100644 --- a/gbench/t.cpp +++ b/gbench/t.cpp @@ -1,13 +1,14 @@ - #include #include #include +#include #include #ifdef __AVX2__ #include #endif +#include #if __GNUC__ >= 3 #define likely(x) __builtin_expect(!!(x), 1) @@ -17,8 +18,26 @@ #define unlikely(x) (x) #endif +#ifdef _MSC_VER +#define ALIGNED(n) _declspec(align(n)) +#else +#define ALIGNED(n) __attribute__((aligned(n))) +#endif + + #define IS_PRINTABLE_ASCII(c) ((unsigned char)(c)-040u < 0137u) +#define CHAR4_TO_INT(a, b, c, d) \ + (unsigned int)((d << 24) | (c << 16) | (b << 8) | a) + + +#define CHECK_END() \ + if (buf == buf_end) { \ + *ret = -2; \ + return NULL; \ + } + + #define CHECK_EOF() \ if (buf == buf_end) { \ *ret = -2; \ @@ -36,6 +55,59 @@ EXPECT_CHAR_NO_CHECK(ch); +// Table for converting to lower case +#define TOLC(c) __lct[(unsigned char)c] +static const unsigned char __lct[] __attribute__((aligned(64))) = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, + 0x40, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, + 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, + 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, + 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, + 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, + 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff +}; + + +static const char *token_char_map = "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0" + "\0\1\0\1\1\1\1\1\0\0\1\1\0\1\1\0\1\1\1\1\1\1\1\1\1\1\0\0\0\0\0\0" + "\0\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\0\0\0\1\1" + "\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\0\1\0\1\0" + "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0" + "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0" + "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0" + "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"; + +static unsigned long TZCNT(unsigned long long in) { + unsigned long res; + asm("tzcnt %1, %0\n\t" : "=r"(res) : "r"(in)); + return res; +} + std::size_t slow_hparse(std::string &text) noexcept { std::size_t spaces = 0; @@ -80,6 +152,33 @@ static const char *findchar(const char *buf, const char *buf_end, const char *ra } return buf; } +static const char *adv_token(const char *buf, int *ret) { + const char *tok_start = buf; + const char *buf_end = buf+512; + static const char ranges2[] = "\000\042\177\177"; + int found2; + buf = findchar(buf, buf+512, ranges2, sizeof(ranges2) - 1, &found2); + if (!found2) { + CHECK_END(); + } else if ( unlikely(*buf != ' ' )) { + *ret = -1; + return NULL; + } + while (1) { + if (*buf == ' ') { + return buf; + } else if (unlikely(!IS_PRINTABLE_ASCII(*buf))) { + if ((unsigned char)*buf < '\040' || *buf == '\177') { + *ret = -1; + return NULL; + } + } + ++buf; + CHECK_END(); + } + *ret = buf - tok_start; + return tok_start; +} static const char *get_token_to_eol(const char *buf, const char *buf_end, int *ret) { @@ -116,92 +215,669 @@ static const char *get_token_to_eol(const char *buf, const char *buf_end, int *r return buf; } +static inline int getSession( const char *buf, size_t buflen ) { + const char *end = buf + buflen; + const char *last = buf; + const char *ses; + int len; -static unsigned long TZCNT(unsigned long long in) { - unsigned long res; - asm("tzcnt %1, %0\n\t" : "=r"(res) : "r"(in)); - return res; + static char ALIGNED(16) ranges1[] = "==" ";;"; + int found; + int state = 0; + do { + last = buf; + buf = findchar(buf, end, ranges1, sizeof(ranges1) - 1, &found); + if ( found ) { + if ( *buf == '=' ) { + //printf( " fnd >%.*s<\n", buf-last, last ); + if ( state == 0 ) { + // Save out the mrsession id + if ( buf-last == 9 && ( *((unsigned int *)(last)) == CHAR4_TO_INT('m', 'r', 's','e') ) ) { + state = 1; + } + buf+=1; + } + } + else if ( *buf == ';' ) { + //printf( " fnd >%.*s<\n", buf-last, last ); + if (state == 1 ) { + ses = last; + len = buf-last; + return len; + } + state = 0; + buf+=1; + while ( *buf == 32 ) buf++; + } + } + } while( found ); + if (state) { + ses = last; + len = buf-last; + return len; + } + return -1; } - -__m256i m13 = _mm256_set1_epi8(13); -static const char *my_get_eol(const char *buf) { +static const char *my_get_eol128(const char *buf) { + //__m128i* pSrc1 = (__m128i *)string; // init pointer to start of string + __m128i m0 = _mm_set1_epi8(13); // vector of 16 `\0` characters while (1) { - __m256i v0 = _mm256_loadu_si256((const __m256i *)buf); - __m256i v1 = _mm256_cmpeq_epi8(v0, m13); - unsigned long vmask = _mm256_movemask_epi8(v1); + __m128i v0 = _mm_loadu_si128((const __m128i *)buf); + __m128i v1 = _mm_cmpeq_epi8(v0, m0); // compare all 16 chars + unsigned int vmask = _mm_movemask_epi8(v1); // get 16 comparison result bits if (vmask != 0) { buf += TZCNT(vmask) + 2; - break; + break; // we found a `\0`, break out of loop } - buf += 32; //pSrc1++; + buf += 16; //pSrc1++; // next 16 characters... } return buf; } -static void parse_sse4( const char* buf ) { - int ret = 0; - while ( ret == 0 && buf != NULL && buf[0] != '\r' ) { - buf = get_token_to_eol( buf, buf+512, &ret); - printf("%d - %.16s\n",(int)buf[0],buf); + //64bits 256bits bytes 8 * 32 +static const char *my_get_eol(const char *buf) { + + __m256i m13 = _mm256_set1_epi8(13); + while (1) + { + __m256i v0 = _mm256_loadu_si256((const __m256i *)buf); + __m256i v1 = _mm256_cmpeq_epi8(v0, m13); + unsigned long vmask = _mm256_movemask_epi8(v1); + if (vmask != 0) { + buf += TZCNT(vmask) + 2; + break; + } + buf += 32; //pSrc1++; } + return buf; } -static void parse_mysse4( const char* buf ) { - int ret = 0; - while ( ret == 0 && buf != NULL && buf[0] != '\r' ) { - buf = my_get_eol( buf ); - printf("%d - %.16s\n",(int)buf[0],buf); + __m256i m32 = _mm256_set1_epi8(32); +static const char *get_to_space(const char *buf, int *len) { + const char *orig = buf; + while (1) + { + __m256i v0 = _mm256_loadu_si256((const __m256i *)buf); + __m256i v1 = _mm256_cmpeq_epi8(v0, m32); + unsigned long vmask = _mm256_movemask_epi8(v1); + if (vmask != 0) { + buf += TZCNT(vmask) + 1; + break; + } + buf += 32; } + *len = buf-orig-1; + return buf; } -//__m256i m13 = _mm256_set1_epi8(13); -__m256i m58 = _mm256_set1_epi8(58); // 0x1313131313131313... - // 0x32333435363713 // abcdef\r - // 32 bit number 0x40 - -static void parse_mine( const char* buf ) { - unsigned long msk; - int i=0,t; // 32B index +__m256i m59 = _mm256_set1_epi8(59); +__m256i m61 = _mm256_set1_epi8(61); +static int getSession_avx2( const char* buf, const char* buf_end ) { + unsigned int msk; + int i=0,tz; // 32B index int cnt = 0; + unsigned int shifted; const char *sbuf = buf; const char *obuf = buf; + int name_or_value = 0; + int found = 0; + do { - int shifted = 0; const char *block_start = obuf+32*i; __m256i b0 = _mm256_loadu_si256((const __m256i *) block_start); - msk = _mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b0, m13), _mm256_cmpeq_epi8(b0, m58) ) ); - printf(" m%d : 0x%08x\n", i, msk); - + msk = _mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b0, m59), _mm256_cmpeq_epi8(b0, m61) ) ); while (1) { + //if ( buf >= buf_end ) { goto sesdone; } shifted = buf-block_start; - t = TZCNT((msk >> shifted)); - printf("DELME shifted %d tzcnt %d\n", shifted, t); - if ( t < 32 ) { - if ( t == 0 ) break; - buf += t+2; - printf("L=%d str=%.*s\n", buf-sbuf-2, buf-sbuf-2, sbuf); + if ( shifted >= 32 ) break; + tz = TZCNT((msk >> shifted)); + if ( tz < 32 ) { + buf += tz; + //printf( " fnd >%.*s<\n", buf-sbuf, sbuf ); + if ( buf >= buf_end ) { goto sesdone; } + if ( name_or_value == 1 ) { + if ( *buf == '=' ) { buf += 1; continue; } // = in value field + if ( found ) { + //printf( " done >%.*s<\n", buf-sbuf, sbuf ); + return buf-sbuf; + } + buf+=1; + name_or_value = 0; + } else { + if ( buf-sbuf == 9 && ( *((unsigned int *)(sbuf)) == CHAR4_TO_INT('m', 'r', 's','e') ) ) { + found = 1; + } + name_or_value = 1; + } + buf += 1; sbuf = buf; } else { - buf = block_start + 32; + buf += 32 - shifted; break; } } i+=1; - cnt += 1; - } while ( cnt < 20 && buf[0] != '\r' ); + if ( buf >= buf_end ) { goto sesdone; } + } while ( buf-obuf < buf_end-obuf ); + +sesdone: + if ( found ) { + //printf( " sesdone >%.*s<\n", buf-sbuf, sbuf ); + return buf-sbuf; + } + return 0; +} + + +static const char *parse_headers(const char *buf, const char *buf_end, int *ret) +{ + int num_headers = 0; + int max_headers = 20; + if ( buf_end <= buf ) { + *ret = -2; + return NULL; + } + for (;; ++num_headers) { + CHECK_EOF(); + if (*buf == '\015') { + ++buf; + EXPECT_CHAR('\012'); + break; + } else if (*buf == '\012') { + ++buf; + break; + } + if (num_headers == max_headers) { + *ret = -1; + return NULL; + } + //printf(">%.*s<", 10, buf); + // Listed small to larger - probably best as most used TODO check bounds + switch ( TOLC(*buf) ) { + case 'h': // Host + ////headers[*num_headers].name = buf; + //headers[*num_headers].name_len = 4; + buf += 6; + goto hvalue; + case 'c': + if ( buf[6] == ':' ) { // Cookie: + //headers[*num_headers].name = buf; + //headers[*num_headers].name_len = 6; + buf += 8; + goto hvalue; + } + if ( buf[10] == ':' ) { // Connection: + //headers[*num_headers].name = buf; + //headers[*num_headers].name_len = 10; + buf += 12; + goto hvalue; + } + if ( buf[11] == ':' ) { // Content-MD5: + //headers[*num_headers].name = buf; + //headers[*num_headers].name_len = 11; + buf += 13; + goto hvalue; + } + if ( buf[12] == ':' ) { // Content-Type: + //headers[*num_headers].name = buf; + //headers[*num_headers].name_len = 12; + buf += 14; + //goto hvalue; + //if ( buf[0] == 'a' && buf[13] == 'r' ) { //"application/mrpacker" + //mrr->flags = 2; + //} + //buf = get_token_to_eol(buf, buf_end, ret); + buf = my_get_eol(buf); + goto skipvalue; + } + if ( buf[13] == ':' ) { // Cache-Control: + //headers[*num_headers].name = buf; + //headers[*num_headers].name_len = 13; + buf += 15; + goto hvalue; + } + if ( buf[14] == ':' ) { // Content-Length: + //headers[*num_headers].name = buf; + //headers[*num_headers].name_len = 14; + buf += 16; + goto hvalue; + } + if ( buf[16] == ':' ) { // CF-Connecting-IP + //headers[*num_headers].name = buf; + //headers[*num_headers].name_len = 16; + buf += 18; + //mrr->ip = buf; + //buf = get_token_to_eol(buf, buf_end, ret); + buf = my_get_eol(buf); + //mrr->ip_len = headers[*num_headers].value_len; + goto skipvalue; + } + break; + //printf( "%.*s\n" , 10, buf); + //printf( "Host: %08x == %08x\n" , MR_CHAR4_INT('o', 's', 't',':'), *((unsigned int *)(buf+1))); + case 'd': + if ( buf[4] == ':' ) { // Date: + //headers[*num_headers].name = buf; + //headers[*num_headers].name_len = 4; + buf += 6; + goto hvalue; + } + if ( buf[3] == ':' ) { // DNT: + //headers[*num_headers].name = buf; + //headers[*num_headers].name_len = 3; + buf += 5; + goto hvalue; + } + break; + case 'x': + if ( buf[9] == ':' ) { // X-Real-IP + //headers[*num_headers].name = buf; + //headers[*num_headers].name_len = 9; + buf += 11; + //mrr->ip = buf; + //buf = get_token_to_eol(buf, buf_end, ret); + buf = my_get_eol(buf); + //mrr->ip_len = headers[*num_headers].value_len; + goto skipvalue; + } + if ( buf[15] == ':' ) { // X-Forwarded-For: + //headers[*num_headers].name = buf; + //headers[*num_headers].name_len = 15; + buf += 17; + //mrr->ip = buf; + buf = get_token_to_eol(buf, buf_end, ret); + //mrr->ip_len = headers[*num_headers].value_len; + goto skipvalue; + //goto hvalue; + } + if ( buf[16] == ':' ) { // X-Forwarded-Host: + //headers[*num_headers].name = buf; + //headers[*num_headers].name_len = 16; + buf += 18; + goto hvalue; + } + break; + case 'f': + if ( buf[5] == ':' ) { // From: + //headers[*num_headers].name = buf; + //headers[*num_headers].name_len = 5; + buf += 7; + goto hvalue; + } + if ( buf[9] == ':' ) { // Forwarded: + //headers[*num_headers].name = buf; + //headers[*num_headers].name_len = 9; + buf += 11; + goto hvalue; + } + break; + case 'i': + if ( buf[13] == ':' ) { // If-None-Match: + //headers[*num_headers].name = buf; + //headers[*num_headers].name_len = 13; + buf += 15; + goto hvalue; + } + if ( buf[17] == ':' ) { // If-Modified-Since: + //headers[*num_headers].name = buf; + //headers[*num_headers].name_len = 17; + buf += 19; + goto hvalue; + } + break; + case 'o': + //headers[*num_headers].name = buf; + //headers[*num_headers].name_len = 6; + buf += 8; + goto hvalue; + case 'r': + //headers[*num_headers].name = buf; + //headers[*num_headers].name_len = 7; + buf += 9; + goto hvalue; + case 't': // Transfer-Encoding: + //headers[*num_headers].name = buf; + //headers[*num_headers].name_len = 17; + buf += 19; + goto hvalue; + case 'u': + if ( buf[10] == ':' ) { // User-Agent: + //headers[*num_headers].name = buf; + //headers[*num_headers].name_len = 10; + buf += 12; + goto hvalue; + } + if ( buf[25] == ':' ) { // Upgrade-Insecure-Requests: + //headers[*num_headers].name = buf; + //headers[*num_headers].name_len = 25; + buf += 27; + goto hvalue; + } + break; + case 'a': + if ( buf[6] == ':' ) { // Accept: + //headers[*num_headers].name = buf; + //headers[*num_headers].name_len = 6; + buf += 8; + goto hvalue; + } + if ( buf[13] == ':' ) { // Authorization: + //headers[*num_headers].name = buf; + //headers[*num_headers].name_len = 13; + buf += 15; + goto hvalue; + } + if ( buf[14] == ':' ) { // Accept-Charset: + //headers[*num_headers].name = buf; + //headers[*num_headers].name_len = 14; + buf += 16; + goto hvalue; + } + if ( buf[15] == ':' ) { // Accept-Encoding: -Datetime + //headers[*num_headers].name = buf; + //headers[*num_headers].name_len = 15; + buf += 17; + goto hvalue; + } + if ( buf[16] == ':' ) { // Accept-Language: + //headers[*num_headers].name = buf; + //headers[*num_headers].name_len = 15; + buf += 17; + goto hvalue; + } + if ( buf[29] == ':' ) { // Access-Control-Request-Method: + //headers[*num_headers].name = buf; + //headers[*num_headers].name_len = 29; + buf += 31; + goto hvalue; + } + if ( buf[30] == ':' ) { // Access-Control-Request-Headers: + //headers[*num_headers].name = buf; + //headers[*num_headers].name_len = 30; + buf += 32; + goto hvalue; + } + break; + + } + if (!(num_headers != 0 && (*buf == ' ' || *buf == '\t'))) { + /* parsing name, but do not discard SP before colon, see + * http://www.mozilla.org/security/announce/2006/mfsa2006-33.html */ + //headers[*num_headers].name = buf; + static const char ranges1[] = "\x00 " /* control chars and up to SP */ + "\"\"" /* 0x22 */ + "()" /* 0x28,0x29 */ + ",," /* 0x2c */ + "//" /* 0x2f */ + ":@" /* 0x3a-0x40 */ + "[]" /* 0x5b-0x5d */ + "{\377"; /* 0x7b-0xff */ + int found; + buf = findchar(buf, buf_end, ranges1, sizeof(ranges1) - 1, &found); + if (!found) { + CHECK_EOF(); + } + while (1) { + if (*buf == ':') { + break; + } else if (!token_char_map[(unsigned char)*buf]) { + *ret = -1; + return NULL; + } + ++buf; + CHECK_EOF(); + } + //if ((headers[*num_headers].name_len = buf - headers[*num_headers].name) == 0) { + //*ret = -1; + //return NULL; + //} + ++buf; + for (;; ++buf) { + CHECK_EOF(); + if (!(*buf == ' ' || *buf == '\t')) { + break; + } + } + } else { + //headers[*num_headers].name = NULL; + //headers[*num_headers].name_len = 0; + } +hvalue: + //if ((buf = get_token_to_eol(buf, buf_end, ret)) == NULL) { + if ((buf = my_get_eol(buf)) == NULL) { + return NULL; + } +skipvalue: + ; + } + return buf; +} +static void find_ranges32(__m256i b0, unsigned long *range0, unsigned long *range1) { + const __m256i rr0 = _mm256_set1_epi8(0x00 - 1); + const __m256i rr1 = _mm256_set1_epi8(0x1f + 1); + const __m256i rr2 = _mm256_set1_epi8(0x3a); + const __m256i rr4 = _mm256_set1_epi8(0x7f); + const __m256i rr7 = _mm256_set1_epi8(0x09); + /* 0<=x */ + __m256i gz0 = _mm256_cmpgt_epi8(b0, rr0); + /* 0== 96) { + b0 = _mm256_loadu_si256((const __m256i_u*) buf + 32*0); + b1 = _mm256_loadu_si256((const __m256i_u*) buf + 32*1); + b2 = _mm256_loadu_si256((const __m256i_u*) buf + 32*2); + b3 = _mm256_loadu_si256((const __m256i_u*) tmpbuf); + } else if (dist >= 64) { + b0 = _mm256_loadu_si256((const __m256i_u*) buf + 32*0); + b1 = _mm256_loadu_si256((const __m256i_u*) buf + 32*1); + b2 = _mm256_loadu_si256((const __m256i_u*) tmpbuf); + b3 = _mm256_setzero_si256(); + } else { + if(dist < 32) { + b0 = _mm256_loadu_si256((const __m256i_u*)tmpbuf); + return find_ranges32(b0, range0, range1); + } else { + b0 = _mm256_loadu_si256((const __m256i_u*) buf + 32*0); + b1 = _mm256_loadu_si256((const __m256i_u*)tmpbuf); + return find_ranges64(b0, b1, range0, range1); + } + } + } else { + /* Load 128 bytes */ + b0 = _mm256_loadu_si256((const __m256i_u*) buf + 32*0); + b1 = _mm256_loadu_si256((const __m256i_u*) buf + 32*1); + b2 = _mm256_loadu_si256((const __m256i_u*) buf + 32*2); + b3 = _mm256_loadu_si256((const __m256i_u*) buf + 32*3); + } + + /* 0<=x */ + __m256i gz0 = _mm256_cmpgt_epi8(b0, rr0); + __m256i gz1 = _mm256_cmpgt_epi8(b1, rr0); + __m256i gz2 = _mm256_cmpgt_epi8(b2, rr0); + __m256i gz3 = _mm256_cmpgt_epi8(b3, rr0); + /* 0=> shft; + tz = TZCNT(bitmap); + if ( tz < 64 ) { // tz is 64 if not found + p += tz; + //printf( " fnd >%.*s<\n", p-buf, buf ); + if ( state == 0 ) { // : + state = 1; + p += 2; buf = p; + } else { // \r + state = 0; + p += 2; buf = p; + if ( *p == '\r' ) goto wedone; + } + } else { + p += 64 - shft; + //printf("DELMEZ %.*s\n", 3, p) ; + } + off = p-prep_start; + //printf("DELME off=%d\n",off); + shft = off&0x3F; + bmOff = off/64; + } while ( bmOff < 8 ) ; + prep_start += 512; + buf = prep_start; + } + +wedone: +// Host: server\r\n +// User-Agent: Mozilla/5.0 (X11; Linux x86_64) Gecko/20130501 Firefox/30.0 AppleWebKit/600.00 Chrome/30.0.0000.0 Trident/10.0 Safari/600.00\r\n + //printf("%.*s\n", p - prep_start, prep_start); + return buf; +} + + + +__m256i m58 = _mm256_set1_epi8(58); // 0x1313131313131313... + // 0x32333435363713 // abcdef\r + // 32 bit number 0x40 +static void parse_mine( const char* buf ) { unsigned long long msk[8]; // 1 bit for each of 512 bytes matching : or \r //__m256i b0,b1,b2,b3,b4,b5,b6,b7,b8,b9,b10,b11,b12,b13,b14,b15; __m256i b0,b1,b2,b3,b4,b5,b6,b7; + __m256i m13 = _mm256_set1_epi8(13); b0 = _mm256_loadu_si256((const __m256i *) (buf + 32*0)); // buf[0] b1 = _mm256_loadu_si256((const __m256i *) (buf + 32*1)); // buf[32] @@ -240,49 +916,358 @@ static void parse_mine3( const char* buf ) { msk[7] = _mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b6, m13), _mm256_cmpeq_epi8(b6, m58) ) ) ^ ((unsigned long) _mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b7, m13), _mm256_cmpeq_epi8(b7, m58) ) ) << 32); - const char *obuf = buf; - const char *sbuf = buf; + //for ( int i = 0; i < 8; i++ ) { + //printf(" m%d : 0x%016llx\n", i, msk[i]); + //} + + // uint64 msk[8] -- 512 bits + // Loop until crlfcrlf or 0xA + // Name = string(buf, tzcnt(msk[i]) ) + // msk >>= len, buf += len // TODO increment i for each 64bits + // Value = string(buf, tzcnt(msk[i])) + // "Host: server\r\n" int i = 0; // msk[i] - int cnt = 0, t; + int l, dist, t; //int cnt = 0; + while (1) { + + // msk[0] is only 64 bits + l = 0; + while(1) { + t = TZCNT(msk[i]); // tz is 6, 'server\r\n' + // msk[0] is all 0s and I get 64+2 + if ( t == 64 ) { + l += t-dist; + dist = 0; + i += 1; + if ( i > 7 ) break; + } else { + l += t; + dist += t+2; + buf += l+2; + if ( t+2 > 64 ) { + msk[i] = 0; + i += 1; + if ( i > 7 ) break; + dist = t+2-64; + msk[i] >>= (t+2-64); + } else { + msk[i] >>= t+2; + } + + break; + } + + } + + if ( i > 7 ) break; + if ( buf[0] == '\r' ) break; + } + + +} +static void parse_mine3( const char* buf ) { + + unsigned long long msk[8]; // 1 bit for each of 512 bytes matching : or \r + + //__m256i b0,b1,b2,b3,b4,b5,b6,b7,b8,b9,b10,b11,b12,b13,b14,b15; + __m256i b0,b1,b2,b3,b4,b5,b6,b7; + __m256i m13 = _mm256_set1_epi8(13); + + const char *obuf = buf; + const char *sbuf = buf; + + int i; // msk[i] + int t; + unsigned int s = 0; + int name_or_value = 0; + + const char *block_start = obuf; + +new512: + i = 0; + buf = obuf; + + b0 = _mm256_loadu_si256((const __m256i *) (buf + 32*0)); // buf[0] + b1 = _mm256_loadu_si256((const __m256i *) (buf + 32*1)); // buf[32] + b2 = _mm256_loadu_si256((const __m256i *) (buf + 32*2)); // buf[64] + b3 = _mm256_loadu_si256((const __m256i *) (buf + 32*3)); // buf[96] + b4 = _mm256_loadu_si256((const __m256i *) (buf + 32*4)); // buf[128] + b5 = _mm256_loadu_si256((const __m256i *) (buf + 32*5)); + b6 = _mm256_loadu_si256((const __m256i *) (buf + 32*6)); + b7 = _mm256_loadu_si256((const __m256i *) (buf + 32*7)); // 256 bytes + + msk[0] = (unsigned int)_mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b0, m13), _mm256_cmpeq_epi8(b0, m58) ) ) | + ((unsigned long) _mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b1, m13), _mm256_cmpeq_epi8(b1, m58) ) ) << 32); + msk[1] = (unsigned int)_mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b2, m13), _mm256_cmpeq_epi8(b2, m58) ) ) | + ((unsigned long) _mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b3, m13), _mm256_cmpeq_epi8(b3, m58) ) ) << 32); + msk[2] = (unsigned int)_mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b4, m13), _mm256_cmpeq_epi8(b4, m58) ) ) | + ((unsigned long) _mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b5, m13), _mm256_cmpeq_epi8(b5, m58) ) ) << 32); + msk[3] = (unsigned int)_mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b6, m13), _mm256_cmpeq_epi8(b6, m58) ) ) | + ((unsigned long) _mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b7, m13), _mm256_cmpeq_epi8(b7, m58) ) ) << 32); + + b0 = _mm256_loadu_si256((const __m256i *) (buf + 32*8)); + b1 = _mm256_loadu_si256((const __m256i *) (buf + 32*9)); + b2 = _mm256_loadu_si256((const __m256i *) (buf + 32*10)); + b3 = _mm256_loadu_si256((const __m256i *) (buf + 32*11)); + b4 = _mm256_loadu_si256((const __m256i *) (buf + 32*12)); + b5 = _mm256_loadu_si256((const __m256i *) (buf + 32*13)); + b6 = _mm256_loadu_si256((const __m256i *) (buf + 32*14)); + b7 = _mm256_loadu_si256((const __m256i *) (buf + 32*15)); + + msk[4] = (unsigned int)_mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b0, m13), _mm256_cmpeq_epi8(b0, m58) ) ) ^ + ((unsigned long) _mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b1, m13), _mm256_cmpeq_epi8(b1, m58) ) ) << 32); + msk[5] = (unsigned int)_mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b2, m13), _mm256_cmpeq_epi8(b2, m58) ) ) ^ + ((unsigned long) _mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b3, m13), _mm256_cmpeq_epi8(b3, m58) ) ) << 32); + msk[6] = (unsigned int)_mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b4, m13), _mm256_cmpeq_epi8(b4, m58) ) ) ^ + ((unsigned long) _mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b5, m13), _mm256_cmpeq_epi8(b5, m58) ) ) << 32); + msk[7] = (unsigned int)_mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b6, m13), _mm256_cmpeq_epi8(b6, m58) ) ) ^ + ((unsigned long) _mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b7, m13), _mm256_cmpeq_epi8(b7, m58) ) ) << 32); + + + // "Host: server\r\n" do { - const char *block_start = obuf+64*i; + block_start = obuf+64*i; while(1) { - t = TZCNT((msk[i]>>(buf-block_start))); - printf("DELME shifted %d tzcnt %d msk %016llx\n", buf-block_start, t, (msk[i]>>(buf-block_start))); + s = buf-block_start; + t = TZCNT((msk[i]>>s)); + //printf("DELME mski %016llx shift %d\n", msk[i], s ); + //printf("DELME shft %016llx\n", msk[i]>>s ); if ( t < 64 ) { - buf += t+2; - printf("L=%d str=%.*s\n", buf-sbuf-2, buf-sbuf-2, sbuf); + buf += t; + if ( name_or_value == 1 ) { + if ( *buf == ':' ) { buf += 1; continue; } // : in value field + name_or_value = 0; + } else { + name_or_value = 1; + } + //printf( " fnd >%.*s<\n", buf-sbuf, sbuf ); + buf += 2; if ( *buf == '\r' ) break; // \r\n\r\n marks the end sbuf = buf; if ( (buf-block_start)> 64 ) break; // TODO? } else { buf = block_start + 64; break; } - + } - - cnt += 1; - i+=1; if ( i > 7 ) break; // TODO - } while ( buf[0] != '\r' ); - - + + i+=1; + if ( buf[0] == '\r' ) goto done; + } while ( i < 8 && buf[0] != '\r' ); + + obuf += 512; + goto new512; +done: + i += 1; } +static void parse_mine2( const char* buf, const char *buf_end ) { + unsigned long msk; + int i=0,tz; // 32B index + int cnt = 0; + unsigned int shifted; + const char *sbuf = buf; + const char *obuf = buf; + int name_or_value = 0; + __m256i m13 = _mm256_set1_epi8(13); + + do { + const char *block_start = obuf+64*i; + __m256i b0 = _mm256_loadu_si256((const __m256i *) block_start); + __m256i b1 = _mm256_loadu_si256((const __m256i *) (block_start+32)); + msk = (unsigned int) _mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b0, m13), _mm256_cmpeq_epi8(b0, m58) ) ) | + ((unsigned long) _mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b1, m13), _mm256_cmpeq_epi8(b1, m58) ) ) << 32); + + //const char *block_start = obuf+32*i; + //__m256i b0 = _mm256_loadu_si256((const __m256i *) block_start); + //msk = _mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b0, m13), _mm256_cmpeq_epi8(b0, m58) ) ); + while (1) { + + shifted = buf-block_start; + if ( shifted >= 64 ) break; + tz = TZCNT((msk >> shifted)); + //printf("DELME mski %016llx shift %d\n", msk, shifted ); + //printf("DELME shft %016llx\n", msk>>shifted ); + if ( tz < 64 ) { + buf += tz; + if ( name_or_value == 1 ) { + if ( *buf == ':' ) { buf += 1; continue; } // : in value field + name_or_value = 0; + } else { + name_or_value = 1; + } + printf( " fnd >%.*s<\n", buf-sbuf, sbuf ); + buf += 2; if ( *buf == '\r' ) return; // \r\n\r\n marks the end + sbuf = buf; + } else { + buf += 64 - shifted; + break; + } -int main() { - char buf[8096] = "Host: server\r\n" + } + i+=1; + } while ( buf < buf_end ); + //} while ( *buf != '\r' ); +} + +static void parse_sse4( const char* buf ) { + int ret = 0; + while ( ret == 0 && buf != NULL && buf[0] != '\r' ) { + buf = get_token_to_eol( buf, buf+512, &ret); + } +} +static void parse_mysse4( const char* buf ) { + int ret = 0; + while ( ret == 0 && buf != NULL && buf[0] != '\r' ) { + buf = my_get_eol( buf ); + } +} +static char buf[8096] = "Host: server\r\n" "User-Agent: Mozilla/5.0 (X11; Linux x86_64) Gecko/20130501 Firefox/30.0 AppleWebKit/600.00 Chrome/30.0.0000.0 Trident/10.0 Safari/600.00\r\n" "Cookie: uid=12345678901234567890; __utma=1.1234567890.1234567890.1234567890.1234567890.12; wd=2560x1600\r\n" -"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\r\n" +"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,* /*;q=0.8\r\n" +"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,* /*;q=0.8\r\n" +"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,* /*;q=0.8\r\n" +"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,* /*;q=0.8\r\n" +"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,* /*;q=0.8\r\n" "Accept-Language: en-US,en;q=0.5\r\n" -"Connection: keep-alive\r\n\r\nzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz " -" " -" "; +"Connection: keep-alive\r\n\r\n"; +static char buf2[8096] = "Host: localhost:8080\r\nUser-Agent: python-requests/2.31.0\r\nAccept-Encoding: gzip, deflate\r\nAccept: * /*\r\nConnection: keep-alive\r\nCookie: foo=b=ar\r\nContent-Length: 0\r\n\r\n"; +static char path[8096] = "/foo/bar/bazfdasfffffffffffffffffffffffffffffffffffffffdfffffffffffffffffffffffffffffffffffffffffffffffffff "; + +//static char cbuf[8096] = "uid=123456781234567890; mrsession=1234567890.1234567890.12; wd=2560x1600"; +//static int clen= strlen("uid=123456781234567890; mrsession=1234567890.1234567890.12; wd=2560x1600"); +static char cbuf[8096] = "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa; mrsession=1234567890.1234567890.12; xxxxxxxxxxxxxxxxxxxxxxxxx=zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz"; +static int clen= strlen("zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa; mrsession=1234567890.1234567890.12; xxxxxxxxxxxxxxxxxxxxxxxxx=zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz"); +static const char *cend = cbuf + clen; + +/* +static void BM_SlowParse(benchmark::State& state) { + // Perform setup here + std::string text = "Host: server\n" +"User-Agent: Mozilla/5.0 (X11; Linux x86_64) Gecko/20130501 Firefox/30.0 AppleWebKit/600.00 Chrome/30.0.0000.0 Trident/10.0 Safari/600.00\n" +"Cookie: uid=12345678901234567890; __utma=1.1234567890.1234567890.1234567890.1234567890.12; wd=2560x1600\n" +"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,* /*;q=0.8\n" +"Accept-Language: en-US,en;q=0.5\n" +"Connection: keep-alive\n"; + + for (auto _ : state) { + // This code gets timed + slow_hparse(text); + } +} +static void BM_sse4_get_eol(benchmark::State& state) { + // Perform setup here + for (auto _ : state) { + // This code gets timed + parse_sse4(buf); + } +} +static void BM_my_get_eol(benchmark::State& state) { + // Perform setup here + for (auto _ : state) { + // This code gets timed + parse_mysse4(buf); + } +} + +static void BM_my_header_parse(benchmark::State& state) { + for (auto _ : state) { + parse_mine(buf); + } +} +static void BM_my2_header_parse(benchmark::State& state) { + for (auto _ : state) { + parse_mine2(buf); + } +} +static void BM_my3_header_parse(benchmark::State& state) { + for (auto _ : state) { + parse_mine3(buf); + } +} + - parse_mine3(buf); +static void BM_old_header_parse(benchmark::State& state) { + int ret = 0; + for (auto _ : state) { + parse_headers(buf,buf+2048,&ret); + } +} +static void BM_avx2_header_parse(benchmark::State& state) { + int ret = 0; + for (auto _ : state) { + parse_headers_avx2(buf,buf+2048,&ret); + } } + +static void BM_adv_token(benchmark::State& state) { + int ret = 0; + int path_len = 0; + for (auto _ : state) { + adv_token(path, &path_len); + } +} +static void BM_adv_token_avx2(benchmark::State& state) { + int ret = 0; + int path_len = 0; + for (auto _ : state) { + get_to_space(path, &path_len); + } +} +static void BM_get_session(benchmark::State& state) { + int ret = 0; + for (auto _ : state) { + getSession(cbuf, clen); + } +} +static void BM_get_session_avx2(benchmark::State& state) { + int ret = 0; + for (auto _ : state) { + getSession_avx2(cbuf, cend); + } +} + + + + + + + +//BENCHMARK(BM_SlowParse); +//BENCHMARK(BM_sse4_get_eol); +//BENCHMARK(BM_my_get_eol); +//BENCHMARK(BM_my3_header_parse); +//BENCHMARK(BM_my2_header_parse); +//BENCHMARK(BM_my_header_parse); +//BENCHMARK(BM_old_header_parse); +//BENCHMARK(BM_avx2_header_parse); +//BENCHMARK(BM_adv_token); +//BENCHMARK(BM_adv_token_avx2); +//BENCHMARK(BM_get_session); +//BENCHMARK(BM_get_session_avx2); +BENCHMARK_MAIN(); + + +*/ +int main() { + + //getSession_avx2(cbuf,cend); + + int ret = 0; + //parse_headers_avx2(buf,buf+512,&ret); + //parse_headers(buf,buf+2048,&ret); + parse_mine2(buf, buf+2048); + //printf(" ret=%d\n",ret); + + //unsigned long long l = 0x80008020ull; + //unsigned int s = 7; + //printf(" WTF %08x\n", l >> s ); + +} + diff --git a/gbench/tst b/gbench/tst index 332d9ac..df15d88 100755 Binary files a/gbench/tst and b/gbench/tst differ diff --git a/readme b/readme index 4c41a02..085e726 100644 --- a/readme +++ b/readme @@ -5,6 +5,8 @@ Updating version: internals app.c and response.c +sudo sh -c "echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor" +sudo sh -c "echo powersave | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor" wrk -t4 -c32 -d2s http://localhost:8080/ @@ -50,15 +52,38 @@ User-Agent: curl/7.51.0 Accept: */* <<< +POST /form HTTP/1.1 +Host: localhost:8080 +User-Agent: python-requests/2.31.0 +Accept-Encoding: gzip, deflate +Accept: */* +Connection: keep-alive +Content-Length: 19 +Content-Type: application/x-www-form-urlencoded + +p1=v1¶m2=value2POST /form HTTP/1.1 +Host: localhost:8080 +User-Agent: python-requests/2.31.0 +Accept-Encoding: gzip, deflate +Accept: */* +Connection: keep-alive +Content-Length: 19 +Content-Type: application/x-www-form-urlencoded + +p1=v1¶m2=value2 +<<< curl http://localhost:8080/ Valgrind doesn't work with AVX2 ... valgrind --tool=memcheck --suppressions=valgrind-python.supp python3 -E -tt ./tst.py +Big headers +curl http://localhost:8080/ -H 'User-Agent: Mozilla/5.0 (X11; Linux x86_64) Gecko/20130501 Firefox/30.0 AppleWebKit/600.00 Chrome/30.0.0000.0 Trident/10.0 Safari/600.00' -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8' -H 'Accept-Language: en-US,en;q=0.5' -H 'Cookie: mrsession=43709dd361cc443e976b05714581a7fb; foo=fdsfdasdfasdfdsfasdfsdfsdfasdfas; short=fazc;' -H 'Connection: keep-alive' + FORM urlencoded TODO more params. Can we fix the performance here? -curl -d "param1=value1¶m2=value2" -X POST http://localhost:8080/ -H "Content-Type: application/x-www-form-urlencoded" -wrk -t4 -c32 -d1s http://localhost:8080/form -s tests/lua/form.lua +curl -d "param1=value1¶m2=value2" -X POST http://localhost:8080/form -H "Content-Type: application/x-www-form-urlencoded" +wrk -t1 -c1 -d1s http://localhost:8080/form -s tests/lua/form.lua cookie test: curl --cookie "mrsession=bnwg23LQbOmdAtcBELdLwsFcyJkN8iGp" http://localhost:8080/ diff --git a/runtests b/runtests new file mode 100755 index 0000000..5e92ad4 --- /dev/null +++ b/runtests @@ -0,0 +1,5 @@ + +sudo sh -c "echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor" +python dotests.py +sudo sh -c "echo powersave | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor" + diff --git a/src/mrhttp/app.py b/src/mrhttp/app.py index a39cd4b..ea45879 100644 --- a/src/mrhttp/app.py +++ b/src/mrhttp/app.py @@ -8,7 +8,8 @@ import asyncio import traceback import socket -import os, sys, random, mrpacker +import os, sys, random, mrpacker, time +from glob import glob import multiprocessing import faulthandler import functools @@ -16,7 +17,7 @@ import inspect, copy #from inspect import signature #getmodulename, isawaitable, signature, stack #from prof import profiler_start,profiler_stop -import uuid, http.cookies +import http.cookies import mrhttp from mrhttp import Protocol @@ -68,10 +69,13 @@ def __init__(self): self.listeners = { "at_start":[], "at_end":[], "after_start":[]} self._mc = None self._mrq = None + self._mrq2 = None self._mrc = None + self.static_cached_files = [] self.session_backend = "memcached" self.uses_session = False self.uses_mrq = False + self.uses_mrq2 = False self.err404 = "404 Not Found

Not Found

The requested page was not found

" self.err400 = "400 Bad Request

Invalid Request

" @@ -115,40 +119,10 @@ def route(self, uri, methods=["GET"], options=[], _type="html"): self.uses_session = True if "mrq" in options: self.uses_mrq = True + if "mrq2" in options: + self.uses_mrq2 = True if not uri.startswith('/'): uri = '/' + uri - #params = {} - #params["methods"] = methods - #params["options"] = options - #params["type"] = _type - #params["mrq"] = None - #for o in options: - #if o.startswith("mrq"): - # - #self.uses_mrq = True - #if self._mrq == None: - #srvs = self.config.get("mrq", None) - #print(srvs) - #if type(srvs) != list or type(srvs[0]) != tuple or len(srvs) == 0: - #print("When using MrQ app.config['mrq'] must be set to a list of (host,port) tuple pairs. Exiting") - #exit(1) - #self._mrq = [] - #if type(srvs) == list and type(srvs[0]) == list: - #for s in srvs: - #self._mrq.append( MrqClient( s, self.loop) ) - #else: - #self._mrq.append( MrqClient( srvs, self.loop) ) - #if o == "mrq": - #o = "mrq0" - #l = [] - #try: - #for n in o[3:]: - #l.append( self._mrq[int(n)] ) - #params["mrq"] = l - #except: - #print("Error mrq route specifies a cluster that doesn't exist") - #print("uri:", uri, "mrq", o) - #exit(1) def response(func): self.router.add_route( func, uri, methods, options, _type ) @@ -164,6 +138,40 @@ def add_routes(self, rs): params["type"] = r[4] self.router.add_route( r[0], r[1], params ) + #TODO Size limit on files? + def static_cached_timer(self): + ts = time.time() # Avoid race + for item in self.static_cached_files: + fn = item[1] + if os.path.getmtime(fn) > self.static_cached_timestamp: + with open(fn, 'rb') as f: + b = f.read() + self.router.update_cached_route( [item[0], b] ) + self.static_cached_timestamp = ts + self.loop.call_later(10, self.static_cached_timer) + + + #TODO Use brotli'd files - if [path].br exists use that instead + def static_cached(self, root, directory): + def removeprefix( prefix, text ): + if text.startswith(prefix): + return text[len(prefix):] + + if not os.path.isdir(directory): + print("WARNING: app.static_cached root dir does not exist") + return + files = glob(os.path.join(directory, '**', '*'), recursive=True) + self.static_cached_timestamp = time.time() + for fn in files: + if os.path.isdir(fn): continue + with open(fn, 'rb') as f: + b = f.read() + + + if not root.startswith('/'): root = '/'+root + uri = root+removeprefix(directory, fn) + self.static_cached_files.append( [uri,fn] ) + self.router.add_cached_route( uri, b ) def _get_idle_and_busy_connections(self): return \ @@ -241,15 +249,23 @@ def serve(self, *, sock, host, port, loop, run_async=False): self._appStart() if self.uses_mrq: - mrqconf = self.config.get("mrq", None) - if not mrqconf: + srvs = self.config.get("mrq", None) + if not srvs: print("When using MrQ app.config['mrq'] must be set. Exiting") exit(1) - srvs = self.config.get("mrq", None) if type(srvs) != list or len(srvs) == 0 or type(srvs[0]) != tuple: print("When using MrQ app.config['mrq'] must be set to a list of (host,port) tuple pairs. Exiting") exit(1) self._mrq = MrqClient( srvs, self.loop) + if self.uses_mrq2: + srvs = self.config.get("mrq2", None) + if not srvs: + print("When using mrq2 app.config['mrq2'] must be set. Exiting") + exit(1) + if type(srvs) != list or len(srvs) == 0 or type(srvs[0]) != tuple: + print("When using MrQ app.config['mrq'] must be set to a list of (host,port) tuple pairs. Exiting") + exit(1) + self._mrq2 = MrqClient( srvs, self.loop) if self.uses_session: @@ -288,29 +304,7 @@ def serve(self, *, sock, host, port, loop, run_async=False): for r in self.requests: r.cleanup() self.requests = None - - #for ref in gc.get_referrers(self.requests[0]): - #if type(ref) == list: - #print("list") - #else: - #print(ref) - #print("DELME refcnt ", sys.getrefcount(self.requests[0])) - #r = self.requests[0] - #print("id requests ", id(self.requests)) - #rs = self.requests - #self.requests = None - #gc.collect() - #print (gc.get_referrers(rs)) - #print("DELME refcnt ", sys.getrefcount(r)) - #for ref in gc.get_referrers(r): - #if type(ref) == list: - #print("list") - #print("id ref ", id(ref)) - #else: - #print(ref) - - # Update the response date string every few seconds def updateDateString(self): self.updateDate( format_date_time(None) ) @@ -319,6 +313,7 @@ def updateDateString(self): def _appStart(self): self.loop.call_soon(self.updateDateString) + self.loop.call_soon(self.static_cached_timer) def _run(self, *, host, port, num_workers=None, debug=None): @@ -435,9 +430,6 @@ def setUserSessionAndCookies(self, request, user_id, user, cookies=http.cookies. skey = userk + k[len(userk):] - # TODO We could have user id be optional and do this if not given - #skey = uuid.uuid4().hex - # Send the session cookie back to the user c = cookies c['mrsession'] = skey diff --git a/src/mrhttp/internals/app.h b/src/mrhttp/internals/app.h index e19d456..016acc5 100644 --- a/src/mrhttp/internals/app.h +++ b/src/mrhttp/internals/app.h @@ -41,6 +41,7 @@ typedef struct { // Clients PyObject *py_mc; PyObject *py_mrq; + PyObject *py_mrq2; PyObject *py_mrc; PyObject *py_redis; PyObject *py_session_backend_type; // int 1,2,3 ( memcached, mrworkserver, mrcache ) diff --git a/src/mrhttp/internals/common.h b/src/mrhttp/internals/common.h index 6bab553..38cd661 100644 --- a/src/mrhttp/internals/common.h +++ b/src/mrhttp/internals/common.h @@ -11,7 +11,6 @@ #define likely(x) __builtin_expect(!!(x), 1) #define unlikely(x) __builtin_expect(!!(x), 0) - #if 0 //unsigned long long cycles = rdtsc(); //unsigned long long ecyc = rdtsc(); diff --git a/src/mrhttp/internals/faststrcmp.h b/src/mrhttp/internals/faststrcmp.h index dde16e2..079f55d 100644 --- a/src/mrhttp/internals/faststrcmp.h +++ b/src/mrhttp/internals/faststrcmp.h @@ -1,4 +1,6 @@ +// TODO Where did this come from? + static const unsigned char lct[] __attribute__((aligned(64))) = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, diff --git a/src/mrhttp/internals/module.h b/src/mrhttp/internals/module.h index 1fef96c..b4d87ed 100644 --- a/src/mrhttp/internals/module.h +++ b/src/mrhttp/internals/module.h @@ -74,6 +74,7 @@ static PyGetSetDef Protocol_getset[] = { static PyMethodDef Router_methods[] = { {"setupRoutes", (PyCFunction)Router_setupRoutes, METH_NOARGS, ""}, + {"update_cached_route", (PyCFunction)Router_update_cached_route, METH_O, ""}, {NULL} }; static PyMethodDef MrhttpApp_methods[] = { @@ -85,6 +86,7 @@ static PyMethodDef MrhttpApp_methods[] = { static PyMemberDef MrhttpApp_members[] = { {"_mc", T_OBJECT, offsetof(MrhttpApp, py_mc), 0, NULL}, {"_mrq", T_OBJECT, offsetof(MrhttpApp, py_mrq), 0, NULL}, + {"_mrq2", T_OBJECT, offsetof(MrhttpApp, py_mrq2), 0, NULL}, {"_redis", T_OBJECT, offsetof(MrhttpApp, py_redis), 0, NULL}, {"_session_client", T_OBJECT, offsetof(MrhttpApp, py_session), 0, NULL}, {"session_backend_type", T_OBJECT, offsetof(MrhttpApp, py_session_backend_type), 0, NULL}, @@ -125,7 +127,7 @@ static PyMemberDef Request_members[] = { {"_files", T_OBJECT, offsetof(Request, py_files), 0, NULL}, {"servers_down",T_OBJECT, offsetof(Request, py_mrq_servers_down),0, NULL}, {"user", T_OBJECT, offsetof(Request, py_user), 0, NULL}, - {"ip", T_OBJECT, offsetof(Request, py_ip), 0, NULL}, + //{"ip", T_OBJECT, offsetof(Request, py_ip), 0, NULL}, {NULL}, }; static PyGetSetDef Request_getset[] = { diff --git a/src/mrhttp/internals/mrhttpparser.c b/src/mrhttp/internals/mrhttpparser.c index 0b9a7a5..faf888d 100644 --- a/src/mrhttp/internals/mrhttpparser.c +++ b/src/mrhttp/internals/mrhttpparser.c @@ -1,3 +1,4 @@ + /* * Copyright (c) 2013-2018 Mark Reed * @@ -56,24 +57,7 @@ static void print_buffer( char* b, int len ) { printf("\n"); } - -#define MR_LC_INT 0x20202020 -#define MR_LC_LONG 0x2020202020202020UL -#define MR_CHAR4_INT(a, b, c, d) \ - (unsigned int)((d << 24) | (c << 16) | (b << 8) | a) -#define MR_CHAR8_INT(a, b, c, d, e, f, g, h) \ - (((long)h << 56) | ((long)g << 48) | ((long)f << 40) \ - | ((long)e << 32) | (d << 24) | (c << 16) | (b << 8) | a) -#define MR_P2LCINT(p) ((*(unsigned int *)(p)) | MR_LC_INT) - -// Match with case conversion -#define C4_INT_LCM(p, a, b, c, d) \ - !((*(unsigned int *)(p) | MR_LC_INT) ^ MR_CHAR4_INT(a, b, c, d)) -#define C8_INT_LCM(p, a, b, c, d, e, f, g, h) \ - !((*(unsigned long *)(p) | MR_LC_LONG) \ - ^ MR_CHAR8_INT(a, b, c, d, e, f, g, h)) - -#define CHECK_EOF() \ +#define CHECK_END() \ if (buf == buf_end) { \ *ret = -2; \ return NULL; \ @@ -86,581 +70,96 @@ static void print_buffer( char* b, int len ) { } #define EXPECT_CHAR(ch) \ - CHECK_EOF(); \ + CHECK_END(); \ EXPECT_CHAR_NO_CHECK(ch); -#define ADVANCE_TOKEN(tok, toklen) \ - do { \ - const char *tok_start = buf; \ - static const char ALIGNED(16) ranges2[] = "\000\042\177\177"; \ - int found2; \ - buf = findchar(buf, buf_end, ranges2, sizeof(ranges2) - 1, &found2); \ - if (!found2) { \ - CHECK_EOF(); \ - } else if ( unlikely(*buf != ' ' )) { \ - *ret = -1; \ - return NULL; \ - } \ - while (1) { \ - if (*buf == ' ') { \ - break; \ - } else if (unlikely(!IS_PRINTABLE_ASCII(*buf))) { \ - if ((unsigned char)*buf < '\040' || *buf == '\177') { \ - *ret = -1; \ - return NULL; \ - } \ - } \ - ++buf; \ - CHECK_EOF(); \ - } \ - tok = tok_start; \ - toklen = buf - tok_start; \ - } while (0) - - -static const char *token_char_map = "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0" - "\0\1\0\1\1\1\1\1\0\0\1\1\0\1\1\0\1\1\1\1\1\1\1\1\1\1\0\0\0\0\0\0" - "\0\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\0\0\0\1\1" - "\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\0\1\0\1\0" - "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0" - "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0" - "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0" - "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"; - -static const char *findchar(const char *buf, const char *buf_end, const char *ranges, size_t ranges_size, int *found) -{ - *found = 0; -#if __SSE4_2__ - if (likely(buf_end - buf >= 16)) { - __m128i ranges16 = _mm_loadu_si128((const __m128i *)ranges); - - size_t left = (buf_end - buf) & ~15; - do { - __m128i b16 = _mm_loadu_si128((const __m128i *)buf); - int r = _mm_cmpestri(ranges16, ranges_size, b16, 16, _SIDD_LEAST_SIGNIFICANT | _SIDD_CMP_RANGES | _SIDD_UBYTE_OPS); - if (unlikely(r != 16)) { - buf += r; - *found = 1; - break; - } - buf += 16; - left -= 16; - } while (likely(left != 0)); - } -#else - /* suppress unused parameter warning */ - (void)buf_end; - (void)ranges; - (void)ranges_size; -#endif - return buf; -} - -static const char *get_token_to_eol(const char *buf, const char *buf_end, const char **token, size_t *token_len, int *ret) -{ - const char *token_start = buf; - -#ifdef __SSE4_2__ - static const char ranges1[] = "\0\010" - /* allow HT */ - "\012\037" - /* allow SP and up to but not including DEL */ - "\177\177" - /* allow chars w. MSB set */ - ; - int found; - buf = findchar(buf, buf_end, ranges1, sizeof(ranges1) - 1, &found); - if (found) - goto FOUND_CTL; -#else - /* find non-printable char within the next 8 bytes, this is the hottest code; manually inlined */ - while (likely(buf_end - buf >= 8)) { -#define DOIT() \ - do { \ - if (unlikely(!IS_PRINTABLE_ASCII(*buf))) \ - goto NonPrintable; \ - ++buf; \ - } while (0) - DOIT(); - DOIT(); - DOIT(); - DOIT(); - DOIT(); - DOIT(); - DOIT(); - DOIT(); -#undef DOIT - continue; - NonPrintable: - if ((likely((unsigned char)*buf < '\040') && likely(*buf != '\011')) || unlikely(*buf == '\177')) { - goto FOUND_CTL; - } - ++buf; - } -#endif - for (;; ++buf) { - CHECK_EOF(); - if (unlikely(!IS_PRINTABLE_ASCII(*buf))) { - if ((likely((unsigned char)*buf < '\040') && likely(*buf != '\011')) || unlikely(*buf == '\177')) { - goto FOUND_CTL; - } - } - } -FOUND_CTL: - if (likely(*buf == '\015')) { - ++buf; - EXPECT_CHAR('\012'); - *token_len = buf - 2 - token_start; - } else if (*buf == '\012') { - *token_len = buf - token_start; - ++buf; - } else { - *ret = -1; - return NULL; - } - *token = token_start; - - return buf; -} - -static const char *is_complete(const char *buf, const char *buf_end, size_t last_len, int *ret) -{ - int ret_cnt = 0; - buf = last_len < 3 ? buf : buf + last_len - 3; - - while (1) { - CHECK_EOF(); - if (*buf == '\015') { - ++buf; - CHECK_EOF(); - EXPECT_CHAR('\012'); - ++ret_cnt; - } else if (*buf == '\012') { - ++buf; - ++ret_cnt; - } else { - ++buf; - ret_cnt = 0; - } - if (ret_cnt == 2) { - return buf; - } - } - - *ret = -2; - return NULL; -} - -#define PARSE_INT(valp_, mul_) \ - if (*buf < '0' || '9' < *buf) { \ - buf++; \ - *ret = -1; \ - return NULL; \ - } \ - *(valp_) = (mul_) * (*buf++ - '0'); - -#define PARSE_INT_3(valp_) \ - do { \ - int res_ = 0; \ - PARSE_INT(&res_, 100) \ - *valp_ = res_; \ - PARSE_INT(&res_, 10) \ - *valp_ += res_; \ - PARSE_INT(&res_, 1) \ - *valp_ += res_; \ - } while (0) - -#ifdef __DELMEAVX2__ static unsigned long TZCNT(unsigned long long in) { unsigned long res; asm("tzcnt %1, %0\n\t" : "=r"(res) : "r"(in)); return res; } +// TODO just len +static int get_len_to_space(const char *buf, const char *buf_end) { + const char *orig = buf; + __m256i m32 = _mm256_set1_epi8(32); + while (1) + { + __m256i v0 = _mm256_loadu_si256((const __m256i *)buf); + __m256i v1 = _mm256_cmpeq_epi8(v0, m32); + unsigned long vmask = _mm256_movemask_epi8(v1); + if (vmask != 0) { + buf += TZCNT(vmask); + return buf-orig; + } + buf += 32; + if ( buf >= buf_end ) return -1; + } +} + static const char *parse_headers_avx2(const char *buf, const char *buf_end, struct mr_header *headers, size_t *num_headers, size_t max_headers, int *ret, struct mr_request *mrr) { - unsigned long long msk[8]; // 1 bit for each of 512 bytes matching : or \r + unsigned long msk; + int i=0, tz; // 32B index + unsigned int shifted; const char *sbuf = buf; - int cnt = 0; + const char *obuf = buf; int name_or_value = 0; -__m256i m13 = _mm256_set1_epi8(13); -__m256i m58 = _mm256_set1_epi8(58); + __m256i m13 = _mm256_set1_epi8(13); // \r + __m256i m58 = _mm256_set1_epi8(58); // : - // Parse in 512B chunks with avx2 instructions do { + const char *block_start = obuf+64*i; + if ( block_start > buf_end ) { printf("DELME hdr too big\n"); *ret = -1; return NULL; } - const char *block_start = buf; // Start of each 64B block - __m256i b0,b1,b2,b3,b4,b5,b6,b7; - - b0 = _mm256_loadu_si256((const __m256i *) (buf + 32*0)); // buf[0] - b1 = _mm256_loadu_si256((const __m256i *) (buf + 32*1)); // buf[32] - b2 = _mm256_loadu_si256((const __m256i *) (buf + 32*2)); // buf[64] - b3 = _mm256_loadu_si256((const __m256i *) (buf + 32*3)); // buf[96] - b4 = _mm256_loadu_si256((const __m256i *) (buf + 32*4)); // buf[128] - b5 = _mm256_loadu_si256((const __m256i *) (buf + 32*5)); - b6 = _mm256_loadu_si256((const __m256i *) (buf + 32*6)); - b7 = _mm256_loadu_si256((const __m256i *) (buf + 32*7)); // 256 bytes - - msk[0] = _mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b0, m13), _mm256_cmpeq_epi8(b0, m58) ) ) | - ((unsigned long) _mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b1, m13), _mm256_cmpeq_epi8(b1, m58) ) ) << 32); - msk[1] = _mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b2, m13), _mm256_cmpeq_epi8(b2, m58) ) ) | - ((unsigned long) _mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b3, m13), _mm256_cmpeq_epi8(b3, m58) ) ) << 32); - msk[2] = _mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b4, m13), _mm256_cmpeq_epi8(b4, m58) ) ) | - ((unsigned long) _mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b5, m13), _mm256_cmpeq_epi8(b5, m58) ) ) << 32); - msk[3] = _mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b6, m13), _mm256_cmpeq_epi8(b6, m58) ) ) | - ((unsigned long) _mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b7, m13), _mm256_cmpeq_epi8(b7, m58) ) ) << 32); - - b0 = _mm256_loadu_si256((const __m256i *) (buf + 32*8)); - b1 = _mm256_loadu_si256((const __m256i *) (buf + 32*9)); - b2 = _mm256_loadu_si256((const __m256i *) (buf + 32*10)); - b3 = _mm256_loadu_si256((const __m256i *) (buf + 32*11)); - b4 = _mm256_loadu_si256((const __m256i *) (buf + 32*12)); - b5 = _mm256_loadu_si256((const __m256i *) (buf + 32*13)); - b6 = _mm256_loadu_si256((const __m256i *) (buf + 32*14)); - b7 = _mm256_loadu_si256((const __m256i *) (buf + 32*15)); - - msk[4] = _mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b0, m13), _mm256_cmpeq_epi8(b0, m58) ) ) | - ((unsigned long) _mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b1, m13), _mm256_cmpeq_epi8(b1, m58) ) ) << 32); - msk[5] = _mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b2, m13), _mm256_cmpeq_epi8(b2, m58) ) ) | - ((unsigned long) _mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b3, m13), _mm256_cmpeq_epi8(b3, m58) ) ) << 32); - msk[6] = _mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b4, m13), _mm256_cmpeq_epi8(b4, m58) ) ) | - ((unsigned long) _mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b5, m13), _mm256_cmpeq_epi8(b5, m58) ) ) << 32); - msk[7] = _mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b6, m13), _mm256_cmpeq_epi8(b6, m58) ) ) | - ((unsigned long) _mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b7, m13), _mm256_cmpeq_epi8(b7, m58) ) ) << 32); - - // "Host: server\r\n" - // Headers end on \r\n\r\n - int i = 0; // msk[i] - int t; - do { - - while(1) { - t = TZCNT((msk[i]>>(buf-block_start))); - if ( t < 64 ) { - buf += t; - //printf(">%.*s<\n", 16, sbuf); - if ( name_or_value == 1 ) { - if ( buf[0] != '\r' ) { buf++; continue; } // Handle : in the value - headers[*num_headers].value = sbuf; - headers[*num_headers].value_len = buf-sbuf; - ++*num_headers; - name_or_value = 0; - } else { - headers[*num_headers].name = sbuf; - headers[*num_headers].name_len = buf-sbuf; - name_or_value = 1; - } - buf += 2; if ( buf[0] == '\r' ) { return buf+2; } // End of headers - sbuf = buf; - if ( (buf-block_start)> 64 ) break; + __m256i b0 = _mm256_loadu_si256((const __m256i *) block_start); + __m256i b1 = _mm256_loadu_si256((const __m256i *) (block_start+32)); + msk = (unsigned int) _mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b0, m13), _mm256_cmpeq_epi8(b0, m58) ) ) | + ((unsigned long) _mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b1, m13), _mm256_cmpeq_epi8(b1, m58) ) ) << 32); + + while (1) { + + shifted = buf-block_start; + if ( shifted >= 64 ) break; + tz = TZCNT((msk >> shifted)); + if ( tz < 64 ) { + buf += tz; + + if ( name_or_value == 1 ) { + if ( *buf == ':' ) { buf += 1; continue; } // : in value field + headers[*num_headers].value = sbuf; + headers[*num_headers].value_len = buf-sbuf; + ++*num_headers; + if (*num_headers >= max_headers) { printf("DELME hdr too many\n"); *ret = -1; return NULL; } + name_or_value = 0; + buf += 2; if ( *buf == '\r' ) { buf+=2; *ret=0; return buf; } // \r\n\r\n marks the end } else { - buf = block_start + 64; - break; + headers[*num_headers].name = sbuf; + headers[*num_headers].name_len = buf-sbuf; + name_or_value = 1; + buf += 2; } - + sbuf = buf; + } else { + buf += 64 - shifted; + break; } - - block_start += 64; - //if ( buf[0] == '\r' ) { return buf+2; } // End of headers - } while ( ++i < 8 ); - - } while (++cnt < 32); - - // If we get here the header was too large so abort. TODO abort with appropriate error code - *ret = -1; - return NULL; -} -#endif -static const char *parse_headers(const char *buf, const char *buf_end, struct mr_header *headers, size_t *num_headers, - size_t max_headers, int *ret, struct mr_request *mrr) -{ - if ( buf_end <= buf ) { - *ret = -2; - return NULL; } - for (;; ++*num_headers) { - CHECK_EOF(); - if (*buf == '\015') { - ++buf; - EXPECT_CHAR('\012'); - break; - } else if (*buf == '\012') { - ++buf; - break; - } - if (*num_headers == max_headers) { - *ret = -1; - return NULL; - } - //printf(">%.*s<", 10, buf); - // Listed small to larger - probably best as most used TODO check bounds - switch ( TOLC(*buf) ) { - case 'h': // Host - headers[*num_headers].name = buf; - headers[*num_headers].name_len = 4; - buf += 6; - goto hvalue; - case 'c': - if ( buf[6] == ':' ) { // Cookie: - headers[*num_headers].name = buf; - headers[*num_headers].name_len = 6; - buf += 8; - goto hvalue; - } - if ( buf[10] == ':' ) { // Connection: - headers[*num_headers].name = buf; - headers[*num_headers].name_len = 10; - buf += 12; - goto hvalue; - } - if ( buf[11] == ':' ) { // Content-MD5: - headers[*num_headers].name = buf; - headers[*num_headers].name_len = 11; - buf += 13; - goto hvalue; - } - if ( buf[12] == ':' ) { // Content-Type: - headers[*num_headers].name = buf; - headers[*num_headers].name_len = 12; - buf += 14; - //goto hvalue; - if ( buf[0] == 'a' && buf[13] == 'r' ) { //"application/mrpacker" - mrr->flags = 2; - } - buf = get_token_to_eol(buf, buf_end, &headers[*num_headers].value, &headers[*num_headers].value_len, ret); - goto skipvalue; - } - if ( buf[13] == ':' ) { // Cache-Control: - headers[*num_headers].name = buf; - headers[*num_headers].name_len = 13; - buf += 15; - goto hvalue; - } - if ( buf[14] == ':' ) { // Content-Length: - headers[*num_headers].name = buf; - headers[*num_headers].name_len = 14; - buf += 16; - goto hvalue; - } - if ( buf[16] == ':' ) { // CF-Connecting-IP - headers[*num_headers].name = buf; - headers[*num_headers].name_len = 16; - buf += 18; - mrr->ip = buf; - buf = get_token_to_eol(buf, buf_end, &headers[*num_headers].value, &headers[*num_headers].value_len, ret); - mrr->ip_len = headers[*num_headers].value_len; - goto skipvalue; - } - break; - //printf( "%.*s\n" , 10, buf); - //printf( "Host: %08x == %08x\n" , MR_CHAR4_INT('o', 's', 't',':'), *((unsigned int *)(buf+1))); - case 'd': - if ( buf[4] == ':' ) { // Date: - headers[*num_headers].name = buf; - headers[*num_headers].name_len = 4; - buf += 6; - goto hvalue; - } - if ( buf[3] == ':' ) { // DNT: - headers[*num_headers].name = buf; - headers[*num_headers].name_len = 3; - buf += 5; - goto hvalue; - } - break; - case 'x': - if ( buf[9] == ':' ) { // X-Real-IP - headers[*num_headers].name = buf; - headers[*num_headers].name_len = 9; - buf += 11; - mrr->ip = buf; - buf = get_token_to_eol(buf, buf_end, &headers[*num_headers].value, &headers[*num_headers].value_len, ret); - mrr->ip_len = headers[*num_headers].value_len; - goto skipvalue; - } - if ( buf[15] == ':' ) { // X-Forwarded-For: - headers[*num_headers].name = buf; - headers[*num_headers].name_len = 15; - buf += 17; - mrr->ip = buf; - buf = get_token_to_eol(buf, buf_end, &headers[*num_headers].value, &headers[*num_headers].value_len, ret); - mrr->ip_len = headers[*num_headers].value_len; - goto skipvalue; - //goto hvalue; - } - if ( buf[16] == ':' ) { // X-Forwarded-Host: - headers[*num_headers].name = buf; - headers[*num_headers].name_len = 16; - buf += 18; - goto hvalue; - } - break; - case 'f': - if ( buf[5] == ':' ) { // From: - headers[*num_headers].name = buf; - headers[*num_headers].name_len = 5; - buf += 7; - goto hvalue; - } - if ( buf[9] == ':' ) { // Forwarded: - headers[*num_headers].name = buf; - headers[*num_headers].name_len = 9; - buf += 11; - goto hvalue; - } - break; - case 'i': - if ( buf[13] == ':' ) { // If-None-Match: - headers[*num_headers].name = buf; - headers[*num_headers].name_len = 13; - buf += 15; - goto hvalue; - } - if ( buf[17] == ':' ) { // If-Modified-Since: - headers[*num_headers].name = buf; - headers[*num_headers].name_len = 17; - buf += 19; - goto hvalue; - } - break; - case 'o': - headers[*num_headers].name = buf; - headers[*num_headers].name_len = 6; - buf += 8; - goto hvalue; - case 'r': - headers[*num_headers].name = buf; - headers[*num_headers].name_len = 7; - buf += 9; - goto hvalue; - case 't': // Transfer-Encoding: - headers[*num_headers].name = buf; - headers[*num_headers].name_len = 17; - buf += 19; - goto hvalue; - case 'u': - if ( buf[10] == ':' ) { // User-Agent: - headers[*num_headers].name = buf; - headers[*num_headers].name_len = 10; - buf += 12; - goto hvalue; - } - if ( buf[25] == ':' ) { // Upgrade-Insecure-Requests: - headers[*num_headers].name = buf; - headers[*num_headers].name_len = 25; - buf += 27; - goto hvalue; - } - break; - case 'a': - if ( buf[6] == ':' ) { // Accept: - headers[*num_headers].name = buf; - headers[*num_headers].name_len = 6; - buf += 8; - goto hvalue; - } - if ( buf[13] == ':' ) { // Authorization: - headers[*num_headers].name = buf; - headers[*num_headers].name_len = 13; - buf += 15; - goto hvalue; - } - if ( buf[14] == ':' ) { // Accept-Charset: - headers[*num_headers].name = buf; - headers[*num_headers].name_len = 14; - buf += 16; - goto hvalue; - } - if ( buf[15] == ':' ) { // Accept-Encoding: -Datetime - headers[*num_headers].name = buf; - headers[*num_headers].name_len = 15; - buf += 17; - goto hvalue; - } - if ( buf[16] == ':' ) { // Accept-Language: - headers[*num_headers].name = buf; - headers[*num_headers].name_len = 15; - buf += 17; - goto hvalue; - } - if ( buf[29] == ':' ) { // Access-Control-Request-Method: - headers[*num_headers].name = buf; - headers[*num_headers].name_len = 29; - buf += 31; - goto hvalue; - } - if ( buf[30] == ':' ) { // Access-Control-Request-Headers: - headers[*num_headers].name = buf; - headers[*num_headers].name_len = 30; - buf += 32; - goto hvalue; - } - break; - - } - if (!(*num_headers != 0 && (*buf == ' ' || *buf == '\t'))) { - /* parsing name, but do not discard SP before colon, see - * http://www.mozilla.org/security/announce/2006/mfsa2006-33.html */ - headers[*num_headers].name = buf; - static const char ALIGNED(16) ranges1[] = "\x00 " /* control chars and up to SP */ - "\"\"" /* 0x22 */ - "()" /* 0x28,0x29 */ - ",," /* 0x2c */ - "//" /* 0x2f */ - ":@" /* 0x3a-0x40 */ - "[]" /* 0x5b-0x5d */ - "{\377"; /* 0x7b-0xff */ - int found; - buf = findchar(buf, buf_end, ranges1, sizeof(ranges1) - 1, &found); - if (!found) { - CHECK_EOF(); - } - while (1) { - if (*buf == ':') { - break; - } else if (!token_char_map[(unsigned char)*buf]) { - *ret = -1; - return NULL; - } - ++buf; - CHECK_EOF(); - } - if ((headers[*num_headers].name_len = buf - headers[*num_headers].name) == 0) { - *ret = -1; - return NULL; - } - ++buf; - for (;; ++buf) { - CHECK_EOF(); - if (!(*buf == ' ' || *buf == '\t')) { - break; - } - } - } else { - headers[*num_headers].name = NULL; - headers[*num_headers].name_len = 0; - } -hvalue: - if ((buf = get_token_to_eol(buf, buf_end, &headers[*num_headers].value, &headers[*num_headers].value_len, ret)) == NULL) { - return NULL; - } -skipvalue: - ; - } - return buf; + i++; + } while ( buf < buf_end ); + *ret = -1; + return buf; } -//#endif - static const char *parse_request(const char *buf, const char *buf_end, const char **method, size_t *method_len, const char **path, size_t *path_len, int *minor_version, struct mr_header *headers, size_t *num_headers, size_t max_headers, int *ret, struct mr_request *mrr) { - /* skip first empty line (some clients add CRLF after POST content) */ - CHECK_EOF(); + // skip first empty line (some clients add CRLF after POST content) + CHECK_END(); if (*buf == '\015') { ++buf; EXPECT_CHAR('\012'); @@ -669,24 +168,31 @@ static const char *parse_request(const char *buf, const char *buf_end, const cha } // parse request line - //ADVANCE_TOKEN(*method, *method_len); + // TODO Support other methods switch (*(unsigned int *)buf) { - case MR_CHAR4_INT('G', 'E', 'T', ' '): - *method = buf; *method_len = 3; buf += 4; goto uri; - case MR_CHAR4_INT('P', 'O', 'S', 'T'): - *method = buf; *method_len = 4; buf += 5; goto uri; + case CHAR4_TO_INT('G', 'E', 'T', ' '): + *method = buf; *method_len = 3; buf += 4; break; + case CHAR4_TO_INT('P', 'O', 'S', 'T'): + *method = buf; *method_len = 4; buf += 5; break; + default: + *ret = -1; + return NULL; } - ++buf; -uri: - ADVANCE_TOKEN(*path, *path_len); + *path = buf; + int l = get_len_to_space(buf, buf_end); + if ( l == -1 ) { + *ret = -1; // TODO Should we return -2 (needs more bytes?) + return NULL; + } + buf += l; *path_len = l; ++buf; switch (*(unsigned long *)buf) { - case MR_CHAR8_INT('H', 'T', 'T', 'P','/','1','.','0'): + case CHAR8_TO_LONG('H', 'T', 'T', 'P','/','1','.','0'): *minor_version = 0; buf += 8; break; - case MR_CHAR8_INT('H', 'T', 'T', 'P','/','1','.','1'): + case CHAR8_TO_LONG('H', 'T', 'T', 'P','/','1','.','1'): *minor_version = 1; buf += 8; break; default: - *ret = -2; + *ret = -2; // TODO should be -1?? return NULL; } @@ -699,12 +205,8 @@ static const char *parse_request(const char *buf, const char *buf_end, const cha *ret = -2; return NULL; } - -#ifdef __DELMEAVX2__ return parse_headers_avx2(buf, buf_end, headers, num_headers, max_headers, ret, mrr); -#else - return parse_headers(buf, buf_end, headers, num_headers, max_headers, ret, mrr); -#endif + //return parse_headers(buf, buf_end, headers, num_headers, max_headers, ret, mrr); } static __inline__ unsigned long long rdtsc(void) @@ -715,7 +217,7 @@ static __inline__ unsigned long long rdtsc(void) } int mr_parse_request(const char *buf_start, size_t len, const char **method, size_t *method_len, const char **path, - size_t *path_len, int *minor_version, struct mr_header *headers, size_t *num_headers, size_t last_len,struct mr_request *mrr) + size_t *path_len, int *minor_version, struct mr_header *headers, size_t *num_headers, struct mr_request *mrr) { const char *buf = buf_start, *buf_end = buf_start + len; size_t max_headers = *num_headers; @@ -730,13 +232,6 @@ int mr_parse_request(const char *buf_start, size_t len, const char **method, siz *minor_version = -1; *num_headers = 0; - - /* if last_len != 0, check if the request is complete (a fast countermeasure - againt slowloris */ - if (last_len != 0 && is_complete(buf, buf_end, last_len, &r) == NULL) { - return r; - } - if ((buf = parse_request(buf, buf_end, method, method_len, path, path_len, minor_version, headers, num_headers, max_headers, &r, mrr)) == NULL) { return r; } @@ -747,6 +242,5 @@ int mr_parse_request(const char *buf_start, size_t len, const char **method, siz } -#undef CHECK_EOF +#undef CHECK_END #undef EXPECT_CHAR -#undef ADVANCE_TOKEN diff --git a/src/mrhttp/internals/mrhttpparser.h b/src/mrhttp/internals/mrhttpparser.h index 09816f4..ed20dfa 100644 --- a/src/mrhttp/internals/mrhttpparser.h +++ b/src/mrhttp/internals/mrhttpparser.h @@ -35,6 +35,12 @@ extern "C" { #endif +#define CHAR4_TO_INT(a, b, c, d) \ + (unsigned int)((d << 24) | (c << 16) | (b << 8) | a) +#define CHAR8_TO_LONG(a, b, c, d, e, f, g, h) \ + (((long)h << 56) | ((long)g << 48) | ((long)f << 40) \ + | ((long)e << 32) | (d << 24) | (c << 16) | (b << 8) | a) + // Table for converting to lower case #define TOLC(c) __lct[(unsigned char)c] static const unsigned char __lct[] __attribute__((aligned(64))) = { @@ -87,7 +93,7 @@ struct mr_request { // These functions return -2 if partial request, -1 if parsing failed, and the number of bytes parsed otherwise int mr_parse_request(const char *buf, size_t len, const char **method, size_t *method_len, const char **path, size_t *path_len, - int *minor_version, struct mr_header *headers, size_t *num_headers, size_t last_len, struct mr_request *mrr); + int *minor_version, struct mr_header *headers, size_t *num_headers, struct mr_request *mrr); struct mr_chunked_decoder { size_t bytes_left_in_chunk; diff --git a/src/mrhttp/internals/parser.c b/src/mrhttp/internals/parser.c index f840630..3b9599a 100644 --- a/src/mrhttp/internals/parser.c +++ b/src/mrhttp/internals/parser.c @@ -1,4 +1,5 @@ + #include #include #include @@ -21,6 +22,18 @@ static void print_buffer( char* b, int len ) { printf("\n"); } +static inline bool _isdigit(char c) { return c >= '0' && c <= '9'; } +static long my_strtol( char* s, int maxlen ) { + long l = 0; + int n = 0; + while (_isdigit(*s)) { + + l = (l * 10) + (*s++ - '0'); + n += 1; + if ( n >= maxlen ) return l; + } + return l; +} static void _reset(Parser* self, bool reset_buffer) { self->body_length = 0; @@ -56,8 +69,6 @@ int parser_data_received(Parser *self, PyObject *py_data, Request *request ) { DBG_PARSER printf("parser data\n%.*s\n",(int)datalen, data); // If we need more space increase the size of the buffer - // Can the headers be larger than our buffer size? -// No, HTTP does not define any limit. However most web servers do limit size of headers they accept. For example in Apache default limit is 8KB, in IIS it's 16K. Server will return 413 Entity Too Large error if headers size exceeds that limit. DBG_PARSER printf("parser datalen %zu buflen %ld buffer size %d\n", datalen, (self->end-self->start), self->buf_size); if ( unlikely( (datalen+(self->end-self->start)) > self->buf_size) ) { while ( (datalen+(self->end-self->start)) > self->buf_size ) self->buf_size *= 2; @@ -79,13 +90,12 @@ int parser_data_received(Parser *self, PyObject *py_data, Request *request ) { char *method, *path; int rc, minor_version; - //struct phr_header headers[100]; - size_t prevbuflen = 0, method_len, path_len;//, num_headers; + size_t method_len, path_len; request->num_headers = 100; // Max allowed headers DBG_PARSER printf("before parser requests\n"); - request->hreq.flags = 0; // TODO clear the mr_request struct - rc = mr_parse_request(self->start, self->end-self->start, (const char**)&method, &method_len, (const char**)&path, &path_len, &minor_version, request->headers, &(request->num_headers), prevbuflen, &(request->hreq)); + //request->hreq.flags = 0; // TODO This isn't currently used in the parser + rc = mr_parse_request(self->start, self->end-self->start, (const char**)&method, &method_len, (const char**)&path, &path_len, &minor_version, request->headers, &(request->num_headers), &(request->hreq)); DBG_PARSER printf("parser requests rc %d\n",rc); if ( rc < 0 ) return rc; // -2 incomplete, -1 error otherwise byte len of headers @@ -106,37 +116,32 @@ int parser_data_received(Parser *self, PyObject *py_data, Request *request ) { //self->body_length = request->hreq.body_length; -#define header_name_equal(val) \ +#define name_compare(val) \ header->name_len == strlen(val) && fast_compare(header->name, val, header->name_len) == 0 -#define header_value_equal(val) \ +#define value_compare(val) \ header->value_len == strlen(val) && fast_compare(header->value, val, header->value_len) == 0 for(struct mr_header* header = request->headers; header < request->headers + request->num_headers; header++) { + if(name_compare("Content-Type")) { + if ( header->value[0] == 'a' && header->value_len == 20 ) { //"application/mrpacker" + request->hreq.flags = 2; + } + } + if(name_compare("Content-Length")) { + self->body_length = my_strtol(header->value, header->value_len); - if(header_name_equal("Content-Length")) { - char * endptr = (char *)header->value + header->value_len; - self->body_length = strtol(header->value, &endptr, 10); - - // TODO If the request is too large + // TODO If the request is too large. I think we already checked - // 0 means error from strtol, but it is also a valid value + // Check for a bad 0 length - ie non digits for the length if ( self->body_length == 0 && !( header->value_len == 1 && *(header->value) == '0') ) { - //TODO ERROR - //error = invalid_headers; - goto error; - } - // If the value was not all digits we'll error here - if(endptr != (char*)header->value + header->value_len) { - //TODO ERROR - //error = invalid_headers; goto error; } - } else if(header_name_equal("Connection")) { - if (header_value_equal("close")) request->keep_alive = false; + } else if(name_compare("Connection")) { + if (value_compare("close")) request->keep_alive = false; } } @@ -146,9 +151,6 @@ int parser_data_received(Parser *self, PyObject *py_data, Request *request ) { DBG_PARSER printf("body:\n%.*s\n", (int)(self->end-self->start),self->start); - // No body - //if ( self->body_length == 0 ) { } - // Need more data if ( self->body_length > ( self->end - self->start ) ) { while ( (self->body_length+(self->end-self->start)) > self->buf_size ) self->buf_size *= 2; @@ -171,9 +173,6 @@ int parser_data_received(Parser *self, PyObject *py_data, Request *request ) { } } - if ( request->hreq.ip != NULL ) { - request->py_ip = PyUnicode_FromStringAndSize(request->hreq.ip, request->hreq.ip_len); - } if(!Protocol_on_body(self->protocol, self->start, self->body_length)) return -1; diff --git a/src/mrhttp/internals/protocol.c b/src/mrhttp/internals/protocol.c index 1c9a498..b3bd66c 100644 --- a/src/mrhttp/internals/protocol.c +++ b/src/mrhttp/internals/protocol.c @@ -328,7 +328,10 @@ void Protocol_on_memcached_reply( SessionCallbackData *scd, char *data, int data if ( !self->closed ) { Route *r = req->route; - if ( r->mrq ) { + if ( r->mrq || r->mrq2 ) { + MrqClient *py_mrq; + if ( r->mrq ) py_mrq = self->app->py_mrq; + if ( r->mrq2) py_mrq = self->app->py_mrq2; int slot = 0; // Pull slot from the first arg. Must be a number though a string won't break @@ -374,7 +377,7 @@ void Protocol_on_memcached_reply( SessionCallbackData *scd, char *data, int data memcpy(p, data, data_sz); p += data_sz; *p++ = ']'; - rc = MrqClient_pushj( (MrqClient*)self->app->py_mrq, slot, tmp, (int)(p-tmp) ); + rc = MrqClient_pushj( py_mrq, slot, tmp, (int)(p-tmp) ); free(tmp); } else { @@ -385,7 +388,7 @@ void Protocol_on_memcached_reply( SessionCallbackData *scd, char *data, int data p += req->body_len; memcpy(p, data, data_sz); p += data_sz; - rc = MrqClient_push ( (MrqClient*)self->app->py_mrq, slot, tmp, (int)(p-tmp) ); + rc = MrqClient_push ( py_mrq, slot, tmp, (int)(p-tmp) ); free(tmp); } @@ -429,9 +432,9 @@ void Protocol_on_memcached_reply( SessionCallbackData *scd, char *data, int data else { // Send body to mrq if ( req->py_mrpack == NULL ) { - MrqClient_pushj( (MrqClient*)self->app->py_mrq, slot, req->body, req->body_len ); + MrqClient_pushj( py_mrq, slot, req->body, req->body_len ); } else { - MrqClient_push ( (MrqClient*)self->app->py_mrq, slot, req->body, req->body_len ); + MrqClient_push ( py_mrq, slot, req->body, req->body_len ); } } } else { @@ -513,13 +516,16 @@ Protocol* Protocol_on_body(Protocol* self, char* body, size_t body_len) { } // if mrq return now as the user is not logged in - if ( r->mrq ) { + if ( r->mrq || r->mrq2 ) { return Protocol_handle_request( self, self->request, r ); } //? PyObject *ret = pipeline_queue(self, (PipelineRequest){true, 0, self->request, task}); } - if ( r->mrq ) { //TODO + if ( r->mrq || r->mrq2 ) { + MrqClient *py_mrq; + if ( r->mrq ) py_mrq = self->app->py_mrq; + if ( r->mrq2) py_mrq = self->app->py_mrq2; DBG printf("Route uses mrq\n"); int slot = 0; // Pull slot from the first arg. Must be a number @@ -531,9 +537,9 @@ Protocol* Protocol_on_body(Protocol* self, char* body, size_t body_len) { // Send body to mrq if ( self->request->py_mrpack == NULL ) { - MrqClient_pushj( (MrqClient*)self->app->py_mrq, slot, self->request->body, self->request->body_len ); + MrqClient_pushj( py_mrq, slot, self->request->body, self->request->body_len ); } else { - MrqClient_push ( (MrqClient*)self->app->py_mrq, slot, self->request->body, self->request->body_len ); + MrqClient_push ( py_mrq, slot, self->request->body, self->request->body_len ); } // TODO set a client member to say success/fail? Have to start failing if slow consumer / connection gone. @@ -556,6 +562,10 @@ Protocol* Protocol_handle_request(Protocol* self, Request* request, Route* r) { if ( self->request == request ) self->request = (Request*)MrhttpApp_get_request( self->app ); } + // If we have cached bytes + if ( r->cached ) { + if(!protocol_write_response(self, request, r->cached)) goto error; + } if(!(result = protocol_callPageHandler(self, r->func, request)) ) { @@ -656,7 +666,7 @@ Protocol* Protocol_handle_request(Protocol* self, Request* request, Route* r) { if(!protocol_write_response(self, request, result)) goto error; - // TODO We aren't closing the connection if issued a connection: close header. This is okay behind nginx + // TODO We aren't closing the connection if issued a connection: close header. //if ( !request->keep_alive ) Protocol_close(self); Py_DECREF(result); @@ -886,6 +896,7 @@ PyObject* protocol_task_done(Protocol* self, PyObject* task) PyObject* result = Py_True; PipelineRequest *r; + // TODO? // task is the task that just finished. // If task is at the head of the Q then return the response. // then continue looping through the Q and return a response for anything that is already done diff --git a/src/mrhttp/internals/request.c b/src/mrhttp/internals/request.c index ffb9306..7fb69d3 100644 --- a/src/mrhttp/internals/request.c +++ b/src/mrhttp/internals/request.c @@ -1,6 +1,4 @@ - - #include #include #include @@ -39,7 +37,7 @@ ((x <= '9' ? 0 : 9) + (x & 0x0f)) #define is_hex(x) ((x >= '0' && x <= '9') || (x >= 'A' && x <= 'F')) -#define CHAR4_INT(a, b, c, d) \ +#define CHAR4_TO_INT(a, b, c, d) \ (unsigned int)((d << 24) | (c << 16) | (b << 8) | a) @@ -54,6 +52,11 @@ //static PyObject* request; +static unsigned long TZCNT(unsigned long long in) { + unsigned long res; + asm("tzcnt %1, %0\n\t" : "=r"(res) : "r"(in)); + return res; +} PyObject* Request_new(PyTypeObject *type, PyObject *args, PyObject *kwds) @@ -78,7 +81,6 @@ void Request_dealloc(Request* self) { Py_XDECREF(self->py_args); Py_XDECREF(self->py_path); Py_XDECREF(self->py_method); - //Py_XDECREF(self->py_ip); Py_XDECREF(self->py_json); Py_XDECREF(self->py_mrpack); Py_XDECREF(self->py_form); @@ -118,7 +120,6 @@ void Request_reset(Request *self) { Py_XDECREF(self->py_file); self->py_file = NULL; Py_XDECREF(self->py_files); self->py_files= NULL; Py_XDECREF(self->py_user); self->py_user= NULL; - self->py_ip = NULL; self->hreq.ip = NULL; self->hreq.flags = 0; Py_XDECREF(self->py_mrq_servers_down); self->py_mrq_servers_down= NULL; @@ -162,77 +163,157 @@ PyObject* Request_get_transport(Request* self, void* closure) { Py_RETURN_NONE; } -//#ifdef __SSE4_2__ +static inline int path_decode(char* buf, int len, int *qs_len) { + unsigned long msk; + int i=0,tz; // 32B index + int cnt = 0; + unsigned int shifted; + char *sbuf = buf; + char *obuf = buf; + char *buf_end = buf+len; + char *wbuf; + int found = 0; -// /spanish/objetos%20voladores%20no%20identificados?foo=bar -static inline size_t sse_decode(char* path, ssize_t length, size_t *qs_len) { - //DBG printf("sse_decode >%.*s<\n", (int)length, path); - if (length == 0) return length; - char *pat = path; - static char ranges1[] = "%%" "??"; - char *end = path + length; - int found; + __m256i m37 = _mm256_set1_epi8(37); + __m256i m63 = _mm256_set1_epi8(63); - // We only findchar once - benchmark one or more % encodings with continuing to use findchar ( Spanish / Chinese ) do { - //DBG printf("sse_decode >%.*s<\n", (int)length, path); - pat = findchar(pat, end, ranges1, sizeof(ranges1) - 1, &found); - if ( found ) { - if(*pat == '%' && is_hex(*(pat + 1)) && is_hex(*(pat + 2))) { - *pat = (hex_to_dec(*(pat + 1)) << 4) + hex_to_dec(*(pat + 2)); - pat+=3; - length -= 2; + const char *block_start = obuf+64*i; + __m256i b0 = _mm256_loadu_si256((const __m256i *) block_start); + __m256i b1 = _mm256_loadu_si256((const __m256i *) (block_start+32)); + msk = (unsigned int) _mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b0, m37), _mm256_cmpeq_epi8(b0, m63) ) ) | + ((unsigned long) _mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b1, m37), _mm256_cmpeq_epi8(b1, m63) ) ) << 32); + + while (1) { + + //if ( buf >= buf_end ) { goto decdone; } + shifted = buf-block_start; + if ( shifted >= 64 ) break; + tz = TZCNT((msk >> shifted)); + if ( tz < 64 ) { + buf += tz; + //printf( " fnd >%.*s<\n", (int)(buf-sbuf), sbuf ); + if ( buf >= buf_end ) { goto decdone; } + if ( *buf == '?' ) { + len -= buf_end-buf; + *qs_len = buf_end-buf; + //printf("path_decode len %d path >%.*s<\n", (int)len, (int)len, obuf); + goto decdone; + } + if ( *buf == '%' ) { + if ( found ) { + memcpy( wbuf, sbuf, buf-sbuf ); + wbuf += buf-sbuf; + *wbuf = (hex_to_dec(buf[1]) << 4) + hex_to_dec(buf[2]); + wbuf++; + } else { + found = 1; + *buf = (hex_to_dec(buf[1]) << 4) + hex_to_dec(buf[2]); + wbuf = buf+1; + } + len -= 2; + buf += 3; + } + sbuf = buf; } else { - *qs_len = end-pat; - length -= end-pat; + buf += 64 - shifted; break; } + } - } while (0); + i+=1; + if ( buf >= buf_end ) { goto decdone; } + } while ( buf < buf_end ); // Why doesn't this work + +decdone: + if ( found ) { + memcpy( wbuf, sbuf, buf_end-sbuf ); + } + return len; +} - if( !found || *pat == '?') return length; +static inline PyObject* parse_query_args( char *buf, size_t len ) { + unsigned long long msk; + int i=0,tz; // 32B index + unsigned int shifted; + char *sbuf = buf; + char *obuf = buf; + int state = 0; + int ignore_me = 0; - char *write = pat; - if ( found ) write -= 2; - char *read = pat; - for (;read < end;) { - if (read[0] == '?') { - length -= end-read; - *qs_len = end-pat; - break; - } - if ( read[0] == '%' ) { - if( is_hex(read[1]) && is_hex(read[2]) ) { - *write = (hex_to_dec(read[1]) << 4) + hex_to_dec(read[2]); - write+=1; - read += 3; - length-=2; - } else { - if (read > write) { - write[0] = read[0]; - write[1] = read[1]; + PyObject* args = PyDict_New(); + PyObject* key = NULL; PyObject* value = NULL; + + if ( len == 0 ) return; + //len = path_decode( buf, len, &ignore_me ); + char *buf_end = buf+len; + + __m256i m38 = _mm256_set1_epi8(38); // & + __m256i m61 = _mm256_set1_epi8(61); // = + do { + const char *block_start = obuf+64*i; + __m256i b0 = _mm256_loadu_si256((const __m256i *) block_start); + __m256i b1 = _mm256_loadu_si256((const __m256i *) (block_start+32)); + msk = (unsigned int) _mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b0, m38), _mm256_cmpeq_epi8(b0, m61) ) ) | + ((unsigned long) _mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b1, m38), _mm256_cmpeq_epi8(b1, m61) ) ) << 32); + while (1) { + + //if ( buf >= buf_end ) { goto decdone; } + shifted = buf-block_start; + if ( shifted >= 64 ) break; + tz = TZCNT((msk >> shifted)); + if ( tz < 64 ) { + buf += tz; + if ( buf >= buf_end ) { goto pdone; } + if ( *buf == '=' ) { + if ( state == 1 ) { buf+=1; continue; } + //printf( " key >%.*s<\n", (int)(buf-sbuf), sbuf ); + //key = PyUnicode_FromStringAndSize(sbuf, buf-sbuf); + len = path_decode( sbuf, buf-sbuf, &ignore_me ); + key = PyUnicode_FromStringAndSize(sbuf, len); + + state = 1; + buf += 1; + } + else if ( *buf == '&' ) { + if ( state == 0 ) { buf+=1; continue; } + //printf( " val >%.*s<\n", (int)(buf-sbuf), sbuf); + //value = PyUnicode_FromStringAndSize(sbuf, buf-sbuf); + len = path_decode( sbuf, buf-sbuf, &ignore_me ); + value = PyUnicode_FromStringAndSize(sbuf, len); + PyDict_SetItem(args, key, value); + Py_XDECREF(key); + Py_XDECREF(value); + + state = 0; + buf+=1; } - read += 2; - write += 2; + sbuf = buf; + } else { + buf += 64 - shifted; + break; } - } else { - if (read > write) { - write[0] = read[0]; - } - read++; - write++; } - } - DBG printf("sse_decode len %d path >%.*s<\n", (int)length, (int)length, path); - - return length; + i+=1; + if ( buf >= buf_end ) { goto pdone; } + } while ( buf < buf_end ); // Why doesn't this work + +pdone: + //printf( " done >%.*s<\n", (int)(buf_end-sbuf), sbuf ); + //value = PyUnicode_FromStringAndSize(sbuf, buf_end-sbuf); + len = path_decode( sbuf, buf_end-sbuf, &ignore_me ); + value = PyUnicode_FromStringAndSize(sbuf, len); + PyDict_SetItem(args, key, value); + Py_XDECREF(key); + Py_XDECREF(value); + return args; } -//#endif + void request_decodePath(Request* self) { if(!self->path_decoded) { - self->path_len = sse_decode( self->path, self->path_len, &(self->qs_len) ); + self->path_len = path_decode( self->path, self->path_len, &(self->qs_len) ); self->path_decoded = true; } } @@ -322,137 +403,150 @@ PyObject* Request_get_headers(Request* self, void* closure) { Py_XINCREF(self->py_headers); return self->py_headers; } -PyObject* Request_get_ip(Request* self, void* closure) { - if(!self->py_ip) { - if ( self->hreq.ip_len ) { - self->py_ip = PyUnicode_FromStringAndSize(self->hreq.ip, self->hreq.ip_len); - } else { - self->py_ip = Py_None; - } - } - Py_INCREF(self->py_ip); - return self->py_ip; -} static inline PyObject* parseCookies( Request* r, char *buf, size_t buflen ) { - char *end = buf + buflen; - char *last = buf; + unsigned int msk; + int i=0,tz; // 32B index + int cnt = 0; + unsigned int shifted; + const char *sbuf = buf; + const char *obuf = buf; + const char *buf_end = buf+buflen; + int name_or_value = 0; + int found = 0; + + __m256i m59 = _mm256_set1_epi8(59); + __m256i m61 = _mm256_set1_epi8(61); + PyObject* cookies = PyDict_New(); PyObject* key = NULL; PyObject* value = NULL; DBG printf("parse cookies: %.*s\n",(int)buflen, buf); - static char ALIGNED(16) ranges1[] = "==" ";;" "\x00 "; // Control chars up to space illegal - int found; - int state = 0; - int grab_session = 0; -//Cookie: key=session_key; bar=2; nosemi=foo - do { - last = buf; - buf = findchar(buf, end, ranges1, sizeof(ranges1) - 1, &found); - if ( found ) { - if ( *buf == '=' ) { - if ( state == 0 ) { - // Save out the mrsession id - if ( buf-last == 9 && ( *((unsigned int *)(last)) == CHAR4_INT('m', 'r', 's','e') ) ) { - DBG printf("Grab session\n"); - grab_session = 1; + do { + const char *block_start = obuf+32*i; + __m256i b0 = _mm256_loadu_si256((const __m256i *) block_start); + msk = _mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b0, m59), _mm256_cmpeq_epi8(b0, m61) ) ); + while (1) { + //if ( buf >= buf_end ) { goto sesdone; } + shifted = buf-block_start; + if ( shifted >= 32 ) break; + tz = TZCNT((msk >> shifted)); + if ( tz < 32 ) { + buf += tz; + DBG printf( " fnd >%.*s<\n", buf-sbuf, sbuf ); + if ( buf >= buf_end ) { goto sesdone; } + if ( name_or_value == 1 ) { + if ( *buf == '=' ) { buf += 1; continue; } // = in value field + if ( found ) { + DBG printf("session key %.*s\n", (int)(buf-sbuf), sbuf); + r->session_id = sbuf; + r->session_id_sz = buf-sbuf; } - key = PyUnicode_FromStringAndSize(last, buf-last); //TODO error - DBG printf("session key %.*s\n", (int)(buf-last), last); - state = 1; + value = PyUnicode_FromStringAndSize(sbuf, buf-sbuf); //TODO error + PyDict_SetItem(cookies, key, value); // == -1) goto loop_error; + Py_XDECREF(key); + Py_XDECREF(value); buf+=1; + name_or_value = 0; } else { - // If we're in the value ignore the = so cookie name/value splits on the first = - while(found && *buf == '=') buf = findchar(++buf, end, ranges1, sizeof(ranges1) - 1, &found); - } - } - else if ( *buf == ';' ) { - if ( state == 0 ) key = PyUnicode_FromString(""); - if (grab_session) { - grab_session = 0; - r->session_id = last; - r->session_id_sz = buf-last; + key = PyUnicode_FromStringAndSize(sbuf, buf-sbuf); //TODO error + if ( buf-sbuf == 9 && ( *((unsigned int *)(sbuf)) == CHAR4_TO_INT('m', 'r', 's','e') ) ) { + found = 1; + } + name_or_value = 1; } - value = PyUnicode_FromStringAndSize(last, buf-last); //TODO error - DBG printf(" value %.*s\n", (int)(buf-last), last); - state = 0; - PyDict_SetItem(cookies, key, value); // == -1) goto loop_error; - Py_XDECREF(key); - Py_XDECREF(value); - buf+=1; - while ( *buf == 32 ) buf++; - } - else { - // Bad character found so skip - state = 0; - while(found && *buf != ';') buf = findchar(++buf, end, ranges1, sizeof(ranges1) - 1, &found); - if ( buf != end ) buf += 1; - while ( *buf == 32 ) buf++; + buf += 1; + sbuf = buf; + } else { + buf += 32 - shifted; + break; } - //else if(*buf == '%' && is_hex(*(buf + 1)) && is_hex(*(buf + 2))) { - //*write = (hex_to_dec(*(buf + 1)) << 4) + hex_to_dec(*(buf + 2)); - //write+=1; - //length -= 2; - //} - } - } while( found ); - - // If the trailing ; is left off we need to finish up - if (state) { - if (grab_session) { - grab_session = 0; - r->session_id = last; - r->session_id_sz = buf-last; - DBG printf("session2 %.*s\n", r->session_id_sz, r->session_id); + } - value = PyUnicode_FromStringAndSize(last, buf-last); //TODO error - PyDict_SetItem(cookies, key, value); // == -1) goto loop_error; + i+=1; + if ( buf >= buf_end ) { goto sesdone; } + } while ( buf-obuf < buf_end-obuf ); + +sesdone: + if ( found ) { + r->session_id = sbuf; + r->session_id_sz = buf_end-sbuf; + } + if ( name_or_value ) { + value = PyUnicode_FromStringAndSize(sbuf, buf_end-sbuf); //TODO error + PyDict_SetItem(cookies, key, value); Py_XDECREF(key); Py_XDECREF(value); } - return cookies; } + +// We can probably create a separate 1 and 2 reg avx2 find func to share static inline void getSession( Request* r, char *buf, size_t buflen ) { - char *end = buf + buflen; - char *last = buf; + unsigned int msk; + int i=0,tz; // 32B index + int cnt = 0; + unsigned int shifted; + const char *sbuf = buf; + const char *obuf = buf; + const char *buf_end = buf+buflen; + int name_or_value = 0; + int found = 0; + + __m256i m59 = _mm256_set1_epi8(59); + __m256i m61 = _mm256_set1_epi8(61); - static char ALIGNED(16) ranges1[] = "==" ";;"; - int found; - int state = 0; - do { - last = buf; - buf = findchar(buf, end, ranges1, sizeof(ranges1) - 1, &found); - if ( found ) { - if ( *buf == '=' ) { - if ( state == 0 ) { - // Save out the mrsession id - if ( buf-last == 9 && ( *((unsigned int *)(last)) == CHAR4_INT('m', 'r', 's','e') ) ) { - DBG printf("Grab session\n"); - state = 1; + do { + const char *block_start = obuf+32*i; + __m256i b0 = _mm256_loadu_si256((const __m256i *) block_start); + msk = _mm256_movemask_epi8( _mm256_or_si256(_mm256_cmpeq_epi8(b0, m59), _mm256_cmpeq_epi8(b0, m61) ) ); + while (1) { + //if ( buf >= buf_end ) { goto sesdone; } + shifted = buf-block_start; + if ( shifted >= 32 ) break; + tz = TZCNT((msk >> shifted)); + if ( tz < 32 ) { + buf += tz; + //printf( " fnd >%.*s<\n", buf-sbuf, sbuf ); + if ( buf >= buf_end ) { goto sesdone; } + if ( name_or_value == 1 ) { + if ( *buf == '=' ) { buf += 1; continue; } // = in value field + if ( found ) { + //printf( " done >%.*s<\n", buf-sbuf, sbuf ); + r->session_id = sbuf; + r->session_id_sz = buf-sbuf; + return; } buf+=1; - } - } - else if ( *buf == ';' ) { - if (state == 1 ) { - r->session_id = last; - r->session_id_sz = buf-last; - return; + name_or_value = 0; + } else { + if ( buf-sbuf == 9 && ( *((unsigned int *)(sbuf)) == CHAR4_TO_INT('m', 'r', 's','e') ) ) { + found = 1; + } + name_or_value = 1; } - state = 0; - buf+=1; - while ( *buf == 32 ) buf++; + buf += 1; + sbuf = buf; + } else { + buf += 32 - shifted; + break; } + } - } while( found ); - if (state) { - r->session_id = last; - r->session_id_sz = buf-last; + i+=1; + if ( buf >= buf_end ) { goto sesdone; } + } while ( buf-obuf < buf_end-obuf ); + +sesdone: + if ( found ) { + r->session_id = sbuf; + r->session_id_sz = buf-sbuf; + //printf( " sesdone >%.*s<\n", buf-sbuf, sbuf ); } } + static inline PyObject* Request_decode_cookies(Request* self) { for(struct mr_header* header = self->headers; header < self->headers + self->num_headers; header++) { @@ -492,67 +586,6 @@ PyObject* Request_get_body(Request* self, void* closure) return self->py_body; } -static inline PyObject* parse_query_args( char *buf, size_t buflen ) { - char *end = buf + buflen; - char *last = buf; - PyObject* args = PyDict_New(); - - if ( buflen == 0 ) return args; - - PyObject* key = NULL; PyObject* value = NULL; - - static char ALIGNED(16) ranges1[] = "==" "&&"; - int found; - int state = 0; - int grab_session = 0; - size_t len; - // foo=bar&key=23%28 - do { - buf = findchar(buf, end, ranges1, sizeof(ranges1) - 1, &found); - if ( found ) { - if ( *buf == '=' ) { - len = sse_decode( last, buf-last, NULL ); - key = PyUnicode_FromStringAndSize(last, len); //TODO error - state = 1; - buf+=1; - last = buf; - } - else if ( *buf == '&' ) { - if ( state == 0 ) key = PyUnicode_FromString(""); - - len = sse_decode( last, buf-last, NULL ); - value = PyUnicode_FromStringAndSize(last, len); - state = 0; - PyDict_SetItem(args, key, value); // == -1) goto loop_error; - Py_XDECREF(key); - Py_XDECREF(value); - buf+=1; - while ( *buf == 32 ) buf++; - last = buf; - } - else { - printf(" ERR found not = or ; %.*s\n", 5, buf ); - } - } - } while( found ); - - if ( buf == end ) { - if ( state == 0 ) key = PyUnicode_FromString(""); - if ( buf == end && *(buf-1) == ' ' ) { - len = sse_decode( last, buf-last-1, NULL ); - value = PyUnicode_FromStringAndSize(last, len); //TODO error - } else { - len = sse_decode( last, buf-last, NULL ); - value = PyUnicode_FromStringAndSize(last, len); //TODO error - } - state = 0; - PyDict_SetItem(args, key, value); // == -1) goto loop_error; - Py_XDECREF(key); - Py_XDECREF(value); - } - - return args; -} PyObject* Request_get_path(Request* self, void* closure) @@ -608,6 +641,7 @@ PyObject* Request_notfound(Request* self) Py_RETURN_NONE; } + PyObject* Request_parse_mp_form(Request* self) { @@ -743,7 +777,7 @@ PyObject* Request_parse_mp_form(Request* self) { body = name = filename = content_type = NULL; } - if ( p[2+bndlen] == '-' ) break; // Last boundary + if ( p[2+bndlen] == '-' ) break; // Last boundary has -- appended state = 1; } } @@ -805,8 +839,8 @@ PyObject* Request_parse_mp_form(Request* self) { //if ( state == 2 ) { //} - - p = findchar(p, pend, crlf, sizeof(crlf) - 1, &found); + //p = findchar(p, pend, crlf, sizeof(crlf) - 1, &found); + p = my_get_eol(p, pend); p += 2; } diff --git a/src/mrhttp/internals/request.h b/src/mrhttp/internals/request.h index 0376864..b659096 100644 --- a/src/mrhttp/internals/request.h +++ b/src/mrhttp/internals/request.h @@ -45,7 +45,6 @@ struct Request { PyObject* py_method; PyObject* transport; PyObject* app; - PyObject* py_ip; PyObject* py_headers; PyObject* py_cookies; PyObject* py_body; diff --git a/src/mrhttp/internals/router.c b/src/mrhttp/internals/router.c index 81ca17e..0c8d487 100644 --- a/src/mrhttp/internals/router.c +++ b/src/mrhttp/internals/router.c @@ -51,6 +51,25 @@ static int numInString( char c, char* s, int len ) { return ret; } +PyObject* Router_update_cached_route(Router* self, PyObject* item) { + + PyObject *path = PyList_GET_ITEM( item, 0 ); + PyObject *b = PyList_GET_ITEM( item, 1 ); + + Py_ssize_t plen; + char *p = PyUnicode_AsUTF8AndSize( path, &plen ); + + Route *r = self->staticRoutes; + for (int i = 0; inumStaticRoutes; i++,r++ ) { + //DBG printf("request path len %d - %.*s\n", (int)request->path_len, (int)request->path_len, request->path); + //DBG printf("route path %.*s \n", (int)r->len, r->path); + if ( plen == r->len && !memcmp(r->path, p, plen) ) { + r->cached = b; + Py_RETURN_NONE; + } + } + Py_RETURN_NONE; +} PyObject *Router_setupRoutes (Router* self) { //PyObject *sroutes = self->pyStaticRoutes; //PyObject_GetAttrString((PyObject*)self, "static_routes"); @@ -69,7 +88,7 @@ PyObject *Router_setupRoutes (Router* self) { r = PyList_GetItem(sroutes, i); rte->iscoro = false; rte->session = false; - rte->mrq = false; rte->append_user = false; + rte->mrq = false; rte->mrq2 = false; rte->append_user = false; PyObject *handler = PyLong_AsVoidPtr(PyDict_GetItemString( r, "handler" )); rte->func = handler; @@ -78,11 +97,16 @@ PyObject *Router_setupRoutes (Router* self) { if ( Py_True == PyDict_GetItemString( r, "iscoro" ) ) rte->iscoro = true; if ( Py_True == PyDict_GetItemString( r, "session" ) ) rte->session = true; if ( Py_True == PyDict_GetItemString( r, "mrq" ) ) rte->mrq = true; + if ( Py_True == PyDict_GetItemString( r, "mrq2" ) ) rte->mrq2 = true; if ( Py_True == PyDict_GetItemString( r, "append_user" ) ) rte->append_user = true; //if ( Py_True == PyDict_GetItemString( r, "append_user" ) ) printf("mrq append user set\n"); o = PyDict_GetItemString( r, "type" ); if (o) rte->mtype = PyLong_AsLong(o); rte->user_key = PyDict_GetItemString( r, "user_key" ); + rte->cached = PyDict_GetItemString( r, "cached" ); + if ( Py_True == PyDict_GetItemString( r, "cache" ) ) { + rte->cached = PyObject_CallFunctionObjArgs(handler, r, NULL); + } DBG printf(" path %.*s func ptr %p\n", (int)rte->len, rte->path, rte->func); } @@ -111,10 +135,11 @@ PyObject *Router_setupRoutes (Router* self) { rte->path = PyUnicode_AsUTF8AndSize( o, &(rte->len) ); DBG printf( " path len %ld str %.*s\n", rte->len, (int)rte->len, rte->path ); - rte->iscoro = false; rte->session = false; rte->mrq = false; + rte->iscoro = false; rte->session = false; rte->mrq = false; rte->mrq2 = false; if ( Py_True == PyDict_GetItemString( r, "iscoro" ) ) rte->iscoro = true; if ( Py_True == PyDict_GetItemString( r, "session" ) ) rte->session = true; if ( Py_True == PyDict_GetItemString( r, "mrq" ) ) rte->mrq = true; + if ( Py_True == PyDict_GetItemString( r, "mrq2" ) ) rte->mrq2 = true; if ( Py_True == PyDict_GetItemString( r, "append_user" ) ) rte->append_user = true; o = PyDict_GetItemString( r, "type" ); if (o) rte->mtype = PyLong_AsLong(o); @@ -178,8 +203,8 @@ Route* router_getRoute(Router* self, Request* request) { Route *r = self->staticRoutes; for (int i = 0; inumStaticRoutes; i++,r++ ) { - DBG printf("request path len %d - %.*s\n", (int)request->path_len, (int)request->path_len, request->path); - DBG printf("route path %.*s \n", (int)r->len, r->path); + //DBG printf("request path len %d - %.*s\n", (int)request->path_len, (int)request->path_len, request->path); + //DBG printf("route path %.*s \n", (int)r->len, r->path); if ( plen == r->len && !memcmp(r->path, request->path, plen) ) { DBG printf("router found path %.*s == %.*s\n", (int)r->len, r->path, (int)request->path_len, request->path); return r; diff --git a/src/mrhttp/internals/router.h b/src/mrhttp/internals/router.h index 36139b2..25887f2 100644 --- a/src/mrhttp/internals/router.h +++ b/src/mrhttp/internals/router.h @@ -15,10 +15,12 @@ typedef struct { bool iscoro; bool session; bool mrq; + bool mrq2; bool append_user; char mtype; int max_byte_size; + PyObject *cached; PyObject *user_key; //char *user_key; //long user_key_len; @@ -38,5 +40,6 @@ PyObject *Router_new (PyTypeObject* self, PyObject *args, PyObject *kwargs); int Router_init (Router* self, PyObject *args, PyObject *kwargs); void Router_dealloc(Router* self); PyObject *Router_setupRoutes(Router* self); +PyObject* Router_update_cached_route(Router* self, PyObject* item); Route* router_getRoute(Router* self, Request* request); diff --git a/src/mrhttp/internals/utils.c b/src/mrhttp/internals/utils.c index 792bfbc..1be2952 100644 --- a/src/mrhttp/internals/utils.c +++ b/src/mrhttp/internals/utils.c @@ -33,23 +33,13 @@ PyObject* myrandint(PyObject* self, PyObject* args) #define unlikely(x) (x) #endif -// Valgrind doesn't support mm_cmpestri so replace findchar -char *valgrind_zfindchar(char *buf, char *buf_end, char *ranges, size_t ranges_size, int *found) -{ - //printf("DELME ranges sz %d\n", ranges_size); - *found = 0; - char *p = buf; - while ( p < buf_end ) { - for ( int i = 0; i < ranges_size; i += 2 ) { - if ( p >= ranges[i] && p <= ranges[i+1] ) { - *found = 1; - return p; - } - } - p++; - } - return p; +unsigned long TZCNT(unsigned long long in) { + unsigned long res; + asm("tzcnt %1, %0\n\t" : "=r"(res) : "r"(in)); + return res; } + + // Search for a range of characters and return a pointer to the location or buf_end if none are found char *findchar(char *buf, char *buf_end, char *ranges, size_t ranges_size, int *found) { @@ -91,6 +81,26 @@ char *findchar(char *buf, char *buf_end, char *ranges, size_t ranges_size, int * return buf; } +char *my_get_eol(char *buf, char *buf_end) { + const char *start = buf; + __m256i m13 = _mm256_set1_epi8(13); + while (1) + { + __m256i v0 = _mm256_loadu_si256((const __m256i *)buf); + __m256i v1 = _mm256_cmpeq_epi8(v0, m13); + unsigned long vmask = _mm256_movemask_epi8(v1); + if (vmask != 0) { + buf += TZCNT(vmask); + if ( buf > buf_end ) return buf_end; + break; + } + buf += 32; //pSrc1++; + if ( buf >= buf_end ) return buf_end; + } + return buf; +} + + static char escbuf[16*1024]; diff --git a/src/mrhttp/request.py b/src/mrhttp/request.py index e793a51..83a33c5 100755 --- a/src/mrhttp/request.py +++ b/src/mrhttp/request.py @@ -20,6 +20,7 @@ class Request(mrhttp.CRequest): response = mrhttp.Response() def __init__(self): super().__init__(self) + self._ip = None pass def parsed_content_type(self): @@ -64,6 +65,18 @@ def file(self): self.parse_mp_form() return self._file + @property + def ip(self): + if self._ip == None: + self._ip = self.headers.get("CF-Connecting-IP") + if self._ip == None: + self._ip = self.headers.get("X-Real-IP") + if self._ip == None: + self._ip = self.headers.get("X-Forwarded-For") + + return self._ip + + @property def files(self): if self._files == None: diff --git a/src/mrhttp/router.py b/src/mrhttp/router.py index 607f2c9..e1b9743 100644 --- a/src/mrhttp/router.py +++ b/src/mrhttp/router.py @@ -16,6 +16,7 @@ def __init__(self): def finalize_routes(self): self.routes.sort(key=lambda x: x["sortlen"],reverse=True) + def add_route(self, handler, uri, methods=['GET'], options=[],_type="html"): if handler.__name__ in self.func_namemap: @@ -47,6 +48,8 @@ def add_route(self, handler, uri, methods=['GET'], options=[],_type="html"): #r["user_key"] = optiondict.get("append_user_key",None) if "session" in options: r["session"] = True if "mrq" in options: r["mrq"] = True + if "mrq2" in options: r["mrq2"] = True + if "cache" in options: r["cache"] = True if "append_user" in options: r["append_user"] = True # Static routes if not "{" in uri: @@ -64,4 +67,14 @@ def add_route(self, handler, uri, methods=['GET'], options=[],_type="html"): r["num_segs"] = len(segs) self.routes.append( r ) + def add_cached_route(self, uri, byts, _type="html"): + def fake_handler(): + print("ERROR fake_handler called") + r = {} + r["path"] = uri + r["methods"] = ["GET"] + r["handler"] = id(fake_handler) + r["cached"] = byts + r["type"] = 0 + self.static_routes.append( r ) diff --git a/tests/load_session.py b/tests/load_session.py new file mode 100644 index 0000000..fd83661 --- /dev/null +++ b/tests/load_session.py @@ -0,0 +1,36 @@ + +import asyncio, time +import asyncmrcache, mrpacker + +import tracemalloc +tracemalloc.start() + +import uvloop +asyncio.set_event_loop_policy(uvloop.EventLoopPolicy()) + +def lcb(client): + print("Lost connection") + +async def run(loop): + + rc = await asyncmrcache.create_client( [("localhost",7000)], loop, lost_cb=lcb) + + k = b"mrsession43709dd361cc443e976b05714581a7fb" + user = {"user":"test","id":12 } + + await rc.set(k,mrpacker.pack(user)) + print(await rc.get(k)) + + k = b"43709dd361cc443e976b05714581a7fb" + await rc.set(k,mrpacker.pack(user)) + print(await rc.get(k)) + + await rc.close() + +if __name__ == '__main__': + loop = asyncio.get_event_loop() + loop.run_until_complete(run(loop)) + loop.close() + print("DONE") + + diff --git a/tests/lua/bighdr.lua b/tests/lua/bighdr.lua new file mode 100755 index 0000000..cf87f16 --- /dev/null +++ b/tests/lua/bighdr.lua @@ -0,0 +1,43 @@ +-- example script demonstrating HTTP pipelining + +init = function(args) + local r = {} + wrk.headers["Content-Type"] = "application/x-www-form-urlencoded" + wrk.headers["User-Agent"] = "Mozilla/5.0 (X11; Linux x86_64) Gecko/20130501 Firefox/30.0 AppleWebKit/600.00 Chrome/30.0.0000.0 Trident/10.0 Safari/600.00" + wrk.headers["Cookie"] = "mrsession=43709dd361cc443e976b05714581a7fb; foo=fdsfdasdfasdfdsfasdfsdfsdfasdfas; short=fazc;" + wrk.headers["Accept"] = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" + wrk.headers["Accept-Language"] = "en-US,en;q=0.5" + table.insert(r, wrk.format(nil, "/")) + table.insert(r, wrk.format(nil, "/")) + table.insert(r, wrk.format(nil, "/")) + table.insert(r, wrk.format(nil, "/")) + table.insert(r, wrk.format(nil, "/")) + table.insert(r, wrk.format(nil, "/")) + table.insert(r, wrk.format(nil, "/")) + table.insert(r, wrk.format(nil, "/")) + table.insert(r, wrk.format(nil, "/")) + table.insert(r, wrk.format(nil, "/")) + table.insert(r, wrk.format(nil, "/")) + table.insert(r, wrk.format(nil, "/")) + table.insert(r, wrk.format(nil, "/")) + table.insert(r, wrk.format(nil, "/")) + table.insert(r, wrk.format(nil, "/")) + table.insert(r, wrk.format(nil, "/")) + table.insert(r, wrk.format(nil, "/")) + table.insert(r, wrk.format(nil, "/")) + table.insert(r, wrk.format(nil, "/")) + table.insert(r, wrk.format(nil, "/")) + table.insert(r, wrk.format(nil, "/")) + table.insert(r, wrk.format(nil, "/")) + table.insert(r, wrk.format(nil, "/")) + table.insert(r, wrk.format(nil, "/")) + table.insert(r, wrk.format(nil, "/")) + table.insert(r, wrk.format(nil, "/")) + table.insert(r, wrk.format(nil, "/")) + + req = table.concat(r) +end + +request = function() + return req +end diff --git a/tests/lua/form.lua b/tests/lua/form.lua index 63b6ce8..8b6278b 100755 --- a/tests/lua/form.lua +++ b/tests/lua/form.lua @@ -1,3 +1,3 @@ wrk.method = "POST" -wrk.body = 'param1=value1¶m2=value2' +wrk.body = 'param1=value1¶m2=value2&c%C3%B3mo=puedes&fffffffffffffffffffffffffffffffffffff%20ffffffffffffffffff=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' wrk.headers["Content-Type"] = "application/x-www-form-urlencoded" diff --git a/tests/lua/pipeline.lua b/tests/lua/pipeline.lua index 5686e19..bc37741 100755 --- a/tests/lua/pipeline.lua +++ b/tests/lua/pipeline.lua @@ -18,6 +18,22 @@ init = function(args) r[14] = wrk.format(nil, "/") r[15] = wrk.format(nil, "/") r[16] = wrk.format(nil, "/") + r[17] = wrk.format(nil, "/") + r[18] = wrk.format(nil, "/") + r[19] = wrk.format(nil, "/") + r[20] = wrk.format(nil, "/") + r[21] = wrk.format(nil, "/") + r[22] = wrk.format(nil, "/") + r[23] = wrk.format(nil, "/") + r[24] = wrk.format(nil, "/") + r[25] = wrk.format(nil, "/") + r[26] = wrk.format(nil, "/") + r[27] = wrk.format(nil, "/") + r[28] = wrk.format(nil, "/") + r[29] = wrk.format(nil, "/") + r[30] = wrk.format(nil, "/") + r[31] = wrk.format(nil, "/") + r[32] = wrk.format(nil, "/") req = table.concat(r) end diff --git a/tests/lua/q-form.lua b/tests/lua/q-form.lua new file mode 100755 index 0000000..a752f57 --- /dev/null +++ b/tests/lua/q-form.lua @@ -0,0 +1,31 @@ +-- example script demonstrating HTTP pipelining + +init = function(args) + local r = {} + wrk.headers["Content-Type"] = "application/x-www-form-urlencoded" + table.insert(r, wrk.format('POST','/form', nil, 'param1=value1¶m2=value2&c%C3%B3mo=puedes&fffffffffffffffffffffffffffffffffffff%20ffffffffffffffffff=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa')) + table.insert(r, wrk.format('POST','/form', nil, 'param1=value1¶m2=value2&c%C3%B3mo=puedes&fffffffffffffffffffffffffffffffffffff%20ffffffffffffffffff=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa')) + table.insert(r, wrk.format('POST','/form', nil, 'param1=value1¶m2=value2&c%C3%B3mo=puedes&fffffffffffffffffffffffffffffffffffff%20ffffffffffffffffff=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa')) + table.insert(r, wrk.format('POST','/form', nil, 'param1=value1¶m2=value2&c%C3%B3mo=puedes&fffffffffffffffffffffffffffffffffffff%20ffffffffffffffffff=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa')) + table.insert(r, wrk.format('POST','/form', nil, 'param1=value1¶m2=value2&c%C3%B3mo=puedes&fffffffffffffffffffffffffffffffffffff%20ffffffffffffffffff=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa')) + table.insert(r, wrk.format('POST','/form', nil, 'param1=value1¶m2=value2&c%C3%B3mo=puedes&fffffffffffffffffffffffffffffffffffff%20ffffffffffffffffff=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa')) + table.insert(r, wrk.format('POST','/form', nil, 'param1=value1¶m2=value2&c%C3%B3mo=puedes&fffffffffffffffffffffffffffffffffffff%20ffffffffffffffffff=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa')) + table.insert(r, wrk.format('POST','/form', nil, 'param1=value1¶m2=value2&c%C3%B3mo=puedes&fffffffffffffffffffffffffffffffffffff%20ffffffffffffffffff=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa')) + table.insert(r, wrk.format('POST','/form', nil, 'param1=value1¶m2=value2&c%C3%B3mo=puedes&fffffffffffffffffffffffffffffffffffff%20ffffffffffffffffff=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa')) + table.insert(r, wrk.format('POST','/form', nil, 'param1=value1¶m2=value2&c%C3%B3mo=puedes&fffffffffffffffffffffffffffffffffffff%20ffffffffffffffffff=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa')) + table.insert(r, wrk.format('POST','/form', nil, 'param1=value1¶m2=value2&c%C3%B3mo=puedes&fffffffffffffffffffffffffffffffffffff%20ffffffffffffffffff=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa')) + table.insert(r, wrk.format('POST','/form', nil, 'param1=value1¶m2=value2&c%C3%B3mo=puedes&fffffffffffffffffffffffffffffffffffff%20ffffffffffffffffff=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa')) + table.insert(r, wrk.format('POST','/form', nil, 'param1=value1¶m2=value2&c%C3%B3mo=puedes&fffffffffffffffffffffffffffffffffffff%20ffffffffffffffffff=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa')) + table.insert(r, wrk.format('POST','/form', nil, 'param1=value1¶m2=value2&c%C3%B3mo=puedes&fffffffffffffffffffffffffffffffffffff%20ffffffffffffffffff=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa')) + table.insert(r, wrk.format('POST','/form', nil, 'param1=value1¶m2=value2&c%C3%B3mo=puedes&fffffffffffffffffffffffffffffffffffff%20ffffffffffffffffff=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa')) + table.insert(r, wrk.format('POST','/form', nil, 'param1=value1¶m2=value2&c%C3%B3mo=puedes&fffffffffffffffffffffffffffffffffffff%20ffffffffffffffffff=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa')) + table.insert(r, wrk.format('POST','/form', nil, 'param1=value1¶m2=value2&c%C3%B3mo=puedes&fffffffffffffffffffffffffffffffffffff%20ffffffffffffffffff=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa')) + table.insert(r, wrk.format('POST','/form', nil, 'param1=value1¶m2=value2&c%C3%B3mo=puedes&fffffffffffffffffffffffffffffffffffff%20ffffffffffffffffff=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa')) + table.insert(r, wrk.format('POST','/form', nil, 'param1=value1¶m2=value2&c%C3%B3mo=puedes&fffffffffffffffffffffffffffffffffffff%20ffffffffffffffffff=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa')) + table.insert(r, wrk.format('POST','/form', nil, 'param1=value1¶m2=value2&c%C3%B3mo=puedes&fffffffffffffffffffffffffffffffffffff%20ffffffffffffffffff=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa')) + req = table.concat(r) +end + +request = function() + return req +end diff --git a/tests/lua/q-form2.lua b/tests/lua/q-form2.lua new file mode 100755 index 0000000..49d7e1e --- /dev/null +++ b/tests/lua/q-form2.lua @@ -0,0 +1,26 @@ +-- example script demonstrating HTTP pipelining + +init = function(args) + local r = {} + wrk.headers["Content-Type"] = "application/x-www-form-urlencoded" + r[1] = wrk.format('POST','/form', {"Content-Type", "application/x-www-form-urlencoded"}, 'param1=value1¶m2=value2&c%C3%B3mo=puedes&fffffffffffffffffffffffffffffffffffff%20ffffffffffffffffff=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa') + r[2] = wrk.format('POST','/form', {"Content-Type", "application/x-www-form-urlencoded"}, 'param1=value1¶m2=value2&c%C3%B3mo=puedes&fffffffffffffffffffffffffffffffffffff%20ffffffffffffffffff=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa') + r[3] = wrk.format('POST','/form', {"Content-Type", "application/x-www-form-urlencoded"}, 'param1=value1¶m2=value2&c%C3%B3mo=puedes&fffffffffffffffffffffffffffffffffffff%20ffffffffffffffffff=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa') + r[4] = wrk.format('POST','/form', {"Content-Type", "application/x-www-form-urlencoded"}, 'param1=value1¶m2=value2&c%C3%B3mo=puedes&fffffffffffffffffffffffffffffffffffff%20ffffffffffffffffff=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa') + r[5] = wrk.format('POST','/form', {"Content-Type", "application/x-www-form-urlencoded"}, 'param1=value1¶m2=value2&c%C3%B3mo=puedes&fffffffffffffffffffffffffffffffffffff%20ffffffffffffffffff=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa') + r[6] = wrk.format('POST','/form', {"Content-Type", "application/x-www-form-urlencoded"}, 'param1=value1¶m2=value2&c%C3%B3mo=puedes&fffffffffffffffffffffffffffffffffffff%20ffffffffffffffffff=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa') + r[7] = wrk.format('POST','/form', {"Content-Type", "application/x-www-form-urlencoded"}, 'param1=value1¶m2=value2&c%C3%B3mo=puedes&fffffffffffffffffffffffffffffffffffff%20ffffffffffffffffff=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa') + r[8] = wrk.format('POST','/form', {"Content-Type", "application/x-www-form-urlencoded"}, 'param1=value1¶m2=value2&c%C3%B3mo=puedes&fffffffffffffffffffffffffffffffffffff%20ffffffffffffffffff=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa') + r[9] = wrk.format('POST','/form', {"Content-Type", "application/x-www-form-urlencoded"}, 'param1=value1¶m2=value2&c%C3%B3mo=puedes&fffffffffffffffffffffffffffffffffffff%20ffffffffffffffffff=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa') + r[10] = wrk.format('POST','/form', {"Content-Type", "application/x-www-form-urlencoded"}, 'param1=value1¶m2=value2&c%C3%B3mo=puedes&fffffffffffffffffffffffffffffffffffff%20ffffffffffffffffff=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa') + r[11] = wrk.format('POST','/form', {"Content-Type", "application/x-www-form-urlencoded"}, 'param1=value1¶m2=value2&c%C3%B3mo=puedes&fffffffffffffffffffffffffffffffffffff%20ffffffffffffffffff=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa') + r[12] = wrk.format('POST','/form', {"Content-Type", "application/x-www-form-urlencoded"}, 'param1=value1¶m2=value2&c%C3%B3mo=puedes&fffffffffffffffffffffffffffffffffffff%20ffffffffffffffffff=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa') + r[13] = wrk.format('POST','/form', {"Content-Type", "application/x-www-form-urlencoded"}, 'param1=value1¶m2=value2&c%C3%B3mo=puedes&fffffffffffffffffffffffffffffffffffff%20ffffffffffffffffff=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa') + r[14] = wrk.format('POST','/form', {"Content-Type", "application/x-www-form-urlencoded"}, 'param1=value1¶m2=value2&c%C3%B3mo=puedes&fffffffffffffffffffffffffffffffffffff%20ffffffffffffffffff=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa') + r[15] = wrk.format('POST','/form', {"Content-Type", "application/x-www-form-urlencoded"}, 'param1=value1¶m2=value2&c%C3%B3mo=puedes&fffffffffffffffffffffffffffffffffffff%20ffffffffffffffffff=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa') + req = table.concat(r) +end + +request = function() + return req +end diff --git a/tests/lua/q-json.lua b/tests/lua/q-json.lua new file mode 100755 index 0000000..ee93362 --- /dev/null +++ b/tests/lua/q-json.lua @@ -0,0 +1,32 @@ +-- example script demonstrating HTTP pipelining + +init = function(args) + local r = {} + wrk.headers["Content-Type"] = "application/json; charset=utf-8" + table.insert(r, wrk.format('POST','/json', {"Content-Type", "application/json"}, '{"name":"json"}')) + table.insert(r, wrk.format('POST','/json', {"Content-Type", "application/json"}, '{"name":"json"}')) + table.insert(r, wrk.format('POST','/json', {"Content-Type", "application/json"}, '{"name":"json"}')) + table.insert(r, wrk.format('POST','/json', {"Content-Type", "application/json"}, '{"name":"json"}')) + table.insert(r, wrk.format('POST','/json', {"Content-Type", "application/json"}, '{"name":"json"}')) + table.insert(r, wrk.format('POST','/json', {"Content-Type", "application/json"}, '{"name":"json"}')) + table.insert(r, wrk.format('POST','/json', {"Content-Type", "application/json"}, '{"name":"json"}')) + table.insert(r, wrk.format('POST','/json', {"Content-Type", "application/json"}, '{"name":"json"}')) + table.insert(r, wrk.format('POST','/json', {"Content-Type", "application/json"}, '{"name":"json"}')) + table.insert(r, wrk.format('POST','/json', {"Content-Type", "application/json"}, '{"name":"json"}')) + table.insert(r, wrk.format('POST','/json', {"Content-Type", "application/json"}, '{"name":"json"}')) + table.insert(r, wrk.format('POST','/json', {"Content-Type", "application/json"}, '{"name":"json"}')) + table.insert(r, wrk.format('POST','/json', {"Content-Type", "application/json"}, '{"name":"json"}')) + table.insert(r, wrk.format('POST','/json', {"Content-Type", "application/json"}, '{"name":"json"}')) + table.insert(r, wrk.format('POST','/json', {"Content-Type", "application/json"}, '{"name":"json"}')) + table.insert(r, wrk.format('POST','/json', {"Content-Type", "application/json"}, '{"name":"json"}')) + table.insert(r, wrk.format('POST','/json', {"Content-Type", "application/json"}, '{"name":"json"}')) + table.insert(r, wrk.format('POST','/json', {"Content-Type", "application/json"}, '{"name":"json"}')) + table.insert(r, wrk.format('POST','/json', {"Content-Type", "application/json"}, '{"name":"json"}')) + table.insert(r, wrk.format('POST','/json', {"Content-Type", "application/json"}, '{"name":"json"}')) + table.insert(r, wrk.format('POST','/json', {"Content-Type", "application/json"}, '{"name":"json"}')) + req = table.concat(r) +end + +request = function() + return req +end diff --git a/tests/lua/q-mrp.lua b/tests/lua/q-mrp.lua new file mode 100755 index 0000000..f60a979 --- /dev/null +++ b/tests/lua/q-mrp.lua @@ -0,0 +1,28 @@ +-- example script demonstrating HTTP pipelining + +init = function(args) + local r = {} + wrk.headers["Content-Type"] = "application/mrpacker" + wrk.headers["Cookie"] = "mrsession=43709dd361cc443e976b05714581a7fb" + table.insert(r, wrk.format('POST','/mrq/1', nil, string.char(0x44,0xc1,0xc2,0xc3,0xc4))) + table.insert(r, wrk.format('POST','/mrq/1', nil, string.char(0x44,0xc1,0xc2,0xc3,0xc4))) + table.insert(r, wrk.format('POST','/mrq/1', nil, string.char(0x44,0xc1,0xc2,0xc3,0xc4))) + table.insert(r, wrk.format('POST','/mrq/1', nil, string.char(0x44,0xc1,0xc2,0xc3,0xc4))) + table.insert(r, wrk.format('POST','/mrq/1', nil, string.char(0x44,0xc1,0xc2,0xc3,0xc4))) + table.insert(r, wrk.format('POST','/mrq/1', nil, string.char(0x44,0xc1,0xc2,0xc3,0xc4))) + table.insert(r, wrk.format('POST','/mrq/1', nil, string.char(0x44,0xc1,0xc2,0xc3,0xc4))) + table.insert(r, wrk.format('POST','/mrq/1', nil, string.char(0x44,0xc1,0xc2,0xc3,0xc4))) + table.insert(r, wrk.format('POST','/mrq/1', nil, string.char(0x44,0xc1,0xc2,0xc3,0xc4))) + table.insert(r, wrk.format('POST','/mrq/1', nil, string.char(0x44,0xc1,0xc2,0xc3,0xc4))) + table.insert(r, wrk.format('POST','/mrq/1', nil, string.char(0x44,0xc1,0xc2,0xc3,0xc4))) + table.insert(r, wrk.format('POST','/mrq/1', nil, string.char(0x44,0xc1,0xc2,0xc3,0xc4))) + table.insert(r, wrk.format('POST','/mrq/1', nil, string.char(0x44,0xc1,0xc2,0xc3,0xc4))) + table.insert(r, wrk.format('POST','/mrq/1', nil, string.char(0x44,0xc1,0xc2,0xc3,0xc4))) + table.insert(r, wrk.format('POST','/mrq/1', nil, string.char(0x44,0xc1,0xc2,0xc3,0xc4))) + req = table.concat(r) +end + +request = function() + return req +end + diff --git a/tests/lua/q-session.lua b/tests/lua/q-session.lua new file mode 100755 index 0000000..2a6133f --- /dev/null +++ b/tests/lua/q-session.lua @@ -0,0 +1,37 @@ +-- example script demonstrating HTTP pipelining + +init = function(args) + local r = {} + wrk.headers["Cookie"] = "mrsession=43709dd361cc443e976b05714581a7fb" + table.insert(r, wrk.format(nil, "/s")) + table.insert(r, wrk.format(nil, "/s")) + table.insert(r, wrk.format(nil, "/s")) + table.insert(r, wrk.format(nil, "/s")) + table.insert(r, wrk.format(nil, "/s")) + table.insert(r, wrk.format(nil, "/s")) + table.insert(r, wrk.format(nil, "/s")) + table.insert(r, wrk.format(nil, "/s")) + table.insert(r, wrk.format(nil, "/s")) + table.insert(r, wrk.format(nil, "/s")) + table.insert(r, wrk.format(nil, "/s")) + table.insert(r, wrk.format(nil, "/s")) + table.insert(r, wrk.format(nil, "/s")) + table.insert(r, wrk.format(nil, "/s")) + table.insert(r, wrk.format(nil, "/s")) + table.insert(r, wrk.format(nil, "/s")) + table.insert(r, wrk.format(nil, "/s")) + table.insert(r, wrk.format(nil, "/s")) + table.insert(r, wrk.format(nil, "/s")) + table.insert(r, wrk.format(nil, "/s")) + table.insert(r, wrk.format(nil, "/s")) + table.insert(r, wrk.format(nil, "/s")) + table.insert(r, wrk.format(nil, "/s")) + table.insert(r, wrk.format(nil, "/s")) + table.insert(r, wrk.format(nil, "/s")) + table.insert(r, wrk.format(nil, "/s")) + req = table.concat(r) +end + +request = function() + return req +end diff --git a/tests/lua/q-upload.lua b/tests/lua/q-upload.lua new file mode 100755 index 0000000..61a839c --- /dev/null +++ b/tests/lua/q-upload.lua @@ -0,0 +1,49 @@ +-- File upload pipelined + +function read_txt_file(path) + local file, errorMessage = io.open(path, "r") + if not file then + error("Could not read the file:" .. errorMessage .. "\n") + end + + local content = file:read "*all" + file:close() + return content +end + +init = function(args) + local r = {} + local Boundary = "----WebKitFormBoundaryePkpFF7tjBAqx29L" + local BodyBoundary = "--" .. Boundary + local LastBoundary = "--" .. Boundary .. "--" + + local CRLF = "\r\n" + + local FileBody = read_txt_file("tests/lua/test.txt") + + local Filename = "test.txt" + + local ContentDisposition = "Content-Disposition: form-data; name=\"file\"; filename=\"" .. Filename .. "\"" + + wrk.method = "POST" + wrk.headers["Content-Type"] = "multipart/form-data; boundary=" .. Boundary + --wrk.headers["Cookie"] = "mrsession=43709dd361cc443e976b05714581a7fb" + local body = BodyBoundary .. CRLF .. ContentDisposition .. CRLF .. CRLF .. FileBody .. CRLF .. LastBoundary .. CRLF + + table.insert(r, wrk.format(nil, "/", nil, body)) + table.insert(r, wrk.format(nil, "/", nil, body)) + table.insert(r, wrk.format(nil, "/", nil, body)) + table.insert(r, wrk.format(nil, "/", nil, body)) + table.insert(r, wrk.format(nil, "/", nil, body)) + table.insert(r, wrk.format(nil, "/", nil, body)) + table.insert(r, wrk.format(nil, "/", nil, body)) + table.insert(r, wrk.format(nil, "/", nil, body)) + table.insert(r, wrk.format(nil, "/", nil, body)) + req = table.concat(r) +end + +request = function() + return req +end + + diff --git a/tests/lua/test.txt b/tests/lua/test.txt index 802f69c..572486c 100644 --- a/tests/lua/test.txt +++ b/tests/lua/test.txt @@ -1,3 +1,30 @@ Line one Line two Line three +Line threeaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +Line threeaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +Line three +Line three +Line threeaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +Line three +Line three +Line three +Line three +Line three +Line three +Line three +Line three +Line three +Line three +Line three +Line three +Line three +Line three +Line three +Line threeaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +Line threeaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +Line three +Line three +Line three +Line three +Line three diff --git a/tests/lua/upload.lua b/tests/lua/upload.lua index 354ddfa..eff8f93 100644 --- a/tests/lua/upload.lua +++ b/tests/lua/upload.lua @@ -18,12 +18,6 @@ local CRLF = "\r\n" local FileBody = read_txt_file("tests/lua/test.txt") --- We don't need different file names here because the test should --- always replace the uploaded file with the new one. This will avoid --- the problem with directories having too much files and slowing down --- the application, which is not what we are trying to test here. --- This will also avoid overloading wrk with more things do to, which --- can influence the test results. local Filename = "test.txt" local ContentDisposition = "Content-Disposition: form-data; name=\"file\"; filename=\"" .. Filename .. "\"" diff --git a/tests/readme b/tests/readme index 5ae9e36..271d3c3 100644 --- a/tests/readme +++ b/tests/readme @@ -1,6 +1,7 @@ Start this up to run the tests -- have in automatically done? -memcached -l 127.0.0.1 -p 11211 -d -m 50 + mrcache -m 64 -i 16 + python load_session.py will load the test user session set methods on pages test not returning a string from a page ( regular and coro ) diff --git a/tests/s1.py b/tests/s1.py index d379a0c..268f9b3 100644 --- a/tests/s1.py +++ b/tests/s1.py @@ -102,7 +102,6 @@ def mrp(r): @app.route('/form') def parseForm(r): - print(r) if r.form == None: return "No form" return json.dumps(r.form) @@ -114,7 +113,7 @@ def parseFiles(r): @app.route('/s',options=['session']) def session(r): if r.user: - return r.user["user"] + return "session" return "session" @app.route('/noreturn') diff --git a/tests/s_bench.py b/tests/s_bench.py index 1e3b969..63d1fbf 100755 --- a/tests/s_bench.py +++ b/tests/s_bench.py @@ -80,7 +80,7 @@ def testing(r): @app.route('/404/') def notFound(r): - return r.NotFound() + return app.err404 @app.route('/500/') def error500(r): @@ -110,7 +110,9 @@ def content(r): @app.route('/form') def parseForm(r): - return r.form["param2"] + if r.form: + return r.form["param2"] + return "No form" @app.route('/json') def parseJ(r): @@ -168,6 +170,17 @@ def t2(r): def longresp(r): return "fart"*128*1000 +@app.route('/upload') +def upload(r): + if r.file == None: + return "No file uploaded" + #for f in r.files: + #print(f) + name = r.file['name'] + typ = r.file['type'] + body = r.file['body'] + return name -app.run(cores=1) + +app.run(cores=4) diff --git a/tests/test_requests.py b/tests/test_requests.py index 0a9cc1c..5eb80a9 100644 --- a/tests/test_requests.py +++ b/tests/test_requests.py @@ -43,12 +43,6 @@ def setup(): def test_one(): - data = {} - s = "lo(ng"*5000 - data["long"] = s - r = requests.post('http://localhost:8080/form',data) # TODO have this timeout quickly - eq(r.status_code, 200) - eq(r.text, '{"long":"' + s + '"}') r = requests.get('http://localhost:8080/foo') eq(r.status_code, 200) @@ -107,7 +101,7 @@ def test_one(): eq(r.text, '{"key": "value"}') r = requests.post('http://localhost:8080/json', json={"name": "value"}) eq(r.text, 'value') - headers = {'Content-type': 'application/mrpacker'} + headers = {'Content-Type': 'application/mrpacker'} o = { "typ":"post", "s":2, "t": 'Blonde: "What does IDK stand for?"', "l":"localhost/sub/3", "txt": 'Brunette: "I don’t know."\nBlonde: "OMG, nobody does!"' } r = requests.post('http://localhost:8080/mrp', data=mrpacker.pack(o), headers=headers) if eq(r.text, 'post') != 0: @@ -132,8 +126,8 @@ def test_one(): eq(r.text, '{"":"v","pa{}ram2":"val(ue2"}') r = requests.post('http://localhost:8080/form', data={"":"v","英文版本":"val(ue2"}) eq(r.text, '{"":"v","英文版本":"val(ue2"}') - r = requests.post('http://localhost:8080/form', data={"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa":"","英":"+=&ue2"}) - eq(r.text, '{"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa":"","英":"+=&ue2"}') + r = requests.post('http://localhost:8080/form', data={"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa&":"","英":"+=&ue2"}) + eq(r.text, '{"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa&":"","英":"+=&ue2"}') #data = {} #s = "lo(ng"*10000 #data["long"] = s @@ -142,6 +136,12 @@ def test_one(): #eq(r.text, '{"long":"' + s + '"}') r = requests.get('http://localhost:8080/form') eq(r.text, "No form") + data = {} + s = "lo(ng"*5000 + data["long"] = s + r = requests.post('http://localhost:8080/form',data) + eq(r.status_code, 200) + eq(r.text, '{"long":"' + s + '"}') # Sessions cookie = {'mrsession': '43709dd361cc443e976b05714581a7fb'} @@ -151,6 +151,15 @@ def test_one(): # Misc r = requests.get('http://localhost:8080/printIP') eq(r.text, "None") + headers = {'CF-Connecting-IP': '123'} + r = requests.get('http://localhost:8080/printIP', headers=headers) + eq(r.text, "123") + headers = {'X-Real-IP': '1234'} + r = requests.get('http://localhost:8080/printIP', headers=headers) + eq(r.text, "1234") + headers = {'X-Forwarded-For': '12'} + r = requests.get('http://localhost:8080/printIP', headers=headers) + eq(r.text, "12") # TODO we can't test bad headers as requests won't send them curl localhost:8080/ -H "ƒtest:ƒart" diff --git a/tests/tst.py b/tests/tst.py index ab51ce4..db17da6 100644 --- a/tests/tst.py +++ b/tests/tst.py @@ -4,38 +4,10 @@ from common import eq,contains,stop_server import mrpacker -if 0: - data = {} - s = "lo(ng"*10000 - data["long"] = s - r = requests.post('http://localhost:8080/form',data) - eq(r.status_code, 200) - if 1: - headers = {'Content-type': 'application/mrpacker'} - o = { "typ":"post", "s":4, "t": 'Blonde: "What does IDK stand for?"', "l":"", "txt": 'Brunette: "I don’t know."\nBlonde: "OMG, nobody does!"' } - o = { "typ": "post", "l": "", "t": "Blonde: \"What does IDK stand for?\"", "txt": "Brunette: \"I don’t know.\"\n\nBlonde: \"OMG, nobody does!\"", "s": 3} - print(len(o["txt"])) - b = mrpacker.pack(o) - s = "" - for c in b: - #s = s + str(hex(int(c))) + ", " - s = s + str(int(c)) + ", " - print(s) - print( mrpacker.unpack(b) ) - #r = requests.post('http://localhost:8080/mrp', data=mrpacker.pack(o), headers=headers) - #eq(r.text, 'post') + r = requests.post('http://localhost:8765', data={"p1":"v1","param2":"value2"}) + #r = requests.post('http://localhost:8765', files={"p1":"v1","param2":"value2"}) + #r = requests.post('http://localhost:8765', data={"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa":"","英":"+=&ue2"}) -l = [66, 114, 117, 110, 101, 116, 116, 101, 58, 32, 34, 73, 32, 100, 111, 110, 226, 128, 153, 116, 32, 107, 110, 111, 119, 46, 34, 10, 10, 66, 108, 111, 110, 100, 101, 58, 32, 34, 79, 77, 71, 44, 32, 110, 111, 98, 111, 100, 121, 32, 100, 111, 101, 115, 33, 34] -print(len(l)) -s = "" -#for c in l: - #s += chr(c) -#print(s) + #eq(r.text, '{"p1":"v1","param2":"value2"}') -z = """ -""" -54 -37, 131, 116, 121, 112, 132, 112, 111, 115, 116, 129, 108, 128, 129, 116, 102, 34, 0, 0, 0, 66, 108, 111, 110, 100, 101, 58, 32, 34, 87, 104, 97, 116, 32, 100, 111, 101, 115, 32, 73, 68, 75, 32, 115, 116, 97, 110, 100, 32, 102, 111, 114, 63, 34, 131, 116, 120, 116, 102, 56, 0, 0, 0, 66, 114, 117, 110, 101, 116, 116, 101, 58, 32, 34, 73, 32, 100, 111, 110, 226, 128, 153, 116, 32, 107, 110, 111, 119, 46, 34, 10, 10, 66, 108, 111, 110, 100, 101, 58, 32, 34, 79, 77, 71, 44, 32, 110, 111, 98, 111, 100, 121, 32, 100, 111, 101, 115, 33, 34, 129, 115, 195, -{'typ': 'post', 'l': '', 't': 'Blonde: "What does IDK stand for?"', 'txt': 'Brunette: "I don’t know."\n\nBlonde: "OMG, nobody does!"', 's': 3} -56 diff --git a/todo b/todo index f4bc67b..c9f6ef5 100644 --- a/todo +++ b/todo @@ -1,4 +1,7 @@ +Handle static files - static root route and path. +Connection buffer size - after a file upload shrink the buffer back down. Behind cloudflare connections will remain open forever. Behind nginx its just one connection so who cares. Out front client connections will close so it doesn't matter? + support HEAD diff --git a/tst.py b/tst.py index b260293..6c2cb56 100755 --- a/tst.py +++ b/tst.py @@ -1,35 +1,38 @@ -import traceback +import traceback, mrjson from mrhttp import app import mrpacker app.config["memcache"] = [ ("127.0.0.1", 11211) ] +#app.static_cached("www","/path/to/www") #@app.on('at_start') #async def setup(): #app.c = asyncmrq.Client() #await app.c.connect(servers=[("127.0.0.1",7100)]) -#@app.route('/',options=['session']) -@app.route('/',_type="html") +#@app.route('/',options=['cache']) +@app.route('/') def index(r): - print( r.headers ) - #print( r.ip ) - return "yay" - #d = r.mrpack - #return d["name"] - #x = r.form - #return x["param2"] + return "hello world" + +@app.route('/123456789123456789') +async def long(r): + return "long" @app.route('/json') def json(r): - return r.json["name"] + return mrjson.dumps({'message': 'Hello, world!'}) @app.route('/mrp',_type="mrp") def mrp(r): return mrpacker.pack("hello") +@app.route('/{}/tst') +def firstarg(r,a): + return a + try: app.run(cores=1) diff --git a/workserver.py b/workserver.py new file mode 100644 index 0000000..61c8566 --- /dev/null +++ b/workserver.py @@ -0,0 +1,12 @@ +import asyncio +import mrworkserver + +async def callback(ws, msgs): + for m in msgs: + print(m) + pass + +ws = mrworkserver.WorkServer(callback=callback) + +ws.run(host="127.0.0.1",port=7100) +