diff --git a/README.textile b/README.textile
index 8749c01..f461318 100644
--- a/README.textile
+++ b/README.textile
@@ -9,6 +9,7 @@ h2. Dependencies
* Python 2.6
* boto
* paramiko
+* csvkit
h2. Installation for users
@@ -58,6 +59,28 @@ It then uses those 4 servers to send 10,000 requests, 250 at a time, to attack O
Lastly, it spins down the 4 servers. *Please remember to do this*--we aren't responsible for your EC2 bills.
+h2. Advanced Usage
+
+
+bees up -s 5 -g public -k frakkingtoasters -z us-west-1a -i ami-aabbccdd -l ubuntu
+for i in `seq 200 200 1000`
+do
+ echo "---- $i -----"
+ bees attack -t 900 -c $i -p query.dat -u http://www.ournewwebbyhotness.com/ --stats-file 15_Min_200_step.csv --non-200-is-failure --testname $i
+done
+bees down
+gnuplot -e "filename='15_Min_200_step'" examples/LoadTest.gpi
+
+
+This spins up 5 servers in the us-west-1a AZ from the specified AMI in security group 'public' using the EC2 keypair 'frakkingtoasters', whose private key is expected to reside at ~/.ssh/frakkingtoasters.pem.
+
+It then runs a series of 15 minute tests (which in this case are a post of some query.dat file) with an increasing number of concurrent users, all the while collecting all the resulting statistics in the 15_Min_200_step.csv file. Note that in this test non 200 responses are considered as errors.
+
+Next, the bees are spun down.
+
+Finally, a graph is created from the csv using gnuplot (an example gnuplot script can be found in the examples dir).
+
+
For complete options type:
diff --git a/beeswithmachineguns/bees.py b/beeswithmachineguns/bees.py
old mode 100644
new mode 100755
index 8141f6b..18698f9
--- a/beeswithmachineguns/bees.py
+++ b/beeswithmachineguns/bees.py
@@ -25,6 +25,10 @@
"""
from multiprocessing import Pool
+from subprocess import check_output, call, CalledProcessError
+from collections import OrderedDict
+from tempfile import NamedTemporaryFile
+#from uuid import uuid4
import os
import re
import socket
@@ -79,7 +83,7 @@ def _get_security_group_ids(connection, security_group_names, subnet):
ids = []
# Since we cannot get security groups in a vpc by name, we get all security groups and parse them by name later
security_groups = connection.get_all_security_groups()
-
+
# Parse the name of each security group and add the id of any match to the group list
for group in security_groups:
for name in security_group_names:
@@ -87,10 +91,17 @@ def _get_security_group_ids(connection, security_group_names, subnet):
if subnet == None:
if group.vpc_id == None:
ids.append(group.id)
- elif group.vpc_id != None:
- ids.append(group.id)
-
- return ids
+ elif group.vpc_id != None:
+ ids.append(group.id)
+ if not ids:
+ print "Couldn't find security group, probably because you have a default vpc, looking for vpc security groups"
+ for group in security_groups:
+ for name in security_group_names:
+ if group.name == name:
+ ids.append(group.id)
+ if not ids:
+ print "Warning: Couldn't find security group, using default!!!"
+ return ids
# Methods
@@ -205,19 +216,26 @@ def _attack(params):
try:
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+
+ if params['gnuplot_filename']:
+ use_compression = True
+ else:
+ use_compression = False
+
client.connect(
params['instance_name'],
username=params['username'],
- key_filename=_get_pem_path(params['key_name']))
+ key_filename=_get_pem_path(params['key_name']),
+ compress=use_compression)
- print 'Bee %i is firing her machine gun. Bang bang!' % params['i']
+ print 'Bee %i is firing her machine gun (post file: %s) at (%s). Bang bang!' % (params['i'], params['post_file'], params['url'])
options = ''
if params['headers'] is not '':
for h in params['headers'].split(';'):
options += ' -H "%s"' % h
- stdin, stdout, stderr = client.exec_command('tempfile -s .csv')
+ stdin, stdout, stderr = client.exec_command('mktemp --suffix=.csv')
params['csv_filename'] = stdout.read().strip()
if params['csv_filename']:
options += ' -e %(csv_filename)s' % params
@@ -225,32 +243,93 @@ def _attack(params):
print 'Bee %i lost sight of the target (connection timed out creating csv_filename).' % params['i']
return None
+ if params['gnuplot_filename']:
+ stdin, stdout, stderr = client.exec_command('mktemp --suffix=.tsv')
+ params['tsv_filename'] = stdout.read().strip()
+ if params['tsv_filename']:
+ options += ' -g %(tsv_filename)s' % params
+ else:
+ print 'Bee %i lost sight of the target (connection timed out creating tsv_filename).' % params['i']
+ return None
+
if params['post_file']:
pem_file_path=_get_pem_path(params['key_name'])
os.system("scp -q -o 'StrictHostKeyChecking=no' -i %s %s %s@%s:/tmp/honeycomb" % (pem_file_path, params['post_file'], params['username'], params['instance_name']))
- options += ' -k -T "%(mime_type)s; charset=UTF-8" -p /tmp/honeycomb' % params
+ options += ' -T "%(mime_type)s; charset=UTF-8" -p /tmp/honeycomb' % params
+ #random_command = "sed -i 's/RANDOM/%s/' /tmp/honeycomb && cat /tmp/honeycomb" % str(uuid4())
+ #stdin, stdout, stderr = client.exec_command(random_command)
+ #print 'posting file: %s' % stdout.read()
+ #options += ' -k -T "%(mime_type)s; charset=UTF-8" -p /tmp/honeycomb' % params
params['options'] = options
- benchmark_command = 'ab -r -n %(num_requests)s -c %(concurrent_requests)s -C "sessionid=NotARealSessionID" %(options)s "%(url)s"' % params
+ if params['timelimit'] > 0:
+ benchmark_command = 'ab -l -r -s 3 -t %(timelimit)s -n 5000000 -c %(concurrent_requests)s -C "sessionid=NotARealSessionID" %(options)s "%(url)s"' % params
+ #benchmark_command = './ab -l 1000 -r -t %(timelimit)s -n 5000000 -c %(concurrent_requests)s -C "sessionid=NotARealSessionID" %(options)s "%(url)s"' % params
+ #benchmark_command = 'ab -r -t %(timelimit)s -n 5000000 -c %(concurrent_requests)s -C "sessionid=NotARealSessionID" %(options)s "%(url)s"' % params
+ else:
+ benchmark_command = './ab -l -r -s 3 -n %(num_requests)s -c %(concurrent_requests)s -C "sessionid=NotARealSessionID" %(options)s "%(url)s"' % params
+ #benchmark_command = './ab -l 1000 -r -n %(num_requests)s -c %(concurrent_requests)s -C "sessionid=NotARealSessionID" %(options)s "%(url)s"' % params
+ #benchmark_command = 'ab -r -n %(num_requests)s -c %(concurrent_requests)s -C "sessionid=NotARealSessionID" %(options)s "%(url)s"' % params
stdin, stdout, stderr = client.exec_command(benchmark_command)
response = {}
ab_results = stdout.read()
+ ab_error = stderr.read()
ms_per_request_search = re.search('Time\ per\ request:\s+([0-9.]+)\ \[ms\]\ \(mean\)', ab_results)
if not ms_per_request_search:
- print 'Bee %i lost sight of the target (connection timed out running ab).' % params['i']
+ #print 'Bee %i lost sight of the target (connection timed out running ab).' % params['i']
+ print 'Bee %i lost sight of the target (connection timed out running ab). ab command: [%s] \nresult: [%s]\nerror:[%s].' % (params['i'], benchmark_command, ab_results, ab_error)
return None
requests_per_second_search = re.search('Requests\ per\ second:\s+([0-9.]+)\ \[#\/sec\]\ \(mean\)', ab_results)
failed_requests = re.search('Failed\ requests:\s+([0-9.]+)', ab_results)
complete_requests_search = re.search('Complete\ requests:\s+([0-9]+)', ab_results)
+ time_taken_search = re.search('Time\ taken\ for\ tests:\s+([0-9]+)', ab_results)
+ non_200_responses_search = re.search('Non\-2xx\ responses:\s+([0-9]+)', ab_results)
+
+ """
+ If there are failed requests, get the breakdown
+ (Connect: 0, Receive: 0, Length: 338, Exceptions: 0)
+ """
+ failed_connect_search = re.search('\s+\(Connect:\s+([0-9.]+)', ab_results)
+ failed_receive_search = re.search('\s+\(.+Receive:\s+([0-9.]+)', ab_results)
+ failed_length_search = re.search('\s+\(.+Length:\s+([0-9.]+)', ab_results)
+ failed_exceptions_search = re.search('\s+\(.+Exceptions:\s+([0-9.]+)', ab_results)
response['ms_per_request'] = float(ms_per_request_search.group(1))
response['requests_per_second'] = float(requests_per_second_search.group(1))
response['failed_requests'] = float(failed_requests.group(1))
response['complete_requests'] = float(complete_requests_search.group(1))
+ response['time_taken'] = float(time_taken_search.group(1))
+
+ if failed_connect_search is None:
+ response['failed_connect'] = 0
+ else:
+ response['failed_connect'] = float(failed_connect_search.group(1))
+
+ if failed_receive_search is None:
+ response['failed_receive'] = 0
+ else:
+ response['failed_receive'] = float(failed_receive_search.group(1))
+
+ if failed_length_search is None:
+ response['failed_length'] = 0
+ else:
+ response['failed_length'] = float(failed_length_search.group(1))
+
+ if failed_exceptions_search is None:
+ response['failed_exceptions'] = 0
+ else:
+ response['failed_exceptions'] = float(failed_exceptions_search.group(1))
+
+ if non_200_responses_search is None:
+ response['non_200_responses'] = 0
+ else:
+ response['non_200_responses'] = float(non_200_responses_search.group(1))
+
+ print 'Bee %i is out of ammo. She is collecting her pollen and flying back to the hive. This may take a while if she has a heavy load and/or the hive is far away...' % params['i']
stdin, stdout, stderr = client.exec_command('cat %(csv_filename)s' % params)
response['request_time_cdf'] = []
@@ -261,7 +340,12 @@ def _attack(params):
print 'Bee %i lost sight of the target (connection timed out reading csv).' % params['i']
return None
- print 'Bee %i is out of ammo.' % params['i']
+ if params['gnuplot_filename']:
+ f = NamedTemporaryFile(suffix=".tsv", delete=False)
+ response['tsv_filename'] = f.name
+ f.close()
+ sftp = client.open_sftp()
+ sftp.get(params['tsv_filename'], response['tsv_filename'])
client.close()
@@ -269,7 +353,7 @@ def _attack(params):
except socket.error, e:
return e
-def _print_results(results, params, csv_filename):
+def _print_results(results, params, csv_filename, gnuplot_filename, stats_filename, existing_stats_file, testname, non_200_is_failure):
"""
Print summarized load-testing results.
"""
@@ -295,20 +379,54 @@ def _print_results(results, params, csv_filename):
print ' No bees completed the mission. Apparently your bees are peace-loving hippies.'
return
- complete_results = [r['complete_requests'] for r in complete_bees]
- total_complete_requests = sum(complete_results)
+ complete_requests = [r['complete_requests'] for r in complete_bees]
+ total_complete_requests = sum(complete_requests)
print ' Complete requests:\t\t%i' % total_complete_requests
- complete_results = [r['failed_requests'] for r in complete_bees]
- total_failed_requests = sum(complete_results)
- print ' Failed requests:\t\t%i' % total_failed_requests
+ if non_200_is_failure:
+ failed_requests = [r['failed_requests']+r['non_200_responses'] for r in complete_bees]
+ else:
+ failed_requests = [r['failed_requests'] for r in complete_bees]
+
+ total_failed_requests = sum(failed_requests)
+ total_failed_percent = total_failed_requests/total_complete_requests
+ print ' Failed requests:\t\t{:,} ({:.2%})'.format(int(total_failed_requests), total_failed_percent)
+
+ non_200_results = [r['non_200_responses'] for r in complete_bees]
+ total_non_200_results = sum(non_200_results)
+
+ if total_failed_requests > 0:
+ failed_connect_requests = [r['failed_connect'] for r in complete_bees]
+ total_failed_connect_requests = sum(failed_connect_requests)
+ failed_receive_requests = [r['failed_receive'] for r in complete_bees]
+ total_failed_receive_requests = sum(failed_receive_requests)
+ failed_length_requests = [r['failed_length'] for r in complete_bees]
+ total_failed_length_requests = sum(failed_length_requests)
+ failed_exceptions_requests = [r['failed_exceptions'] for r in complete_bees]
+ total_failed_exception_requests = sum(failed_exceptions_requests)
+ if non_200_is_failure:
+ print ' (Connect: %i, Receive: %i, Length: %i, Exception: %i, Non-200: %i)' % \
+ (total_failed_connect_requests, total_failed_receive_requests, total_failed_length_requests, total_failed_exception_requests, total_non_200_results)
+ else:
+ print ' (Connect: %i, Receive: %i, Length: %i, Exception: %i)' % \
+ (total_failed_connect_requests, total_failed_receive_requests, total_failed_length_requests, total_failed_exception_requests)
+
+ if (not non_200_is_failure) and total_non_200_results > 0:
+ print ' Non-200 Responses:\t\t%i' % total_non_200_results
- complete_results = [r['requests_per_second'] for r in complete_bees]
- mean_requests = sum(complete_results)
+ requests_per_second = [r['requests_per_second'] for r in complete_bees]
+ mean_requests = sum(requests_per_second)
print ' Requests per second:\t%f [#/sec]' % mean_requests
- complete_results = [r['ms_per_request'] for r in complete_bees]
- mean_response = sum(complete_results) / num_complete_bees
+ if non_200_is_failure:
+ successful_requests_per_second = [(r['complete_requests']-r['failed_requests']-r['non_200_responses'])/r['time_taken'] for r in complete_bees]
+ else:
+ successful_requests_per_second = [(r['complete_requests']-r['failed_requests'])/r['time_taken'] for r in complete_bees]
+ successful_mean_requests = sum(successful_requests_per_second)
+ print ' Successful Requests per second:\t%f [#/sec]' % successful_mean_requests
+
+ ms_per_request = [r['ms_per_request'] for r in complete_bees]
+ mean_response = sum(ms_per_request) / num_complete_bees
print ' Time per request:\t\t%f [ms] (mean of bees)' % mean_response
# Recalculate the global cdf based on the csv files collected from
@@ -331,13 +449,13 @@ def _print_results(results, params, csv_filename):
print ' 50%% responses faster than:\t%f [ms]' % request_time_cdf[49]
print ' 90%% responses faster than:\t%f [ms]' % request_time_cdf[89]
- if mean_response < 500:
+ if mean_response < 500 and total_failed_percent < 0.001:
print 'Mission Assessment: Target crushed bee offensive.'
- elif mean_response < 1000:
+ elif mean_response < 1000 and total_failed_percent < 0.01:
print 'Mission Assessment: Target successfully fended off the swarm.'
- elif mean_response < 1500:
+ elif mean_response < 1500 and total_failed_percent < 0.05:
print 'Mission Assessment: Target wounded, but operational.'
- elif mean_response < 2000:
+ elif mean_response < 2000 and total_failed_percent < 0.10:
print 'Mission Assessment: Target severely compromised.'
else:
print 'Mission Assessment: Swarm annihilated target.'
@@ -354,14 +472,74 @@ def _print_results(results, params, csv_filename):
for r in results:
row.append(r['request_time_cdf'][i]["Time in ms"])
writer.writerow(row)
+
+ if gnuplot_filename:
+ print 'Joining gnuplot files from all bees.'
+ files = [r['tsv_filename'] for r in results if r is not None]
+ # using csvkit utils to join the tsv files from all of the bees, adding a column to show which bee produced each line. using sort because of performance problems with csvsort.
+ #command = "csvstack -t -n bee -g " + ",".join(["%(i)s" % p for p in complete_bees_params]) + " " + " ".join(files) + " | csvcut -c 2-7,1 | sort -nk 5 -t ',' | sed 's/,/\t/g' > " + gnuplot_filename
+ # csvkit took too long for joining files, using builtins instead
+ command = "head -1 " + files[0] + " > " + gnuplot_filename + " && cat " + " ".join(files) + " | grep -v starttime >> " + gnuplot_filename
+ call(command, shell=True)
+ # removing temp files
+ call(["rm"] + files)
+
+ if stats_filename:
+ print 'Calculating statistics.'
+ try:
+ csvstat_results = check_output(["csvstat", "-tc", "ttime", gnuplot_filename])
+ except CalledProcessError as e:
+ print 'Error running csvstat: %d output: [%s]' % (e.returncode, e.output)
+ csvstat_results = """
+ Dummy values:
+ Min: 0
+ Max: 0
+ Mean: 0
+ Median: 0
+ Standard Deviation: 0
+ """
+
+ min_search = re.search('\sMin:\s+([0-9]+)', csvstat_results)
+ max_search = re.search('\sMax:\s+([0-9]+)', csvstat_results)
+ mean_search = re.search('\sMean:\s+([0-9.]+)', csvstat_results)
+ median_search = re.search('\sMedian:\s+([0-9.]+)', csvstat_results)
+ stdev_search = re.search('\sStandard\ Deviation:\s+([0-9.]+)', csvstat_results)
+
+ stats = OrderedDict()
+ stats['Name'] = testname
+ stats['Total'] = int(total_complete_requests)
+ stats['Success'] = int(total_complete_requests-total_failed_requests)
+ stats['% Success'] = stats['Success']/total_complete_requests
+ stats['Error'] = int(total_failed_requests)
+ stats['% Error'] = total_failed_percent
+ stats['TotalPerSecond'] = mean_requests
+ stats['SuccessPerSecond'] = successful_mean_requests
+ stats['Min'] = int(min_search.group(1))
+ stats['Max'] = int(max_search.group(1))
+ stats['Mean'] = float(mean_search.group(1))
+ stats['Median'] = float(median_search.group(1))
+ stats['StdDev'] = float(stdev_search.group(1))
+ for i in range(5, 100, 5):
+ stats['P' + str(i)] = request_time_cdf[i]
+
+ with open(stats_filename, 'a') as stream:
+ writer = csv.DictWriter(stream, fieldnames=stats)
+ if not existing_stats_file:
+ writer.writeheader()
+ writer.writerow(stats)
-def attack(url, n, c, **options):
+def attack(urls, n, c, t, **options):
"""
Test the root url of this site.
"""
username, key_name, zone, instance_ids = _read_server_list()
headers = options.get('headers', '')
csv_filename = options.get("csv_filename", '')
+ gnuplot_filename = options.get("gnuplot_filename", '')
+ stats_filename = options.get("stats_filename", '')
+ existing_stats_file = False
+ testname = options.get("testname", '')
+ non_200_is_failure = options.get("non_200_is_failure", False)
if csv_filename:
try:
@@ -369,6 +547,21 @@ def attack(url, n, c, **options):
except IOError, e:
raise IOError("Specified csv_filename='%s' is not writable. Check permissions or specify a different filename and try again." % csv_filename)
+ if stats_filename:
+ existing_stats_file = os.path.isfile(stats_filename)
+ try:
+ stream = open(stats_filename, 'a')
+ except IOError, e:
+ raise IOError("Specified stats_filename='%s' is not writable. Check permissions or specify a different filename and try again." % stats_filename)
+ if not gnuplot_filename:
+ gnuplot_filename = os.path.splitext(stats_filename)[0] + "." + testname + ".tsv"
+
+ if gnuplot_filename:
+ try:
+ stream = open(gnuplot_filename, 'w')
+ except IOError, e:
+ raise IOError("Specified gnuplot_filename='%s' is not writable. Check permissions or specify a different filename and try again." % gnuplot_filename)
+
if not instance_ids:
print 'No bees are ready to attack.'
return
@@ -388,46 +581,70 @@ def attack(url, n, c, **options):
instance_count = len(instances)
- if n < instance_count * 2:
- print 'bees: error: the total number of requests must be at least %d (2x num. instances)' % (instance_count * 2)
- return
if c < instance_count:
- print 'bees: error: the number of concurrent requests must be at least %d (num. instances)' % instance_count
- return
- if n < c:
- print 'bees: error: the number of concurrent requests (%d) must be at most the same as number of requests (%d)' % (c, n)
+ instance_count = c
+ del instances[c:]
+ print 'bees: warning: the number of concurrent requests is lower than the number of bees, only %d of the bees will be used' % instance_count
+ connections_per_instance = int(float(c) / instance_count)
+ if instance_count < len(urls):
+ print "bees: error: the number of urls (%d) can't exceed the number of bees (%d)" % (len(urls), instance_count)
return
+ if instance_count % len(urls):
+ print "bees: warning: the load will not be evenly distributed between the urls because they can't be evenly divided between the bees [(%d bees) mod (%d urls) != 0]" % (instance_count, len(urls))
+ post_files = options.get('post_files')
+ if post_files:
+ if instance_count < len(post_files):
+ print "bees: error: the number of post_files (%d) can't exceed the number of bees (%d)" % (len(post_files), instance_count)
+ return
+ if instance_count % len(post_files):
+ print "bees: warning: the load will not be evenly distributed between the post_files because they can't be evenly divided between the bees [(%d bees) mod (%d post_files) != 0]" % (instance_count, len(post_files))
+ if t > 0:
+ print 'Each of %i bees will fire for %s seconds, %s at a time.' % (instance_count, t, connections_per_instance)
+ requests_per_instance = 50000;
+ else:
+ if n < instance_count * 2:
+ print 'bees: error: the total number of requests must be at least %d (2x num. instances)' % (instance_count * 2)
+ return
+ if n < c:
+ print 'bees: error: the number of concurrent requests (%d) must be at most the same as number of requests (%d)' % (c, n)
+ return
- requests_per_instance = int(float(n) / instance_count)
- connections_per_instance = int(float(c) / instance_count)
+ requests_per_instance = int(float(n) / instance_count)
- print 'Each of %i bees will fire %s rounds, %s at a time.' % (instance_count, requests_per_instance, connections_per_instance)
+ print 'Each of %i bees will fire %s rounds, %s at a time.' % (instance_count, requests_per_instance, connections_per_instance)
params = []
for i, instance in enumerate(instances):
+ post_file = False
+ if post_files:
+ post_file = post_files[len(post_files) - (i % len(post_files)) - 1] # reverse iteration so it won't coinside with the urls iteration
params.append({
'i': i,
'instance_id': instance.id,
'instance_name': instance.public_dns_name,
- 'url': url,
+ 'url': urls[i % len(urls)],
+ #'url': urls[i % len(urls)] + "?uuid=" + str(uuid4()),
'concurrent_requests': connections_per_instance,
'num_requests': requests_per_instance,
+ 'timelimit': t,
'username': username,
'key_name': key_name,
'headers': headers,
- 'post_file': options.get('post_file'),
+ 'post_file': post_file,
'mime_type': options.get('mime_type', ''),
+ 'gnuplot_filename': gnuplot_filename,
})
- print 'Stinging URL so it will be cached for the attack.'
+# print 'Stinging URLs so they will be cached for the attack.'
# Ping url so it will be cached for testing
- dict_headers = {}
- if headers is not '':
- dict_headers = headers = dict(h.split(':') for h in headers.split(';'))
- request = urllib2.Request(url, headers=dict_headers)
- urllib2.urlopen(request).read()
+# dict_headers = {}
+# if headers is not '':
+# dict_headers = headers = dict(h.split(':') for h in headers.split(';'))
+# for url in urls:
+# request = urllib2.Request(url, headers=dict_headers)
+# urllib2.urlopen(request).read()
print 'Organizing the swarm.'
@@ -437,6 +654,6 @@ def attack(url, n, c, **options):
print 'Offensive complete.'
- _print_results(results, params, csv_filename)
+ _print_results(results, params, csv_filename, gnuplot_filename, stats_filename, existing_stats_file, testname, non_200_is_failure)
print 'The swarm is awaiting new orders.'
diff --git a/beeswithmachineguns/main.py b/beeswithmachineguns/main.py
index 1b84154..db0931d 100644
--- a/beeswithmachineguns/main.py
+++ b/beeswithmachineguns/main.py
@@ -27,127 +27,94 @@
import bees
from urlparse import urlparse
-from optparse import OptionParser, OptionGroup
+from argparse import ArgumentParser
def parse_options():
"""
Handle the command line arguments for spinning up bees
"""
- parser = OptionParser(usage="""
-bees COMMAND [options]
+ parser = ArgumentParser(description="""
+ Bees with Machine Guns.
+ A utility for arming (creating) many bees (small EC2 instances) to attack
+ (load test) targets (web applications).
+ """)
-Bees with Machine Guns
+ subparsers = parser.add_subparsers(title='commands', dest='command')
+ up_cmd = subparsers.add_parser("up", help='Start a batch of load testing servers.', description=
+ """Start a batch of load testing servers.
+ In order to spin up new servers you will need to specify at least the -k command, which is the name of the EC2 keypair to use for creating and connecting to the new servers. The bees will expect to find a .pem file with this name in ~/.ssh/.""")
-A utility for arming (creating) many bees (small EC2 instances) to attack
-(load test) targets (web applications).
+ # Required
+ up_cmd.add_argument('-k', '--key', metavar="KEY", dest='key', required=True, help="The ssh key pair name to use to connect to the new servers.")
-commands:
- up Start a batch of load testing servers.
- attack Begin the attack on a specific url.
- down Shutdown and deactivate the load testing servers.
- report Report the status of the load testing servers.
- """)
+ up_cmd.add_argument('-s', '--servers', metavar="SERVERS", dest='servers', type=int, default=5, help="The number of servers to start (default: 5).")
+ up_cmd.add_argument('-g', '--group', metavar="GROUP", dest='group', default='default', help="The security group(s) to run the instances under (default: default).")
+ up_cmd.add_argument('-z', '--zone', metavar="ZONE", dest='zone', default='us-east-1d', help="The availability zone to start the instances in (default: us-east-1d).")
+ up_cmd.add_argument('-i', '--instance', metavar="INSTANCE", dest='instance', default='ami-ff17fb96', help="The instance-id to use for each server from (default: ami-ff17fb96).")
+ up_cmd.add_argument('-t', '--type', metavar="TYPE", dest='type', default='t1.micro', help="The instance-type to use for each server (default: t1.micro).")
+ up_cmd.add_argument('-l', '--login', metavar="LOGIN", dest='login', default='newsapps', help="The ssh username name to use to connect to the new servers (default: newsapps).")
+ up_cmd.add_argument('-v', '--subnet', metavar="SUBNET", dest='subnet', default=None, help="The vpc subnet id in which the instances should be launched. (default: None).")
- up_group = OptionGroup(parser, "up",
- """In order to spin up new servers you will need to specify at least the -k command, which is the name of the EC2 keypair to use for creating and connecting to the new servers. The bees will expect to find a .pem file with this name in ~/.ssh/.""")
+ attack_cmd = subparsers.add_parser("attack", help='Begin the attack on a specific url.', description=
+ """Begin the attack on a specific url.
+ Beginning an attack requires only that you specify the -u option with the URL you wish to target.""")
# Required
- up_group.add_option('-k', '--key', metavar="KEY", nargs=1,
- action='store', dest='key', type='string',
- help="The ssh key pair name to use to connect to the new servers.")
-
- up_group.add_option('-s', '--servers', metavar="SERVERS", nargs=1,
- action='store', dest='servers', type='int', default=5,
- help="The number of servers to start (default: 5).")
- up_group.add_option('-g', '--group', metavar="GROUP", nargs=1,
- action='store', dest='group', type='string', default='default',
- help="The security group(s) to run the instances under (default: default).")
- up_group.add_option('-z', '--zone', metavar="ZONE", nargs=1,
- action='store', dest='zone', type='string', default='us-east-1d',
- help="The availability zone to start the instances in (default: us-east-1d).")
- up_group.add_option('-i', '--instance', metavar="INSTANCE", nargs=1,
- action='store', dest='instance', type='string', default='ami-ff17fb96',
- help="The instance-id to use for each server from (default: ami-ff17fb96).")
- up_group.add_option('-t', '--type', metavar="TYPE", nargs=1,
- action='store', dest='type', type='string', default='t1.micro',
- help="The instance-type to use for each server (default: t1.micro).")
- up_group.add_option('-l', '--login', metavar="LOGIN", nargs=1,
- action='store', dest='login', type='string', default='newsapps',
- help="The ssh username name to use to connect to the new servers (default: newsapps).")
- up_group.add_option('-v', '--subnet', metavar="SUBNET", nargs=1,
- action='store', dest='subnet', type='string', default=None,
- help="The vpc subnet id in which the instances should be launched. (default: None).")
-
- parser.add_option_group(up_group)
-
- attack_group = OptionGroup(parser, "attack",
- """Beginning an attack requires only that you specify the -u option with the URL you wish to target.""")
+ attack_cmd.add_argument('-u', '--url', metavar="URL", dest='urls', action='append', required=True, help="URL(s) of the target to attack.")
- # Required
- attack_group.add_option('-u', '--url', metavar="URL", nargs=1,
- action='store', dest='url', type='string',
- help="URL of the target to attack.")
- attack_group.add_option('-p', '--post-file', metavar="POST_FILE", nargs=1,
- action='store', dest='post_file', type='string', default=False,
- help="The POST file to deliver with the bee's payload.")
- attack_group.add_option('-m', '--mime-type', metavar="MIME_TYPE", nargs=1,
- action='store', dest='mime_type', type='string', default='text/plain',
- help="The MIME type to send with the request.")
- attack_group.add_option('-n', '--number', metavar="NUMBER", nargs=1,
- action='store', dest='number', type='int', default=1000,
- help="The number of total connections to make to the target (default: 1000).")
- attack_group.add_option('-c', '--concurrent', metavar="CONCURRENT", nargs=1,
- action='store', dest='concurrent', type='int', default=100,
- help="The number of concurrent connections to make to the target (default: 100).")
- attack_group.add_option('-H', '--headers', metavar="HEADERS", nargs=1,
- action='store', dest='headers', type='string', default='',
+ attack_cmd.add_argument('-p', '--post-file', metavar="POST_FILE", dest='post_files', action='append', help="The POST file(s) to deliver with the bee's payload.")
+ attack_cmd.add_argument('-m', '--mime-type', metavar="MIME_TYPE", dest='mime_type', default='text/plain', help="The MIME type to send with the request.")
+ attack_cmd.add_argument('-n', '--number', metavar="NUMBER", dest='number', type=int, default=1000, help="The number of total connections to make to the target (default: 1000).")
+ attack_cmd.add_argument('-c', '--concurrent', metavar="CONCURRENT", dest='concurrent', type=int, default=100, help="The number of concurrent connections to make to the target (default: 100).")
+ attack_cmd.add_argument('-H', '--headers', metavar="HEADERS", dest='headers', default='',
help="HTTP headers to send to the target to attack. Multiple headers should be separated by semi-colons, e.g header1:value1;header2:value2")
- attack_group.add_option('-e', '--csv', metavar="FILENAME", nargs=1,
- action='store', dest='csv_filename', type='string', default='',
- help="Store the distribution of results in a csv file for all completed bees (default: '').")
-
- parser.add_option_group(attack_group)
+ attack_cmd.add_argument('-e', '--csv', metavar="FILENAME", dest='csv_filename', default='', help="Store the distribution of results in a csv file for all completed bees (default: '').")
+ attack_cmd.add_argument('-g', '--gnuplot', metavar="FILENAME", dest='gnuplot_filename', default='', help="Write all measured values out as a 'gnuplot' or TSV (Tab separate values) file (default: '').")
+ attack_cmd.add_argument('-t', '--timelimit', metavar="TIMELIMIT", dest='timelimit', type=int, default=0,
+ help="Maximum number of seconds to spend for benchmarking. This implies a -n 50000 internally. Use this to benchmark the server within a fixed total amount of time (default: no limit).")
+ attack_cmd.add_argument('--stats-file', metavar="FILENAME", dest='stats_filename', default='',
+ help="Store detailed graph ready stats across multiple tests in a csv file. Will create gnuplot files even if the -g/--gnuplot wasn't specified (default: '').")
+ attack_cmd.add_argument('--testname', metavar="NAME", dest='testname', default='unnamed', help="Name of current test. To be used in conjunction with --stats-file (default: 'unnamed').")
+ attack_cmd.add_argument('--non-200-is-failure', dest='non_200_is_failure', action='store_true', default=False, help="Treat non-200 responses as failures (treated as success by default).")
- (options, args) = parser.parse_args()
+ down_cmd = subparsers.add_parser("down", help='Shutdown and deactivate the load testing servers.', description='Shutdown and deactivate the load testing servers.')
+ report_cmd = subparsers.add_parser("report", help='Report the status of the load testing servers.', description='Report the status of the load testing servers.')
- if len(args) <= 0:
- parser.error('Please enter a command.')
+ options = parser.parse_args()
- command = args[0]
+ command = options.command
if command == 'up':
- if not options.key:
- parser.error('To spin up new instances you need to specify a key-pair name with -k')
-
if options.group == 'default':
print 'New bees will use the "default" EC2 security group. Please note that port 22 (SSH) is not normally open on this group. You will need to use to the EC2 tools to open it before you will be able to attack.'
bees.up(options.servers, options.group, options.zone, options.instance, options.type, options.login, options.key, options.subnet)
elif command == 'attack':
- if not options.url:
- parser.error('To run an attack you need to specify a url with -u')
-
- parsed = urlparse(options.url)
- if not parsed.scheme:
- parsed = urlparse("http://" + options.url)
+ for url in options.urls:
+ parsed = urlparse(url)
+ if not parsed.scheme:
+ parsed = urlparse("http://" + url)
- if not parsed.path:
- parser.error('It appears your URL lacks a trailing slash, this will disorient the bees. Please try again with a trailing slash.')
+ if not parsed.path:
+ parser.error('It appears your URL lacks a trailing slash, this will disorient the bees. Please try again with a trailing slash.')
additional_options = dict(
headers=options.headers,
- post_file=options.post_file,
+ post_files=options.post_files,
mime_type=options.mime_type,
csv_filename=options.csv_filename,
+ gnuplot_filename=options.gnuplot_filename,
+ stats_filename=options.stats_filename,
+ testname=options.testname,
+ non_200_is_failure=options.non_200_is_failure,
)
- bees.attack(options.url, options.number, options.concurrent, **additional_options)
+ bees.attack(options.urls, options.number, options.concurrent, options.timelimit, **additional_options)
elif command == 'down':
bees.down()
elif command == 'report':
bees.report()
-
def main():
parse_options()
diff --git a/examples/LoadTest.gpi b/examples/LoadTest.gpi
new file mode 100644
index 0000000..38bddf6
--- /dev/null
+++ b/examples/LoadTest.gpi
@@ -0,0 +1,70 @@
+# script to generate graphs from a load test done by Bees with Machine Guns
+#
+# usage:
+# gnuplot -e "filename=''" LoadTest.gpi
+#
+
+# output to a jpeg file
+set terminal jpeg size 1440,900
+
+# This sets the aspect ratio of the graph
+set size 1, 1
+set lmargin 12
+set rmargin 10
+
+set output filename.'.jpg'
+
+# Where to place the legend/key
+set key left top
+
+set multiplot layout 2, 1 title filename font "Bold,20"
+
+# Draw gridlines oriented on the y axis
+set grid y
+# Label the x-axis
+#set xlabel 'Iteration'
+set xlabel 'Concurrent Requests'
+# Tell gnuplot to use commas as the delimiter instead of spaces (default)
+set datafile separator ','
+set key autotitle columnhead
+
+#
+# first graph
+#
+set title "Requests/Second(green) and % Errors(red)" font "Bold,14"
+set ytics nomirror
+set y2tics
+set ylabel 'Requests/Second' textcolor lt 2
+set y2label 'Error Percentage' textcolor lt 1
+set decimal locale
+#set format "%'.0f"
+set format "%'g"
+set format y2 "%g %%"
+set yrange [0:]
+set y2range [0:10]
+
+# Plot the data
+plot filename.'.csv' using 1:7 with lines lt 5 lw 3 axes x1y1, \
+ '' using 1:($7-($7-$8)/2) with lines lt 2 lw 3 axes x1y1, \
+ '' using 1:($6*50) with lines lt 1 lw 2 axes x1y2
+# the creative arithmetic above is done in order to overcome a bug in ab in which it counts each error twice, and since successful hits are calculated as total-bad it also has to be fixed.
+# '' using 1:8 with lines lt 2 lw 3 axes x1y1, \
+# '' using 1:($6*100) with lines lt 1 lw 3 axes x1y2
+unset y2tics
+unset y2label
+set yrange [*:*]
+
+#
+# second graph
+#
+set title "Response Time" font "Bold,14"
+unset ylabel
+set ylabel "ms"
+
+set bars 4.0
+set style fill solid
+
+# Plot the data
+plot filename.'.csv' using 1:15:9:32:31 with candlesticks lt 2 title 'Min/P10/Med/P90/P95' whiskerbars 0.6, \
+ '' using 1:12:12:12:12 with candlesticks lt -1 notitle,\
+ '' using 1:11 with lines lt -1 lw 3
diff --git a/examples/LoadTestIter.gpi b/examples/LoadTestIter.gpi
new file mode 100644
index 0000000..9197558
--- /dev/null
+++ b/examples/LoadTestIter.gpi
@@ -0,0 +1,71 @@
+# script to generate graphs from a load test done by Bees with Machine Guns
+#
+# usage:
+# gnuplot -e "filename=''" LoadTest.gpi
+#
+
+# output to a jpeg file
+set terminal jpeg size 1440,900
+
+# This sets the aspect ratio of the graph
+set size 1, 1
+set lmargin 12
+set rmargin 10
+
+set output filename.'.jpg'
+
+# Where to place the legend/key
+set key left top
+
+set multiplot layout 2, 1 title filename font "Bold,20"
+
+# Draw gridlines oriented on the y axis
+set grid y
+set xtics 1
+# Label the x-axis
+set xlabel 'Iteration'
+#set xlabel 'Concurrent Requests'
+# Tell gnuplot to use commas as the delimiter instead of spaces (default)
+set datafile separator ','
+set key autotitle columnhead
+
+#
+# first graph
+#
+set title "Requests/Second(green) and % Errors(red)" font "Bold,14"
+set ytics nomirror
+set y2tics
+set ylabel 'Requests/Second' textcolor lt 2
+set y2label 'Error Percentage' textcolor lt 1
+set decimal locale
+#set format "%'.0f"
+set format "%'g"
+set format y2 "%g %%"
+set yrange [0:]
+set y2range [0:10]
+
+# Plot the data
+plot filename.'.csv' using 1:7 with lines lt 5 lw 3 axes x1y1, \
+ '' using 1:($7-($7-$8)/2) with lines lt 2 lw 3 axes x1y1, \
+ '' using 1:($6*50) with lines lt 1 lw 3 axes x1y2
+# the creative arithmetic above is done in order to overcome a bug in ab in which it counts each error twice, and since successful hits are calculated as total-bad it also has to be fixed.
+# '' using 1:8 with lines lt 2 lw 3 axes x1y1, \
+# '' using 1:($6*100) with lines lt 1 lw 3 axes x1y2
+unset y2tics
+unset y2label
+set yrange [*:*]
+
+#
+# second graph
+#
+set title "Response Time" font "Bold,14"
+unset ylabel
+set ylabel "ms"
+
+set bars 4.0
+set style fill solid
+
+# Plot the data
+plot filename.'.csv' using 1:15:9:32:31 with candlesticks lt 2 title 'Min/P10/Med/P90/P95' whiskerbars 0.6, \
+ '' using 1:12:12:12:12 with candlesticks lt -1 notitle,\
+ '' using 1:11 with lines lt -1 lw 3
diff --git a/examples/LoadTestNames.gpi b/examples/LoadTestNames.gpi
new file mode 100644
index 0000000..c1e6473
--- /dev/null
+++ b/examples/LoadTestNames.gpi
@@ -0,0 +1,71 @@
+# script to generate graphs from a load test done by Bees with Machine Guns
+#
+# usage:
+# gnuplot -e "filename=''" LoadTest.gpi
+#
+
+# output to a jpeg file
+set terminal jpeg size 1440,900
+
+# This sets the aspect ratio of the graph
+set size 1, 1
+set lmargin 12
+set rmargin 10
+
+set output filename.'.jpg'
+
+# Where to place the legend/key
+set key left top
+
+set multiplot layout 2, 1 title filename font "Bold,20"
+
+# Draw gridlines oriented on the y axis
+set grid y
+# Label the x-axis
+set xlabel 'Iteration'
+#set xlabel 'Concurrent Requests'
+# Tell gnuplot to use commas as the delimiter instead of spaces (default)
+set datafile separator ','
+set key autotitle columnhead
+
+#
+# first graph
+#
+set title "Requests/Second(green) and % Errors(red)" font "Bold,14"
+set ytics nomirror
+set y2tics
+set ylabel 'Requests/Second' textcolor lt 2
+set y2label 'Error Percentage' textcolor lt 1
+set decimal locale
+#set format "%'.0f"
+set format "%'g"
+set format y2 "%g %%"
+set yrange [0:]
+set y2range [0:10]
+#set boxwidth 0.5
+#set style fill solid
+
+# Plot the data
+plot filename.'.csv' using 1:8:xtic(2) with lines lt 5 lw 3 axes x1y1, \
+ '' using 1:($8-($8-$9)/2) with lines lt 2 lw 3 axes x1y1, \
+ '' using 1:($7*50) with lines lt 1 lw 3 axes x1y2
+# the creative arithmetic above is done in order to overcome a bug in ab in which it counts each error twice, and since successful hits are calculated as total-bad it also has to be fixed.
+unset y2tics
+unset y2label
+set yrange [*:*]
+#unset boxwidth
+
+#
+# second graph
+#
+set title "Response Time" font "Bold,14"
+unset ylabel
+set ylabel "ms"
+
+set bars 4.0
+set style fill solid
+
+# Plot the data
+plot filename.'.csv' using 1:16:10:33:32:xtic(2) with candlesticks lt 2 title 'Min/P10/Med/P90/P95' whiskerbars 0.6, \
+ '' using 1:13:13:13:13 with candlesticks lt -1 notitle,\
+ '' using 1:12 with lines lt -1 lw 3
diff --git a/requirements.txt b/requirements.txt
index 89e631f..cd711f9 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,2 +1,3 @@
-boto==2.8.0
-paramiko==1.10.1
+boto>=2.8.0
+paramiko>=1.10.1
+csvkit>=0.5.0