From 2b10547c455ca4df3cced17637c573fa157c3a14 Mon Sep 17 00:00:00 2001 From: David Bailey Date: Tue, 5 May 2015 21:42:33 -0700 Subject: [PATCH] uploading --- basemap.py | 30 +++++++++++++++++ finance.py | 92 +++++++++++++++++++++++++++++++++++++++++++++++++++ freq.py | 5 +++ getFlows.py | 61 ++++++++++++++++++++++++++++++++++ gis.py | 33 ++++++++++++++++++ gpx.py | 8 +++++ map.py | 81 +++++++++++++++++++++++++++++++++++++++++++++ risk.py | 40 ++++++++++++++++++++++ splunk.py | 50 ++++++++++++++++++++++++++++ splunkfile.py | 30 +++++++++++++++++ stocks.py | 57 +++++++++++++++++++++++++++++++ 11 files changed, 487 insertions(+) create mode 100644 basemap.py create mode 100644 finance.py create mode 100644 freq.py create mode 100644 getFlows.py create mode 100644 gis.py create mode 100644 gpx.py create mode 100644 map.py create mode 100644 risk.py create mode 100644 splunk.py create mode 100644 splunkfile.py create mode 100644 stocks.py diff --git a/basemap.py b/basemap.py new file mode 100644 index 0000000..e400070 --- /dev/null +++ b/basemap.py @@ -0,0 +1,30 @@ +from mpl_toolkits.basemap import Basemap +from matplotlib import pyplot +import numpy as np +import CoreLocation + +manager = CoreLocation.CLLocationManager.alloc().init() +manager.delegate() +manager.startUpdatingLocation() +loc = manager.location() +if loc is None: + lat, lon = 0,0 +else: + coord = loc.coordinate() + lat, lon = coord.latitude, coord.longitude + +//def convert_to_decimal(degrees, arcminutes, arcseconds): +// return float(degrees + arcminutes/60. + arcseconds/3600.) + +m = Basemap(projection='merc',llcrnrlat=-80,urcrnrlat=80,llcrnrlon=-180,urcrnrlon=180,lat_ts=20,resolution='c') + +m.drawcoastlines() +m.drawstates() +m.bluemarble() +m.drawparallels(np.arange(-90.,91.,30.)) +m.drawmeridians(np.arange(-180.,181.,60.)) +m.drawmapboundary(fill_color='aqua') +m.scatter(lon,lat,100,latlon=1,color='m',marker='.') + +pyplot.title("Mercator Projection | Latitude: " + str(lat) + ", Longitude: " + str(lon)) +pyplot.show() diff --git a/finance.py b/finance.py new file mode 100644 index 0000000..c88a5ea --- /dev/null +++ b/finance.py @@ -0,0 +1,92 @@ +import math +import csv +import numpy +import numpy as np +import scipy as sp +import matplotlib as mpl +import matplotlib.pyplot as plt +import scipy.fftpack +import datetime +import pandas as pd +from scipy import stats + +def pv(fv,r): + return (fv/(1+r)) + +def pvSum(c,r): //PV as a sum of cash flows c w/ interest rate + sum=0 + for n, ck in enumerate(c): + print n,ck,r + sum += ck/((1+r)**n) + return sum + +def annuity(A,r): //Annuity pays A forever + return A/r + +def annuityEnd(A,r,n)Annuity that pays A until n + return annuity(A,r)*(1-1/((1+r)**n)) + +def discountRate(spotRate,t): + return 1/((1+spotRate)**t) + +def fv(pv,r): + return (pv*(1+r)) + +def p(f,i,n): + return (f/((1+i)**n)) +vec_p = np.vectorize(p) +ptest = np.array([np.array([ 1., 2., 3.]), np.array(10), np.array(10)]) +vec_p(ptest[0],ptest[1],ptest[2]) + +def f(p,i,n): + return (p*((1+i)**n)) + +def fe(p,i,n): + return (p*(math.e**(i*n))) + +def pe(f,i,n): + return (f/(math.e**(i*n))) + +cpiv = [] +cpid = [] +with open('CPIAUCSL.csv', 'rb') as cpifile: + cpi = csv.reader(cpifile) + for row in cpi: + cpiv.append(row[1]) + cpid.append(row[0]) + +cpiv.pop(0) +cpid.pop(0) + +cpidd = [] +for item in cpid: + cpidd.append(mpl.dates.date2num(datetime.datetime.strptime(item, "%Y-%m-%d"))) + +plt.plot_date(x=cpidd,y=cpiv) +plt.show() + +c = [] +b = 0 +for a in cpiv: + if b: c.append((float(a)-b)/float(a)) + b = float(a) + +numpy.mean(c)*12 + +x = np.poly1d([1,0]) + +cpif = scipy.fftpack.fft(cpim) + +def ERi(Rf,Bi,ERm): //CAPM + return (Rf+Bi*(ERm-Rf)) + +df = pd.read_csv('CPIAUCSL.csv') + +x = [5.05, 6.75, 3.21, 2.66] +y = [1.65, 26.5, -5.93, 7.96] +gradient, intercept, r_value, p_value, std_err = stats.linregress(x,y) +t = np.arange([0.0, 10.0, 0.1]) +z = gradient*t+intercept +plt.plot(t,z) +plt.scatter(x,y) +plt.show() diff --git a/freq.py b/freq.py new file mode 100644 index 0000000..d365936 --- /dev/null +++ b/freq.py @@ -0,0 +1,5 @@ +import pandas as pd +import matplotlib.pyplot as plt +freq = pd.read_csv('frequency.csv', sep=',') +freq['count'].plot() +plt.show() diff --git a/getFlows.py b/getFlows.py new file mode 100644 index 0000000..cf7c216 --- /dev/null +++ b/getFlows.py @@ -0,0 +1,61 @@ +from lxml import etree +from cStringIO import StringIO +import pycurl +import pandas as pd +import datetime + +start = datetime.datetime(2014,9,19) +finish = datetime.datetime(2014,11,2) +delta = datetime.timedelta(hours=1) +times = [] +while start <= finish: + times.append(start) + start += delta + +for start in times: + print start.strftime("%Y-%m-%dT%H:%M:%SZ") + end = start + delta + request1 = '' + request2 = '' + request3 = '175,53' + request4 = '' + request = request1 + request2 + request3 + request4 + buffer = StringIO() + co = pycurl.Curl() + co.setopt(co.UNRESTRICTED_AUTH,1) + co.setopt(co.URL,"https://lancope.example.com/smc/swsService/flows") + co.setopt(co.POST, 1) + co.setopt(co.INFILESIZE,len(request) + 1) + co.setopt(co.WRITEFUNCTION, buffer.write) + co.setopt(co.POSTFIELDS, request) + co.setopt(co.SSL_VERIFYPEER, 0L) + co.setopt(co.SSL_VERIFYHOST, 0L) + co.setopt(co.USERPWD,"username:password") + try: + co.perform() + except: + print "POST failed" + exit(1) + + co.close() + out = buffer.getvalue() + buffer.close() + doc = etree.fromstring(out) + netflows = [('client', 'clientHostName', 'clientPort', 'clientPackets', 'clientBytes', 'server', 'serverHostName', 'serverPort', 'serverPackets', 'serverBytes', 'startTime', 'lastTime', 'activeDuration')] + for elem in doc.getiterator('{http://www.lancope.com/sws/sws-service}flow'): + startTime = elem.get('start-time') + lastTime = elem.get('last-time') + activeDuration = elem.get('active-duration') + client = elem[0].get('ip-address') + clientHostName = elem[0].get('host-name') + clientPort = elem[0].get('port') + clientPackets = elem[0].get('packets') + clientBytes = elem[0].get('bytes') + server = elem[1].get('ip-address') + serverHostName = elem[1].get('host-name') + serverPort = elem[1].get('port') + serverPackets = elem[1].get('packets') + serverBytes = elem[1].get('bytes') + netflows.append((client, clientHostName, clientPort, clientPackets, clientBytes, server, serverHostName, serverPort, serverPackets, serverBytes, startTime, lastTime, activeDuration)) + + pd.DataFrame(netflows).to_csv(start.strftime("%Y-%m-%dT%H%M%SZ") + end.strftime("%Y-%m-%dT%H%M%SZ") + '.csv') diff --git a/gis.py b/gis.py new file mode 100644 index 0000000..d8971b8 --- /dev/null +++ b/gis.py @@ -0,0 +1,33 @@ +import shapely.geometry +import matplotlib.pyplot as plt +import fiona.collection +import descartes + +places = {'Los Angeles': (34.0204989,-118.4117325), 'Phoenix': (33.6054149,-112.125051), 'Albuquerque': (35.0824099,-106.6764794)} + +path = [(x, y) for y, x in places.values()] +ls = shapely.geometry.LineString(path) + +with fiona.collection("tl_2014_us_state/tl_2014_us_state.shp") as features: + states = [shapely.geometry.shape(f['geometry']) for f in features] + +fig = plt.figure(figsize=(8,5), dpi=180) +ax = fig.add_subplot(111) +ax.axis([-125, -65, 25, 50]) +ax.axis('off') + +ax.plot(*ls.xy, color='#FFFFFF') + +for state in states: + if state.geom_type =='Polygon': + state = [state] + for poly in state: + poly_patch = descartes.PolygonPatch(poly, fc='#6699cc', ec='#000000') + ax.add_patch(poly_patch) + +for x, y in path: + buffered = shapely.geometry.Point(x, y).buffer(1) + ax.add_patch(descartes.PolygonPatch(buffered, fc='#EEEEEE', ec='#000000')) + + +fig.show() diff --git a/gpx.py b/gpx.py new file mode 100644 index 0000000..472240f --- /dev/null +++ b/gpx.py @@ -0,0 +1,8 @@ +import gpxpy + +gpx_file = open("file", 'r') +gpx = gpxpy.parse(gpx_file) + +points = [] +for point in gpx.tracks[0].segments[0].points: + points.append((point.latitude,point.longitude,point.elevation,point.time)) diff --git a/map.py b/map.py new file mode 100644 index 0000000..83b2631 --- /dev/null +++ b/map.py @@ -0,0 +1,81 @@ +import overpy +import numpy +from matplotlib import pyplot +from shapely.geometry import LineString +from shapely.geometry import shape +from shapely.ops import polygonize_full +from descartes import PolygonPatch +from fiona import collection +from geopandas import GeoSeries +from geopandas import GeoDataFrame + +api = overpy.Overpass() + +countyRelation = api.query("rel(396479);(._;>;);out;") +primaryHighwaysInBoundingBox = api.query("way(33.5277,-119.3500,34.9895,-117.1115)[highway=primary];(._;>);out;") + +countyWaysAboveIslands = [] +for way in countyRelation.ways: + above = 1 + for node in way.nodes: + if (node.lat < 33.6078): + above = 0 + if (above): + countyWaysAboveIslands.append(way) + +countyLineStrings = [] +for way in countyWaysAboveIslands: + lineString = [] + for node in way.nodes: + lineString.append((node.lat,node.lon)) + countyLineStrings.append(LineString(lineString)) + +polygons, dangles, cuts, invalids = polygonize_full(countyLineStrings) +countyPolygon = polygons.geoms[0] + +highwayLineStrings = [] +for way in primaryHighwaysInBoundingBox.ways: + line = [] + for node in way.nodes: + line.append((node.lat,node.lon)) + wayLineString = LineString(line) + if countyPolygon.contains(wayLineString): highwayLineStrings.append(wayLineString) + +fig = pyplot.figure(figsize=(100,100)) +ax = fig.add_subplot(111) + +for line in highwayLineStrings: + x, y = line.xy + ax.plot(x, y, color='#999999', linewidth=1, zorder=1) + +patch = PolygonPatch(countyPolygon, fc='#6699cc', ec='#6699cc', alpha=0.5, zorder=2) +ax.add_patch(patch) + +fig.savefig('test.png') + +// debug info + +from shapely.geometry import MultiLineString +test = MultiLineString(highwayLineStrings) +patch = PolygonPatch(countyPolygon, fc='#6699cc', ec='#6699cc', alpha=0.5, zorder=2) +ax.add_patch(patch) + +object.__dict__ +result.nodes +result.nodes.[0].lat +result.ways +result.ways[0]._node_ids[0] +result.relations +result.relations[0].members[0].ref + +len(streetsInBB.ways[0].get_nodes(resolve_missing=True)) + +for way in aboveIslands: + gray = gray + .025 + for node in way.nodes: + ax.scatter(node.lon, node.lat, color=[gray,gray,gray], s=100, zorder=1) + +states = [shapely.geometry.shape(f['geometry']) for f in features] + +http://overpass.osm.rambler.ru/cgi/interpreter?data=%5Bout:json%5D;relation(396479);out; + diff --git a/risk.py b/risk.py new file mode 100644 index 0000000..938694b --- /dev/null +++ b/risk.py @@ -0,0 +1,40 @@ +import numpy as np +//from pylab import * +import matplotlib.pyplot as plot +import math + +top = np.array([[0,1,2],[2,2,1]]) +mid = np.array([[0,1,2],[2,1,0]]) +bot = np.array([[0,1,2],[2,0,0]]) + +plot.plot(top[0],top[1]) +plot.plot(mid[0],mid[1]) +plot.plot(bot[0],bot[1]) + +t = np.arange(0.0, 10.0, 0.1) +top2 = 10-np.arange(0.0, 1.0, 0.01) +top = 10-(.3*t)**2 +mid = -t + 10 +bot = 10-np.sqrt(t) +bot2 = 1/(t) + +plot.plot(t,top2) +plot.plot(t,top) +plot.plot(t,mid) +plot.plot(t,bot) +plot.plot(t,bot2) + +plot.scatter(2,6) +plot.scatter(2,5) +plot.scatter(9,1) + +plot.xlabel('Cost') +plot.ylabel('Risk') +plot.title('Risk vs. Cost') +plot.grid(True) +plot.show() + + +\\Risk = Money * Time +\\ Time = constant * Money +\\ Outsource Time = constant1 * Time + constant2 * Money diff --git a/splunk.py b/splunk.py new file mode 100644 index 0000000..1db2a1d --- /dev/null +++ b/splunk.py @@ -0,0 +1,50 @@ +import socket +import splunklib.client as client +import splunklib.results as results +import re +import numpy as np +import pandas as pd +from geoip import geolite2 +from ipwhois import IPWhois +import ipwhois + +headers = [('address','user','user2','logontype','time','raw')] +data = [] + +get_address = lambda x: re.findall('(Source Network Address:\t)(.*?)(\n)',x)[0][1] +get_user = lambda x: re.findall('(Security ID:\t\t)(.*?)(\n)',x)[0][1] +get_user2 = lambda x: re.findall('(Security ID:\t\t)(.*?)(\n)',x)[1][1] +get_logontype = lambda x: re.findall('(Logon Type:\t\t\t)(.*?)(\n)',x)[0][1] +search = "search user EventCode=4624 | head 10" + +service = client.connect( + host="splunk.example.com", + port=port, + username="username", + password="password") + +socket.setdefaulttimeout(None) + +response = service.jobs.oneshot(search) +reader = results.ResultsReader(response) + +for result in reader: + data.append((get_address(result['_raw']),get_user(result['_raw']),get_user2(result['_raw']),get_logontype(result['_raw']),result['_time'],result['_raw'])) + +df = pd.DataFrame(data) + +for addr in df[0]: + try: + print socket.gethostbyaddr(addr) + except socket.herror: + print None, None, None + try: + match = geolite2.lookup(addr) + print match.country + except AttributeError: + print None + try: + print IPWhois(addr).lookup() + except ipwhois.ipwhois.IPDefinedError: + print None + diff --git a/splunkfile.py b/splunkfile.py new file mode 100644 index 0000000..2087770 --- /dev/null +++ b/splunkfile.py @@ -0,0 +1,30 @@ +import pandas as pd +import re + //checked = input[(input['_raw'].str.contains(name, case=False))] + +names = ['name1','name2'] + +for name in names: + infile = name + '.csv' + outfile = 'success-' + name + '.csv' + input = pd.read_csv(infile, sep=',') + checked = input + if not checked.empty: + success = checked[(checked['_raw'].str.contains('EventCode=4624'))] + success.index = range(0,len(success)) + failure = checked[(checked['_raw'].str.contains('EventCode=4625'))] + failure.index = range(0,len(failure)) + get_address = lambda x: re.findall('(Source Network Address:\\t)(.*?)(\\r)',x)[0][1] + success['ip'] = success._raw.apply(get_address) + failure['ip'] = failure._raw.apply(get_address) + get_user = lambda x: re.findall('(Security ID:\\t)(.*?)(\\r)',x)[0][1] + success['user'] = success._raw.apply(get_user) + get_user2 = lambda x: re.findall('(Security ID:\\t)(.*?)(\\r)',x)[1][1] + success['user2'] = success._raw.apply(get_user2) + get_logontype = lambda x: re.findall('(Logon Type:\\t)(.*?)(\\r)',x)[0][1] + success['logontype'] = success._raw.apply(get_logontype) + failure['user'] = failure._raw.apply(get_user) + success_csv = pd.DataFrame([success._time,success.ip,success.user,success.user2,success.logontype]).transpose() + failure_csv = pd.DataFrame([failure._time,failure.ip]) + success_csv.to_csv(outfile) + failure_csv.to_csv("failure.csv") diff --git a/stocks.py b/stocks.py new file mode 100644 index 0000000..b80b2af --- /dev/null +++ b/stocks.py @@ -0,0 +1,57 @@ +import datetime +import numpy as np +import statsmodels.api as sm +import statsmodels.formula.api as smf +import matplotlib.pyplot as plt +import pandas as pd +from scipy import stats +from pandas.io.data import DataReader + +stock = DataReader("IVV", "yahoo", start=datetime.datetime(1950, 1, 1)) + +regression = sm.OLS(stock['Adj Close'], sm.add_constant(range(len(stock.index)), prepend=True)).fit() +stock['regression'] = regression.fittedvalues +m = regression.params.x1 +b = regression.params.const + +def y(x,m,b): + return m*x+b + +yv = np.vectorize(y) + +def v(x): + return (stock['Adj Close'][x]-stock['Adj Close'][x-1])/stock['Adj Close'][x-1] + +delta = [] +for x in range(1, len(stock.index)): delta.append(v(x)) + +plt.plot(stock.index, stock['Adj Close'], 'b-', stock.index, stock['regression'], 'r-', stock.index, yv(range(len(stock.index)),np.array(delta).mean()*100,65.54), 'g-') + +plt.show() + + +np.random.normal(size=200) + +value=65.54 +values=[] +for x in range(0,2000): + value = value + np.array(delta).mean() * 100 * np.random.normal() + np.random.normal() + values.append(value) + +plt.plot(range(0,2000),values) +plt.show() + +plt.plot(stats.binom.cdf(range(0,20),50,.2)) + +x = numpy.arange(-10, 10, .01) +from scipy.stats import norm +rv1 = norm(loc = 0., scale = 2.0) +plt.plot(x,rv1.pdf(x)) +plt.plot(numpy.fft.fft(rv1.pdf(x))) +a = [0,1,1,0] +plt.plot(numpy.convolve(a,rv1.pdf(x))) + +import math +def g(t): return math.exp(-t**2/2) + +