From 4c372ae8f6b9b49b6e7d2cb11f0253b8e9f03ed3 Mon Sep 17 00:00:00 2001 From: Eran Raichstein Date: Fri, 29 Nov 2019 06:43:47 +0200 Subject: [PATCH] Add support for multijson example --- .gitignore | 1 + tools/csvstoskyui/README.md | 3 +- .../{topologyrules.conf => dump.conf} | 0 .../data/example_multijson/DataCenter1.conf | 16 + .../data/example_multijson/DataCenter2.conf | 16 + .../data/example_multijson/applications.csv | 3 + .../data/example_multijson/config.js | 423 ++++++++++++++++++ .../data/example_multijson/data_centers.csv | 3 + .../data/example_multijson/dump.conf | 4 + .../data/example_multijson/hosts.csv | 5 + .../data/example_multijson/vms.csv | 9 + .../{topologyrules.conf => dump.conf} | 0 .../{topologyrules.conf => dump.conf} | 0 tools/csvstoskyui/do.sh | 41 +- 14 files changed, 521 insertions(+), 3 deletions(-) rename tools/csvstoskyui/data/example_datacenter/{topologyrules.conf => dump.conf} (100%) create mode 100644 tools/csvstoskyui/data/example_multijson/DataCenter1.conf create mode 100644 tools/csvstoskyui/data/example_multijson/DataCenter2.conf create mode 100644 tools/csvstoskyui/data/example_multijson/applications.csv create mode 100644 tools/csvstoskyui/data/example_multijson/config.js create mode 100644 tools/csvstoskyui/data/example_multijson/data_centers.csv create mode 100644 tools/csvstoskyui/data/example_multijson/dump.conf create mode 100644 tools/csvstoskyui/data/example_multijson/hosts.csv create mode 100644 tools/csvstoskyui/data/example_multijson/vms.csv rename tools/csvstoskyui/data/example_scale/{topologyrules.conf => dump.conf} (100%) rename tools/csvstoskyui/data/example_submariner/{topologyrules.conf => dump.conf} (100%) diff --git a/.gitignore b/.gitignore index abf7bfe..d5a63f8 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,7 @@ skydivetopology.json .DS_Store .env .idea/ +out/ npm-debug.log* yarn-debug.log* yarn-error.log* diff --git a/tools/csvstoskyui/README.md b/tools/csvstoskyui/README.md index 24b6b0e..4b98e91 100644 --- a/tools/csvstoskyui/README.md +++ b/tools/csvstoskyui/README.md @@ -9,10 +9,11 @@ 1. data/example_datacenter 1. data/example_submariner 1. data/example_scale +1. data/example_multijson ### Note: this project is using [Skydive UI](https://github.com/skydive-project/skydive-ui) ### Installation * Clone this repository -* Execute `./do.sh` +* Execute `./do.sh -d example_datacenter` * Browse to \ No newline at end of file diff --git a/tools/csvstoskyui/data/example_datacenter/topologyrules.conf b/tools/csvstoskyui/data/example_datacenter/dump.conf similarity index 100% rename from tools/csvstoskyui/data/example_datacenter/topologyrules.conf rename to tools/csvstoskyui/data/example_datacenter/dump.conf diff --git a/tools/csvstoskyui/data/example_multijson/DataCenter1.conf b/tools/csvstoskyui/data/example_multijson/DataCenter1.conf new file mode 100644 index 0000000..c1ac41b --- /dev/null +++ b/tools/csvstoskyui/data/example_multijson/DataCenter1.conf @@ -0,0 +1,16 @@ +{ + "data_center": {"filename": "data/example_multijson/data_centers.csv", + "valuesfilter": {"Name": "DataCenter1"}, + "show": True}, + "host": {"filename": "data/example_multijson/hosts.csv", + "show": True, + "valuesfilter": {"DataCenter": "DataCenter1"}, + "edges": [{"from": ["DataCenter"], "to": {"data_center": ["Name"]}, "RelationType": "ownership"}]}, + "vm": {"filename": "data/example_multijson/vms.csv", + "show": True, + "valuesfilter": {"Host": "Host1|Host2"}, + "edges": [{"from": ["Host"], "to": {"host": ["Name"]}, "RelationType": "ownership"}, + {"from": ["Application"], "to": {"application": ["Name"]}, "RelationType": "application"}]}, + "application": {"filename": "data/example_multijson/applications.csv", + "show": True} +} \ No newline at end of file diff --git a/tools/csvstoskyui/data/example_multijson/DataCenter2.conf b/tools/csvstoskyui/data/example_multijson/DataCenter2.conf new file mode 100644 index 0000000..68b9eb2 --- /dev/null +++ b/tools/csvstoskyui/data/example_multijson/DataCenter2.conf @@ -0,0 +1,16 @@ +{ + "data_center": {"filename": "data/example_multijson/data_centers.csv", + "valuesfilter": {"Name": "DataCenter2"}, + "show": True}, + "host": {"filename": "data/example_multijson/hosts.csv", + "show": True, + "valuesfilter": {"DataCenter": "DataCenter2"}, + "edges": [{"from": ["DataCenter"], "to": {"data_center": ["Name"]}, "RelationType": "ownership"}]}, + "vm": {"filename": "data/example_multijson/vms.csv", + "show": True, + "valuesfilter": {"Host": "Host3|Host4"}, + "edges": [{"from": ["Host"], "to": {"host": ["Name"]}, "RelationType": "ownership"}, + {"from": ["Application"], "to": {"application": ["Name"]}, "RelationType": "application"}]}, + "application": {"filename": "data/example_multijson/applications.csv", + "show": True} +} \ No newline at end of file diff --git a/tools/csvstoskyui/data/example_multijson/applications.csv b/tools/csvstoskyui/data/example_multijson/applications.csv new file mode 100644 index 0000000..3110748 --- /dev/null +++ b/tools/csvstoskyui/data/example_multijson/applications.csv @@ -0,0 +1,3 @@ +Name +Database +Web diff --git a/tools/csvstoskyui/data/example_multijson/config.js b/tools/csvstoskyui/data/example_multijson/config.js new file mode 100644 index 0000000..327447e --- /dev/null +++ b/tools/csvstoskyui/data/example_multijson/config.js @@ -0,0 +1,423 @@ +var config = { + "nodeAttrs": function (node) { + console.log(node.data) + var name = node.data.Name + if (name.length > 24) { + name = node.data.Name.substring(0, 24) + "." + } + + var attrs = { classes: [node.data.Type], name: name, icon: "\uf192", iconClass: '', weight: 0 } + + if (node.data.OfPort) { + attrs.weight = 15 + } + + if (node.data.Manager === "k8s") { + attrs.icon = "/assets/icons/k8s.png" + attrs.weight = 1 + } + + switch (node.data.Type) { + case "data_center": + attrs.icon = "\uf109" + attrs.weight = 2 + break + case "application": + attrs.icon = "\uf109" + attrs.weight = 3 + break + case "host": + attrs.icon = "\uf109" + attrs.weight = 4 + break + case "vm": + attrs.icon = "\uf109" + attrs.weight = 5 + break + case "host": + attrs.icon = "\uf109" + attrs.weight = 13 + break + case "switch": + attrs.icon = "\uf6ff" + break + case "bridge": + case "ovsbridge": + attrs.icon = "\uf6ff" + attrs.weight = 14 + break + case "erspan": + attrs.icon = "\uf1e0" + break + case "geneve": + case "vxlan": + case "gre": + case "gretap": + attrs.icon = "\uf55b" + break + case "device": + case "internal": + case "interface": + case "tun": + case "tap": + attrs.icon = "\uf796" + attrs.weight = 17 + break + case "veth": + attrs.icon = "\uf4d7" + attrs.weight = 17 + break + case "switchport": + attrs.icon = "\uf0e8" + break + case "patch": + case "port": + case "ovsport": + attrs.icon = "\uf0e8" + attrs.weight = 15 + break + case "netns": + attrs.icon = "\uf24d" + attrs.weight = 18 + break + case "libvirt": + attrs.icon = "\uf109" + attrs.weight = 19 + break + case "cluster": + attrs.icon = "/assets/icons/cluster.png" + break + case "configmap": + attrs.icon = "/assets/icons/configmap.png" + break + case "container": + attrs.icon = "/assets/icons/container.png" + break + case "cronjob": + attrs.icon = "/assets/icons/cronjob.png" + break + case "daemonset": + attrs.icon = "/assets/icons/daemonset.png" + break + case "deployment": + attrs.icon = "/assets/icons/deployment.png" + break + case "endpoints": + attrs.icon = "/assets/icons/endpoints.png" + break + case "ingress": + attrs.icon = "/assets/icons/ingress.png" + break + case "job": + attrs.icon = "/assets/icons/job.png" + break + case "node": + attrs.icon = "\uf109" + break + case "persistentvolume": + attrs.icon = "/assets/icons/persistentvolume.png" + break + case "persistentvolumeclaim": + attrs.icon = "/assets/icons/persistentvolumeclaim.png" + break + case "pod": + attrs.icon = "/assets/icons/pod.png" + break + case "networkpolicy": + attrs.icon = "/assets/icons/networkpolicy.png" + break + case "namespace": + attrs.icon = "\uf24d" + break + case "replicaset": + attrs.icon = "/assets/icons/replicaset.png" + break + case "replicationcontroller": + attrs.icon = "/assets/icons/replicationcontroller.png" + break + case "secret": + attrs.icon = "/assets/icons/secret.png" + break + case "service": + attrs.icon = "/assets/icons/service.png" + break + case "statefulset": + attrs.icon = "/assets/icons/statefulset.png" + break + case "storageclass": + attrs.icon = "/assets/icons/storageclass.png" + break + default: + attrs.icon = "\uf192" + break + } + + if (node.data.Manager === "docker") { + attrs.icon = "\uf395" + attrs.iconClass = "font-brands" + } + + if (node.data.IPV4 && node.data.IPV4.length) { + attrs.weight = 13 + } + + if (node.data.Driver && ["tap", "veth", "tun", "openvswitch"].indexOf(node.data.Driver) < 0) { + attrs.weight = 13 + } + + if (node.data.Probe === "fabric") { + attrs.weight = 10 + } + + return attrs + }, + "nodeMenu": function (node) { + + var menu = [ + { class: "", text: "Capture", disabled: false, callback: () => { console.log("Capture") } }, + { class: "", text: "Capture all", disabled: true, callback: () => { console.log("Capture all") } }, + { class: "", text: "Injection", disabled: false, callback: () => { console.log("Injection") } }, + { class: "", text: "Flows", disabled: false, callback: () => { console.log("Flows") } } + ] + + if (node.data.Expandable && node.data.Expandable === "True") { + menu.push ({ class: "", text: "Expend", disabled: false, callback: () => { window.location.replace("?data=/assets/"+node.data.Name+".json"); } }) + menu.push ({ class: "", text: "Home", disabled: false, callback: () => { window.location.replace("?data=/assets/dump.json"); } }) + } + + return menu + }, + "nodeTags": function (data) { + switch (data.Type) { + case "VLAN": + return ["logical"] + case "tenant": + return ["logical"] + default: + return ["physical"] + } + }, + "defaultNodeTag": "physical", + "nodeTabTitle": function (node) { + return node.data.Name.substring(0, 8) + }, + "groupBy": function (node) { + return node.data.Type && node.data.Type !== "host" ? node.data.Type : null + }, + "weightTitles": { + 0: "Not classified", + 1: "Kubernetes", + 2: "data_center", + 3: "application", + 4: "host", + 5: "vm", + 10: "Fabric", + 13: "Physical", + 14: "Bridges", + 15: "Ports", + 17: "Virtual", + 18: "Namespaces", + 19: "VMs" + }, + "suggestions": [ + "data.IPV4", + "data.MAC", + "data.Name" + ], + "nodeDataFields": [ + { + field: "", + title: "General", + expanded: true, + icon: "\uf05a", + sortKeys: function (data) { + return ['Name', 'Type', 'MAC', 'Driver', 'State'] + }, + filterKeys: function (data) { + switch (data.Type) { + case "host": + return ['Name'] + default: + return ['Name', 'Type', 'MAC', 'Driver', 'State'] + } + } + }, + { + field: "Sockets", + expanded: false, + icon: "\uf1e6" + }, + { + field: "Captures", + expanded: false, + icon: "\uf51f", + normalizer: function (data) { + for (let capture of data) { + capture.ID = capture.ID.split('-')[0] + } + return data + } + }, + { + field: "Injections", + expanded: false, + icon: "\uf48e" + }, + { + field: "Docker", + expanded: false, + icon: "\uf395", + iconClass: "font-brands" + }, + { + field: "IPV4", + expanded: true, + icon: "\uf1fa" + }, + { + field: "IPV6", + expanded: true, + icon: "\uf1fa" + }, + { + field: "LastUpdateMetric", + title: "Last metrics", + expanded: false, + icon: "\uf201", + normalizer: function (data) { + return { + RxPackets: data.RxPackets ? data.RxPackets.toLocaleString() : 0, + RxBytes: data.RxBytes ? prettyBytes(data.RxBytes) : 0, + TxPackets: data.TxPackets ? data.TxPackets.toLocaleString() : 0, + TxBytes: data.TxPackets ? prettyBytes(data.TxBytes) : 0, + Start: data.Start ? new Date(data.Start).toLocaleString() : 0, + Last: data.Last ? new Date(data.Last).toLocaleString() : 0 + } + }, + graph: function (data) { + return { + type: "LineChart", + data: [ + [ + { type: "datetime", label: "time" }, + "RxBytes", + "TxBytes" + ], + [new Date(data.Last || 0), data.RxBytes || 0, data.TxBytes || 0] + ] + } + } + }, + { + field: "Metric", + title: "Total metrics", + expanded: false, + icon: "\uf201", + normalizer: function (data) { + return { + RxPackets: data.RxPackets ? data.RxPackets.toLocaleString() : 0, + RxBytes: data.RxBytes ? prettyBytes(data.RxBytes) : 0, + TxPackets: data.TxPackets ? data.TxPackets.toLocaleString() : 0, + TxBytes: data.TxPackets ? prettyBytes(data.TxBytes) : 0, + Last: data.Last ? new Date(data.Last).toLocaleString() : 0 + } + } + }, + { + field: "Features", + expanded: false, + icon: "\uf022" + }, + { + field: "FDB", + expanded: false, + icon: "\uf0ce" + }, + { + field: "Neighbors", + expanded: false, + icon: "\uf0ce" + }, + { + field: "RoutingTables", + title: "Routing tables", + expanded: false, + icon: "\uf0ce", + normalizer: function (data) { + var rows = [] + for (let table of data) { + if (!table.Routes) { + continue + } + for (let route of table.Routes) { + if (!route.NextHops) { + continue + } + for (let nh of route.NextHops) { + rows.push({ + ID: table.ID, + Src: table.Src, + Protocol: route["Protocol"], + Prefix: route["Prefix"], + Priority: nh["Priority"], + IP: nh["IP"], + IfIndex: nh["IfIndex"] + }) + } + } + } + + return rows + } + } + ], + "linkAttrs": function (link) { + var attrs = { classes: [link.data.RelationType], icon: "\uf362", directed: false } + + if (link.data.Directed) { + attrs.directed = true + } + + return attrs + }, + "linkTabTitle": function (link) { + var src = link.source.data.Name + var dst = link.target.data.Name + if (src && dst) { + return src.substring(0, 8) + " / " + dst.substring(0, 8) + } + return link.id.split("-")[0] + }, + "linkDataFields": [ + { + field: "", + title: "General", + expanded: true, + icon: "\uf05a", + }, + { + field: "NSM", + title: "Network Service Mesh", + expanded: true, + icon: "\uf542", + }, + { + field: "NSM.Source", + title: "Source", + expanded: false, + icon: "\uf018", + }, + { + field: "NSM.Via", + title: "Via", + expanded: false, + icon: "\uf018", + }, + { + field: "NSM.Destination", + title: "Destination", + expanded: false, + icon: "\uf018", + }, + ] +} \ No newline at end of file diff --git a/tools/csvstoskyui/data/example_multijson/data_centers.csv b/tools/csvstoskyui/data/example_multijson/data_centers.csv new file mode 100644 index 0000000..8734973 --- /dev/null +++ b/tools/csvstoskyui/data/example_multijson/data_centers.csv @@ -0,0 +1,3 @@ +Name,Location,Expandable +DataCenter1,East,True +DataCenter2,West,True diff --git a/tools/csvstoskyui/data/example_multijson/dump.conf b/tools/csvstoskyui/data/example_multijson/dump.conf new file mode 100644 index 0000000..8432c38 --- /dev/null +++ b/tools/csvstoskyui/data/example_multijson/dump.conf @@ -0,0 +1,4 @@ +{ + "data_center": {"filename": "data/example_multijson/data_centers.csv", + "show": True} +} \ No newline at end of file diff --git a/tools/csvstoskyui/data/example_multijson/hosts.csv b/tools/csvstoskyui/data/example_multijson/hosts.csv new file mode 100644 index 0000000..34b5018 --- /dev/null +++ b/tools/csvstoskyui/data/example_multijson/hosts.csv @@ -0,0 +1,5 @@ +Name,CPUs,Memory,DataCenter +Host1,4,64,DataCenter1 +Host2,4,64,DataCenter1 +Host3,4,128,DataCenter2 +Host4,16,64,DataCenter2 diff --git a/tools/csvstoskyui/data/example_multijson/vms.csv b/tools/csvstoskyui/data/example_multijson/vms.csv new file mode 100644 index 0000000..8a17bdf --- /dev/null +++ b/tools/csvstoskyui/data/example_multijson/vms.csv @@ -0,0 +1,9 @@ +Name,Host,Application +VM1,Host1,Web +VM2,Host1,Database +VM3,Host2,Database +VM4,Host2,Database +VM5,Host3,Database +VM6,Host4,Web +VM7,Host4,Database + diff --git a/tools/csvstoskyui/data/example_scale/topologyrules.conf b/tools/csvstoskyui/data/example_scale/dump.conf similarity index 100% rename from tools/csvstoskyui/data/example_scale/topologyrules.conf rename to tools/csvstoskyui/data/example_scale/dump.conf diff --git a/tools/csvstoskyui/data/example_submariner/topologyrules.conf b/tools/csvstoskyui/data/example_submariner/dump.conf similarity index 100% rename from tools/csvstoskyui/data/example_submariner/topologyrules.conf rename to tools/csvstoskyui/data/example_submariner/dump.conf diff --git a/tools/csvstoskyui/do.sh b/tools/csvstoskyui/do.sh index 807f3c5..54c7eff 100755 --- a/tools/csvstoskyui/do.sh +++ b/tools/csvstoskyui/do.sh @@ -1,4 +1,41 @@ #!/bin/bash -python csvstoskyui.py data/example_scale/topologyrules.conf > skydivetopology.json + dir="$(realpath "$(dirname "${BASH_SOURCE[0]}")")" -docker run -p 8080:8080 -v "$dir"/skydivetopology.json:/usr/src/skydive-ui/assets/dump.json -v "$dir"/data/example_scale/config.js:/usr/src/skydive-ui/assets/config.js skydive/skydive-ui +data_dir="example_datacenter" +usage="$(basename "$0") [-h] [-d] -- convert csvs from example folder to json and start skydive ui from container + +where: + -h show this help text + -d example direcory + +Avaliable example direcories (for -d): + +$(dir -1 data) + +Example: + +./$(basename "$0") -d $data_dir +" + +if [ "$1" == "-d" ]; then + data_dir=$2 +else + echo "$usage" >&2 + exit 0 +fi + +echo Using data dir : "$dir"/data/$data_dir/ + +unset mountconf_to_docker +destjson_dir="out" +rm -r $destjson_dir +mkdir $destjson_dir +for rules_filename in data/$data_dir/*.conf; do + echo Using rules file: $rules_filename + destjsonfile=$(basename "$rules_filename" .conf).json + python csvstoskyui.py $rules_filename > $destjson_dir/$destjsonfile + mountconf_to_docker=$mountconf_to_docker" -v "$dir"/"$destjson_dir"/"$destjsonfile":/usr/src/skydive-ui/assets/"$destjsonfile +done + +echo Starting skydive ui conainer +docker run -p 8080:8080 $mountconf_to_docker -v "$dir"/data/$data_dir/config.js:/usr/src/skydive-ui/assets/config.js skydive/skydive-ui