telekom-security · t3chn0m4g3 · Nov 27, 2018 · Feb 27, 2018 · Mar 15, 2018 · Mar 15, 2018
diff --git a/.DS_Store b/.DS_Store
diff --git a/CONTRIBUTING.MD b/CONTRIBUTING.MD
@@ -6,32 +6,25 @@ Thank you for your decision to contribute to T-Pot.
 
 Please feel free to post your problems, ideas and issues [here](https://github.com/dtag-dev-sec/tpotce/issues). We will try to answer ASAP, but to speed things up we encourage you to ...
 - [ ] Use the [search function](https://github.com/dtag-dev-sec/tpotce/issues?utf8=%E2%9C%93&q=) first
-- [ ] Check the [FAQ](#faq)
+- [ ] Check the FAQs in our [WIKI](https://github.com/dtag-dev-sec/tpotce/wiki)
 - [ ] Provide [basic support information](#info) with regard to your issue
 
 Thank you :smiley:
 
--
-
-<a name="faq"></a>
-### FAQ
-
-##### Where can I find the honeypot logs?
-###### The honeypot logs are located in `/data/`. You have to login via ssh and run `sudo su -` and then `cd /data/`. Do not change any permissions here or T-Pot will fail to work.
 
 -
 
 
 <a name="info"></a>
 ### Basic support information
 
-- What T-Pot version are you currtently using?
+- What T-Pot version are you currently using?
 - Are you running on a Intel NUC or a VM?
 - How long has your installation been running?
 - Did you install any upgrades or packages?
 - Did you modify any scripts?
 - Have you turned persistence on/off?
-- How much RAM available (login via ssh and run `htop`)?
+- How much RAM is available (login via ssh and run `htop`)?
 - How much stress are the CPUs under (login via ssh and run `htop`)?
 - How much swap space is being used (login via ssh and run `htop`)?
 - How much free disk space is available (login via ssh and run `sudo df -h`)?

diff --git a/README.md b/README.md
diff --git a/bin/backup_es_folders.sh b/bin/backup_es_folders.sh
@@ -5,7 +5,7 @@ myES="http://127.0.0.1:64298/"
 myESSTATUS=$(curl -s -XGET ''$myES'_cluster/health' | jq '.' | grep -c green)
 if ! [ "$myESSTATUS" = "1" ]
   then
-    echo "### Elasticsearch is not available, try starting via 'systemctl start elk'."
+    echo "### Elasticsearch is not available, try starting via 'systemctl start tpot'."
     exit
   else
     echo "### Elasticsearch is available, now continuing."
@@ -16,7 +16,7 @@ fi
 myCOUNT=1
 myDATE=$(date +%Y%m%d%H%M)
 myELKPATH="/data/elk/data"
-myKIBANAINDEXNAME=$(curl -s -XGET ''$myES'_cat/indices/' | grep .kibana | awk '{ print $4 }')
+myKIBANAINDEXNAME=$(curl -s -XGET ''$myES'_cat/indices/' | grep -w ".kibana_1" | awk '{ print $4 }')
 myKIBANAINDEXPATH=$myELKPATH/nodes/0/indices/$myKIBANAINDEXNAME
 
 # Let's ensure normal operation on exit or if interrupted ...

diff --git a/bin/clean.sh b/bin/clean.sh
@@ -32,10 +32,14 @@ fuLOGROTATE () {
   local myHONEYTRAPATTACKSTGZ="/data/honeytrap/attacks.tgz"
   local myHONEYTRAPDL="/data/honeytrap/downloads/"
   local myHONEYTRAPDLTGZ="/data/honeytrap/downloads.tgz"
+  local myTANNERF="/data/tanner/files/"
+  local myTANNERFTGZ="/data/tanner/files.tgz"
 
 # Ensure correct permissions and ownerships for logrotate to run without issues
 chmod 760 /data/ -R
 chown tpot:tpot /data -R
+chmod 644 /data/nginx/conf -R
+chmod 644 /data/nginx/cert -R
 
 # Run logrotate with force (-f) first, so the status file can be written and race conditions (with tar) be avoided
 logrotate -f -s $mySTATUS $myCONF
@@ -47,23 +51,32 @@ if [ "$(fuEMPTY $myDIONAEABI)" != "0" ]; then tar cvfz $myDIONAEABITGZ $myDIONAE
 if [ "$(fuEMPTY $myDIONAEABIN)" != "0" ]; then tar cvfz $myDIONAEABINTGZ $myDIONAEABIN; fi
 if [ "$(fuEMPTY $myHONEYTRAPATTACKS)" != "0" ]; then tar cvfz $myHONEYTRAPATTACKSTGZ $myHONEYTRAPATTACKS; fi
 if [ "$(fuEMPTY $myHONEYTRAPDL)" != "0" ]; then tar cvfz $myHONEYTRAPDLTGZ $myHONEYTRAPDL; fi
+if [ "$(fuEMPTY $myTANNERF)" != "0" ]; then tar cvfz $myTANNERFTGZ $myTANNERF; fi
 
 # Ensure correct permissions and ownership for previously created archives
-chmod 760 $myCOWRIETTYTGZ $myCOWRIEDLTGZ $myDIONAEABITGZ $myDIONAEABINTGZ $myHONEYTRAPATTACKSTGZ $myHONEYTRAPDLTGZ
-chown tpot:tpot $myCOWRIETTYTGZ $myCOWRIEDLTGZ $myDIONAEABITGZ $myDIONAEABINTGZ $myHONEYTRAPATTACKSTGZ $myHONEYTRAPDLTGZ
+chmod 760 $myCOWRIETTYTGZ $myCOWRIEDLTGZ $myDIONAEABITGZ $myDIONAEABINTGZ $myHONEYTRAPATTACKSTGZ $myHONEYTRAPDLTGZ $myTANNERFTGZ
+chown tpot:tpot $myCOWRIETTYTGZ $myCOWRIEDLTGZ $myDIONAEABITGZ $myDIONAEABINTGZ $myHONEYTRAPATTACKSTGZ $myHONEYTRAPDLTGZ $myTANNERFTGZ
 
 # Need to remove subfolders since too many files cause rm to exit with errors
-rm -rf $myCOWRIETTYLOGS $myCOWRIEDL $myDIONAEABI $myDIONAEABIN $myHONEYTRAPATTACKS $myHONEYTRAPDL
+rm -rf $myCOWRIETTYLOGS $myCOWRIEDL $myDIONAEABI $myDIONAEABIN $myHONEYTRAPATTACKS $myHONEYTRAPDL $myTANNERF
 
 # Recreate subfolders with correct permissions and ownership
-mkdir -p $myCOWRIETTYLOGS $myCOWRIEDL $myDIONAEABI $myDIONAEABIN $myHONEYTRAPATTACKS $myHONEYTRAPDL
-chmod 760 $myCOWRIETTYLOGS $myCOWRIEDL $myDIONAEABI $myDIONAEABIN $myHONEYTRAPATTACKS $myHONEYTRAPDL 
-chown tpot:tpot $myCOWRIETTYLOGS $myCOWRIEDL $myDIONAEABI $myDIONAEABIN $myHONEYTRAPATTACKS $myHONEYTRAPDL
+mkdir -p $myCOWRIETTYLOGS $myCOWRIEDL $myDIONAEABI $myDIONAEABIN $myHONEYTRAPATTACKS $myHONEYTRAPDL $myTANNERF
+chmod 760 $myCOWRIETTYLOGS $myCOWRIEDL $myDIONAEABI $myDIONAEABIN $myHONEYTRAPATTACKS $myHONEYTRAPDL $myTANNERF
+chown tpot:tpot $myCOWRIETTYLOGS $myCOWRIEDL $myDIONAEABI $myDIONAEABIN $myHONEYTRAPATTACKS $myHONEYTRAPDL $myTANNERF
 
 # Run logrotate again to account for previously created archives - DO NOT FORCE HERE!
 logrotate -s $mySTATUS $myCONF
 }
 
+# Let's create a function to clean up and prepare ciscoasa data
+fuCISCOASA () {
+  if [ "$myPERSISTENCE" != "on" ]; then rm -rf /data/ciscoasa/*; fi
+  mkdir -p /data/ciscoasa/log
+  chmod 760 /data/ciscoasa -R
+  chown tpot:tpot /data/ciscoasa -R
+}
+
 # Let's create a function to clean up and prepare conpot data
 fuCONPOT () {
   if [ "$myPERSISTENCE" != "on" ]; then rm -rf /data/conpot/*; fi
@@ -101,27 +114,35 @@ fuELK () {
   # ELK data will be kept for <= 90 days, check /etc/crontab for curator modification
   # ELK daemon log files will be removed
   if [ "$myPERSISTENCE" != "on" ]; then rm -rf /data/elk/log/*; fi
-  mkdir -p /data/elk 
+  mkdir -p /data/elk
   chmod 760 /data/elk -R
   chown tpot:tpot /data/elk -R
 }
 
-# Let's create a function to clean up and prepare emobility data
-fuEMOBILITY () {
-  if [ "$myPERSISTENCE" != "on" ]; then rm -rf /data/emobility/*; fi
-  mkdir -p /data/emobility/log
-  chmod 760 /data/emobility -R
-  chown tpot:tpot /data/emobility -R
-}
-
 # Let's create a function to clean up and prepare glastopf data
 fuGLASTOPF () {
   if [ "$myPERSISTENCE" != "on" ]; then rm -rf /data/glastopf/*; fi
-  mkdir -p /data/glastopf
+  mkdir -p /data/glastopf/db /data/glastopf/log
   chmod 760 /data/glastopf -R
   chown tpot:tpot /data/glastopf -R
 }
 
+# Let's create a function to clean up and prepare glastopf data
+fuGLUTTON () {
+  if [ "$myPERSISTENCE" != "on" ]; then rm -rf /data/glutton/*; fi
+  mkdir -p /data/glutton/log
+  chmod 760 /data/glutton -R
+  chown tpot:tpot /data/glutton -R
+}
+
+# Let's create a function to clean up and prepare heralding data
+fuHERALDING () {
+  if [ "$myPERSISTENCE" != "on" ]; then rm -rf /data/heralding/*; fi
+  mkdir -p /data/heralding/log
+  chmod 760 /data/heralding -R
+  chown tpot:tpot /data/heralding -R
+}
+
 # Let's create a function to clean up and prepare honeytrap data
 fuHONEYTRAP () {
   if [ "$myPERSISTENCE" != "on" ]; then rm -rf /data/honeytrap/*; fi
@@ -138,6 +159,22 @@ fuMAILONEY () {
   chown tpot:tpot /data/mailoney/ -R
 }
 
+# Let's create a function to clean up and prepare mailoney data
+fuMEDPOT () {
+  if [ "$myPERSISTENCE" != "on" ]; then rm -rf /data/medpot/*; fi
+  mkdir -p /data/medpot/log/
+  chmod 760 /data/medpot/ -R
+  chown tpot:tpot /data/medpot/ -R
+}
+
+# Let's create a function to clean up nginx logs
+fuNGINX () {
+  if [ "$myPERSISTENCE" != "on" ]; then rm -rf /data/nginx/log/*; fi
+  touch /data/nginx/log/error.log
+  chmod 644 /data/nginx/conf -R
+  chmod 644 /data/nginx/cert -R
+}
+
 # Let's create a function to clean up and prepare rdpy data
 fuRDPY () {
   if [ "$myPERSISTENCE" != "on" ]; then rm -rf /data/rdpy/*; fi
@@ -170,15 +207,14 @@ fuP0F () {
   chown tpot:tpot -R /data/p0f
 }
 
-# Let's create a function to clean up and prepare vnclowpot data
-fuVNCLOWPOT () {
-  if [ "$myPERSISTENCE" != "on" ]; then rm -rf /data/vnclowpot/*; fi
-  mkdir -p /data/vnclowpot/log/
-  chmod 760 /data/vnclowpot/ -R
-  chown tpot:tpot /data/vnclowpot/ -R
+# Let's create a function to clean up and prepare p0f data
+fuTANNER () {
+  if [ "$myPERSISTENCE" != "on" ]; then rm -rf /data/tanner/*; fi
+  mkdir -p /data/tanner/log /data/tanner/files
+  chmod 760 -R /data/tanner
+  chown tpot:tpot -R /data/tanner
 }
 
-
 # Avoid unwanted cleaning
 if [ "$myPERSISTENCE" = "" ];
   then
@@ -201,19 +237,22 @@ if [ "$myPERSISTENCE" = "on" ];
     fuLOGROTATE
   else
     echo "Cleaning up and preparing data folders."
+    fuCISCOASA
     fuCONPOT
     fuCOWRIE
     fuDIONAEA
     fuELASTICPOT
     fuELK
-    fuEMOBILITY
     fuGLASTOPF
+    fuGLUTTON
+    fuHERALDING
     fuHONEYTRAP
     fuMAILONEY
+    fuMEDPOT
+    fuNGINX
     fuRDPY
     fuSPIDERFOOT
     fuSURICATA
     fuP0F
-    fuVNCLOWPOT
+    fuTANNER
   fi
-
diff --git a/bin/dps.sh b/bin/dps.sh
@@ -1,44 +1,15 @@
 #/bin/bash
-# Show current status of all running containers
+# Show current status of T-Pot containers
 myPARAM="$1"
-myIMAGES="$(cat /opt/tpot/etc/tpot.yml | grep -v '#' | grep container_name | cut -d: -f2)"
+myCONTAINERS="$(cat /opt/tpot/etc/tpot.yml | grep -v '#' | grep container_name | cut -d: -f2 | sort | tr -d " ")"
 myRED="[1;31m"
 myGREEN="[1;32m"
 myBLUE="[1;34m"
 myWHITE="[0;0m"
 myMAGENTA="[1;35m"
 
-function fuCONTAINERSTATUS {
-local myNAME="$1"
-local mySTATUS="$(/usr/bin/docker ps -f name=$myNAME --format "table {{.Status}}" -f status=running -f status=exited | tail -n 1)"
-myDOWN="$(echo "$mySTATUS" | grep -o -E "(STATUS|NAMES|Exited)")"
-
-case "$myDOWN" in
-  STATUS)
-    mySTATUS="$myRED"DOWN"$myWHITE"
-  ;;
-  NAMES)
-    mySTATUS="$myRED"DOWN"$myWHITE"
-  ;;
-  Exited)
-    mySTATUS="$myRED$mySTATUS$myWHITE"
-  ;;
-  *)
-    mySTATUS="$myGREEN$mySTATUS$myWHITE"
-  ;;
-esac
-
-printf "$mySTATUS"
-}
-
-function fuCONTAINERPORTS {
-local myNAME="$1"
-local myPORTS="$(/usr/bin/docker ps -f name=$myNAME --format "table {{.Ports}}" -f status=running -f status=exited | tail -n 1 | sed s/","/",\n\t\t\t\t\t\t\t"/g)"
-
-if [ "$myPORTS" != "PORTS" ];
-  then
-    printf "$myBLUE$myPORTS$myWHITE"
-fi
+function fuGETSTATUS {
+grc docker ps -f status=running -f status=exited --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}" | grep -v "NAME" | sort
 }
 
 function fuGETSYS {
@@ -51,16 +22,20 @@ echo
 
 while true
   do
+    myDPS=$(fuGETSTATUS)
+    myDPSNAMES=$(echo "$myDPS" | awk '{ print $1 }' | sort)
     fuGETSYS
-    printf "%-19s %-36s %s\n" "NAME" "STATUS" "PORTS"
-    for i in $myIMAGES; do
-          myNAME="$myMAGENTA$i$myWHITE"
-          printf "%-32s %-49s %s" "$myNAME" "$(fuCONTAINERSTATUS $i)" "$(fuCONTAINERPORTS $i)"
-          echo
-      if [ "$myPARAM" = "vv" ]; 
-        then
-          /usr/bin/docker exec -t "$i" /bin/ps awfuwfxwf | egrep -v -E "awfuwfxwf|/bin/ps" 
-      fi      
+    printf "%-21s %-28s %s\n" "NAME" "STATUS" "PORTS"
+    if [ "$myDPS" != "" ];
+      then
+        echo "$myDPS"
+    fi
+    for i in $myCONTAINERS; do
+      myAVAIL=$(echo "$myDPSNAMES" | grep -o "$i" | uniq | wc -l)      	    
+      if [ "$myAVAIL" = "0" ];
+	then
+	  printf "%-28s %-28s\n" "$myRED$i" "DOWN$myWHITE"
+      fi
     done
     if [[ $myPARAM =~ ^([1-9]|[1-9][0-9]|[1-9][0-9][0-9])$ ]];
       then 

diff --git a/bin/dump_es.sh b/bin/dump_es.sh
@@ -20,7 +20,7 @@ trap fuCLEANUP EXIT
 
 # Set vars
 myDATE=$(date +%Y%m%d%H%M)
-myINDICES=$(curl -s -XGET ''$myES'_cat/indices/' | grep logstash | awk '{ print $3 }' | sort | grep -v 1970)
+myINDICES=$(curl -s -XGET ''$myES'_cat/indices/' | awk '{ print $3 }' | sort | grep -v 1970)
 myES="http://127.0.0.1:64298/"
 myCOL1="[0;34m"
 myCOL0="[0;0m"
@@ -41,5 +41,5 @@ for i in $myINDICES;
 
 # Build tar archive
 echo $myCOL1"### Now building tar archive: es_dump_"$myDATE".tgz" $myCOL0
-tar cvf es_dump_$myDATE.tar tmp/*
+tar cvf es_dump_$myDATE.tar tmp/.
 echo $myCOL1"### Done."$myCOL0
diff --git a/bin/export_kibana-objects.sh b/bin/export_kibana-objects.sh
@@ -1,7 +1,8 @@
 #!/bin/bash
-# Export all Kibana objects
+# Export all Kibana objects through Kibana Saved Objects API
 # Make sure ES is available
 myES="http://127.0.0.1:64298/"
+myKIBANA="http://127.0.0.1:64296/"
 myESSTATUS=$(curl -s -XGET ''$myES'_cluster/health' | jq '.' | grep -c green)
 if ! [ "$myESSTATUS" = "1" ]
   then
@@ -14,10 +15,11 @@ fi
 
 # Set vars
 myDATE=$(date +%Y%m%d%H%M)
-myINDEXCOUNT=$(curl -s -XGET ''$myES'.kibana/index-pattern/logstash-*' | tr '\\' '\n' | grep "scripted" | wc -w)
-myDASHBOARDS=$(curl -s -XGET ''$myES'.kibana/dashboard/_search?filter_path=hits.hits._id&pretty&size=10000'  | jq '.hits.hits[] | {_id}' | jq -r '._id')
-myVISUALIZATIONS=$(curl -s -XGET ''$myES'.kibana/visualization/_search?filter_path=hits.hits._id&pretty&size=10000' | jq '.hits.hits[] | {_id}' | jq -r '._id')
-mySEARCHES=$(curl -s -XGET ''$myES'.kibana/search/_search?filter_path=hits.hits._id&pretty&size=10000' | jq '.hits.hits[] | {_id}' | jq -r '._id')
+myINDEXCOUNT=$(curl -s -XGET ''$myKIBANA'api/saved_objects/_find?type=index-pattern' | jq '.saved_objects[].attributes' | tr '\\' '\n' | grep "scripted" | wc -w)
+myINDEXID=$(curl -s -XGET ''$myKIBANA'api/saved_objects/_find?type=index-pattern' | jq '.saved_objects[].id' | tr -d '"')
+myDASHBOARDS=$(curl -s -XGET ''$myKIBANA'api/saved_objects/_find?type=dashboard&per_page=300' | jq '.saved_objects[].id' | tr -d '"')
+myVISUALIZATIONS=$(curl -s -XGET ''$myKIBANA'api/saved_objects/_find?type=visualization&per_page=300' | jq '.saved_objects[].id' | tr -d '"')
+mySEARCHES=$(curl -s -XGET ''$myKIBANA'api/saved_objects/_find?type=search&per_page=300' | jq '.saved_objects[].id' | tr -d '"')
 myCOL1="[0;34m"
 myCOL0="[0;0m"
 
@@ -29,8 +31,8 @@ trap fuCLEANUP EXIT
 
 # Export index patterns
 mkdir -p patterns
-echo $myCOL1"### Now exporting"$myCOL0 $myINDEXCOUNT $myCOL1"index patterns." $myCOL0
-curl -s -XGET ''$myES'.kibana/index-pattern/logstash-*?' | jq '._source' > patterns/index-patterns.json
+echo $myCOL1"### Now exporting"$myCOL0 $myINDEXCOUNT $myCOL1"index pattern fields." $myCOL0
+curl -s -XGET ''$myKIBANA'api/saved_objects/index-pattern/'$myINDEXID'' | jq '. | {attributes}' > patterns/$myINDEXID.json &
 echo
 
 # Export dashboards
@@ -39,7 +41,7 @@ echo $myCOL1"### Now exporting"$myCOL0 $(echo $myDASHBOARDS | wc -w) $myCOL1"das
 for i in $myDASHBOARDS;
   do
     echo $myCOL1"###### "$i $myCOL0
-    curl -s -XGET ''$myES'.kibana/dashboard/'$i'' | jq '._source' > dashboards/$i.json
+    curl -s -XGET ''$myKIBANA'api/saved_objects/dashboard/'$i'' | jq '. | {attributes}' > dashboards/$i.json &
   done;
 echo
 
@@ -49,7 +51,7 @@ echo $myCOL1"### Now exporting"$myCOL0 $(echo $myVISUALIZATIONS | wc -w) $myCOL1
 for i in $myVISUALIZATIONS;
   do
     echo $myCOL1"###### "$i $myCOL0
-    curl -s -XGET ''$myES'.kibana/visualization/'$i'' | jq '._source' > visualizations/$i.json
+    curl -s -XGET ''$myKIBANA'api/saved_objects/visualization/'$i'' | jq '. | {attributes}' > visualizations/$i.json &
   done;
 echo
 
@@ -59,10 +61,13 @@ echo $myCOL1"### Now exporting"$myCOL0 $(echo $mySEARCHES | wc -w) $myCOL1"searc
 for i in $mySEARCHES;
   do
     echo $myCOL1"###### "$i $myCOL0
-    curl -s -XGET ''$myES'.kibana/search/'$i'' | jq '._source' > searches/$i.json
+    curl -s -XGET ''$myKIBANA'api/saved_objects/search/'$i'' | jq '. | {attributes}' > searches/$i.json &
   done;
 echo
 
+# Wait for background exports to finish
+wait
+
 # Building tar archive
 echo $myCOL1"### Now building archive"$myCOL0 "kibana-objects_"$myDATE".tgz"
 tar cvfz kibana-objects_$myDATE.tgz patterns dashboards visualizations searches > /dev/null