Merge pull request #53 from meadej/main

Pipeline updates (allows configuration variables in place of static values) and documentation
IQTLabs · May 13, 2023 · 47480af · 47480af
2 parents 6dfa866 + 714ce84
commit 47480af
Show file tree

Hide file tree

Showing 6 changed files with 177 additions and 49 deletions.
diff --git a/.env-example b/.env-example
@@ -0,0 +1,54 @@
+HOSTNAME=skyscan
+
+#Axis-PTZ
+TZ="New York" #Local timezone in "TZ database name" format. https://en.wikipedia.org/wiki/List_of_tz_database_time_zones
+LAT=38.889444  # Latitude of the camera
+LONG=-77.035278 # Longitude of the camera
+ALT=200                 # Altitude of the camera, this should be in METERS above sea level
+ROLL=0                  # Roll Angle of Camera Mount from Inertial
+PITCH=0                 # Pitch Angle of Camera Mount from Inertial
+YAW=0                   # Yaw Angle of Camera Mount from Inertial
+MIN_ELEVATION=0        # The minimum elevation for the camera.
+FEEDER_ID=long-api-key-goes-here  # Your FlightAware feeder ID (required)
+AXIS_USERNAME=root          # The username for the Axis camera
+AXIS_PASSWORD=password          # The Password for the Axis camera
+AXIS_IP=192.168.1.220         # The IP address of the camera
+CAMERA_MOVE_SPEED=50        # The speed at which the Axis will move for Pan/Tilt (0-100)
+CAMERA_DELAY=0.5            # How many seconds after issuing a Pan/Tilt command should a picture be taken
+CAMERA_ZOOM=1000            # The zoom setting for the camera (0-9999)
+CAMERA_LEAD=0.5            # How many seconds ahead of a plane's predicted location should the camera be positioned
+RTL_DEV=1                   # The device ID for the RTL-SDR - set using the rtl_eeprom program
+GPS_SERIAL=/dev/ttyACM0     # GPS module serial port
+
+#Edge-detect 
+WEIGHTS=/data/weights/localizer.pt
+AGNOSTIC_NMS=True
+NOSAVE=True
+CONF_THRES=0.25
+IMG_SIZE=640
+DEVICE=CPU
+IOU_THRES=0.45
+SOURCE_DIR=/data/tosort
+PLANE_DIR=/data/plane
+NOPLANE_DIR=/data/noplane
+LOG_DIR=/data/log
+VIEW_IMG=False
+SAVE_TXT=False
+SAVE_JSON=True
+SAVE_CONF=False
+CLASSES=0
+AUGMENT=False
+MQTT_IP=mqtt
+MQTT_TOPIC=/skyscan/captures/data/processed
+CAMERA_TOPIC=skyscan/${HOSTNAME}/image/edgetech-axis-ptz/JSON
+NO_TRACE=False
+UPDATE=False
+OMP_NUM_THREADS=1
+
+# Coral Detect Variables
+CORAL_WEIGHTS=/data/weights/localizer.tflite
+
+# Tracker Variables
+MQTT_IP=mqtt
+TRACKER_INPUT_TOPIC=skyscan/${HOSTNAME}/ADSB/edgetech-pingusb/JSON
+TRACKER_PUBLISH_TOPIC=skyscan/${HOSTNAME}/flight/edgetech-tracker/JSON
diff --git a/.github/workflows/README.md b/.github/workflows/README.md
@@ -0,0 +1,21 @@
+# SkyScan GitHub Workflows
+
+## Overview
+
+SkyScan and associated projects use a number of GitHub action workflows to verify code security, secrets integrity, and provide project building and deployment capability. These workflows and their required secrets and configuration variables are detailed below. 
+
+For information on adding configuration variables and secrets, see the GitHub Actions documentation [here](https://docs.github.com/en/actions/learn-github-actions/variables#creating-configuration-variables-for-an-environment) and [here](https://docs.github.com/en/actions/security-guides/encrypted-secrets#creating-encrypted-secrets-for-a-repository).
+
+## Docker Container Building 
+
+SkyScan uses the action outlined in `dockerbuild.yml` to automatically build Dockerfiles located in the project and push them to Docker Hub. This action is triggered everytime there is a push to the `main` branch. 
+
+### Secrets
+* `DOCKER_USERNAME` (required) - The login username for Docker Hub. 
+* `DOCKER_NAMESPACE` (required) - The Docker Hub account that this image will be pushed to. The `DOCKER_USERNAME` Docker Hub user must have write permissions to `DOCKER_NAMESPACE`. Oftentimes they are the same value. 
+* `DOCKER_TOKEN` (required) - A Docker Hub personal access token associated with the `DOCKER_USERNAME` account. 
+
+### Configuration Variables
+* `PROJECT_NAME` (required) - The overarching name of the project. 
+* `DOCKER_BUILD_PLATFORMS` (optional, defaults to `linux/amd64`) - A comma separated list of the target platforms the images will be built for. 
+* `DOCKER_BUILD_FOLDERS` (optional, defaults to `.`) - A comma separated list of folders under the main repository containing Dockerfiles to be built. 
diff --git a/.github/workflows/dockerbuild.yml b/.github/workflows/dockerbuild.yml
@@ -39,21 +39,21 @@ jobs:
     - name: Build Images
       env: 
         # Every folder in the repo that has a Dockerfile within it, comma separated 
-        DOCKERFOLDERS: "tracker,piaware,pan-tilt-pi,notebook-server,egi,axis-ptz"
-        PROJECT_NAME: "skyscan"
+        DOCKERFOLDERS: "${{ vars.DOCKER_BUILD_FOLDERS }}"
+        PROJECT_NAME: "${{ vars.PROJECT_NAME }}"
         REPO_NAME: "${{ secrets.DOCKER_NAMESPACE }}"
+        PLATFORMS: "${{ vars.DOCKER_BUILD_PLATFORMS }}"
       run: |
         IFS=","
-        read -ra ARR <<< "$DOCKERFOLDERS"
+        read -ra ARR <<< "${DOCKERFOLDERS:-.}"
         for folder in "${ARR[@]}"
         do 
           IFS="/"
           read -ra NAMEFOLDER <<< $folder
           SUBNAME=${NAMEFOLDER[0]}
-          echo "Building $folder"
-          echo $SUBNAME
+          echo "Building $folder..."
           docker buildx build "$folder" --push \
           --tag $REPO_NAME/$PROJECT_NAME-$SUBNAME:latest \
           --tag $REPO_NAME/$PROJECT_NAME-$SUBNAME:${{ steps.get_tag.outputs.IMAGE_TAG }} \
-          --platform linux/arm64,linux/amd64 
+          --platform "${PLATFORMS:-linux/amd64}"
         done
diff --git a/axis-ptz/camera.py b/axis-ptz/camera.py
@@ -619,7 +619,7 @@ def moveCamera(ip, username, password, mqtt_client):
                     get_jpeg_request()
                     capture_metadata = get_json_request()
                     mqtt_client.publish(
-                        "skyscan/captures/data",
+                        publish_topic,
                         json.dumps(capture_metadata),
                         0,
                         False
@@ -816,6 +816,7 @@ def main():
     global cameraConfig
     global flight_topic
     global object_topic
+    global publish_topic
     global logging_directory
     global Active
 
@@ -892,6 +893,12 @@ def main():
         action="store_true", 
         help="Verbose output"
     )
+    parser.add_argument(
+        "--publish-topic",
+        type=str,
+        help="The topic to publish capture information to",
+        default="skyscan/captures/data"
+    )
     parser.add_argument(
         "-f",
         "--flat-file-structure",
@@ -956,6 +963,7 @@ def main():
 
     flight_topic = args.mqtt_flight_topic
     object_topic = args.mqtt_object_topic
+    publish_topic = args.publish_topic
     print(
         "connecting to MQTT broker at "
         + args.mqtt_host

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -2,49 +2,62 @@ version: '3.7'
 
 services:
   tracker:
-    image: iqtlabs/skyscan-tracker
-    command: "./flighttracker.py -m mqtt -H piaware -l ${LAT} -L ${LONG} -a ${ALT} -P skyscan/planes/json -T skyscan/flight/json -M ${MIN_ELEVATION} -c ${CAMERA_LEAD}"
+    image: iqtlabs/skyscan-tracker:${SERVICES_VERSION:-latest}
+    command: "./flighttracker.py -m mqtt -H piaware -l ${LAT} -L ${LONG} -a ${ALT} -P skyscan/planes/json -T skyscan/${HOSTNAME}/flight/tracker/JSON -M ${MIN_ELEVATION} -c ${CAMERA_LEAD}"
     volumes:
       - ./data:/data
-    ports:
-      - 5000:5000
-    depends_on:
-      - mqtt
+    env_file:
+      - .env
+    deploy:
+      resources:
+        limits:
+          cpus: '1.0'
     restart: unless-stopped
 
   axis-ptz:
-    image: iqtlabs/skyscan-axis-ptz
-    command: "./camera.py -m mqtt -t skyscan/flight/json -u ${AXIS_USERNAME} -p ${AXIS_PASSWORD} -a ${AXIS_IP} -z ${CAMERA_ZOOM} -s ${CAMERA_MOVE_SPEED} -d ${CAMERA_DELAY} --lat ${LAT} --lon ${LONG} --alt ${ALT} --roll ${ROLL} --pitch ${PITCH} --yaw ${YAW}"
+    image: iqtlabs/edgetech-axis-ptz-controller
+    command: "./camera.py --publish-topic skyscan/${HOSTNAME}/image/edgetech-axis-ptz/JSON -f -m mqtt -t skyscan/${HOSTNAME}/flight/tracker/JSON -u ${AXIS_USERNAME} -p ${AXIS_PASSWORD} -a ${AXIS_IP} -z ${CAMERA_ZOOM} -s ${CAMERA_MOVE_SPEED} -d ${CAMERA_DELAY} --lat ${LAT} --lon ${LONG} --alt ${ALT} --roll ${ROLL} --pitch ${PITCH} --yaw ${YAW}"
     volumes:
       - /flash/raw:/app/capture
+      - /flash/processed/log:/flash/processed/log
     depends_on:
       - mqtt
-    restart: unless-stopped
-
-  piaware:
-    image: iqtlabs/skyscan-piaware
-    tty: true
-    container_name: piaware
-    devices:
-      - /dev/bus/usb:/dev/bus/usb
-    ports:
-      - 8080:80
-      - 30003:30003
-      - 30005:30005
-    environment:
-      - TZ=${TZ}
-      - LAT=${LAT}
-      - LONG=${LONG}
-      - FEEDER_ID=${FEEDER_ID}
-      - RECEIVER_TYPE=rtlsdr
-      - DUMP1090_DEVICE=${RTL_DEV}
+    env_file:
+      - .env
+    deploy:
+      resources:
+        limits:
+          cpus: '1.0'
     restart: unless-stopped
 
-  notebook:
-    image: iqtlabs/skyscan-notebook-server
-    ports:
-        - "8888:8888"
+  coral-detect:
+    image: iqtlabs/skyscan-coral-detect
+    volumes:
+      - /flash/raw:/data/tosort
+      - /flash/coral/plane:/data/plane
+      - /flash/coral/noplane:/data/noplane
+      - /flash/coral/log:/data/log
+      - /flash/weights:/data/weights
+    deploy:
+      resources:
+        limits:
+          cpus: '1.0'
     restart: unless-stopped
+
+  edge-detect:
+    image: iqtlabs/skyscan-yolo-detect
+    volumes:
+      - /flash/coral/plane:/data/tosort
+      - /flash/edge/plane:/data/plane
+      - /flash/edge/log:/data/log
+      - /flash/edge/noplane:/data/noplane
+      - /flash/weights:/data/weights
+    deploy:
+      resources:
+        limits:
+          cpus: '1.0'
+    env_file:
+      - .env
     depends_on:
       - mqtt
 
@@ -53,14 +66,8 @@ services:
     ports:
         - "9001:9001"
         - "1883:1883"
-    restart: unless-stopped
-
-  egi:
-    image: iqtlabs/skyscan-egi
-    entrypoint: bash -c "gpsd ${GPS_SERIAL} -F /var/run/gpsd.sock && python3 egi_mqtt.py -m mqtt -l ${LAT} -L ${LONG} -a ${ALT} -r ${ROLL} -p ${PITCH} -y ${YAW}"
-    devices:
-      - /dev/ttyACM0
-    privileged: true
-    depends_on:
-      - mqtt
-    restart: unless-stopped
+    deploy:
+      resources:
+        limits:
+          cpus: '1.0'
+    restart: unless-stopped
diff --git a/install-skyscan.sh b/install-skyscan.sh
@@ -0,0 +1,38 @@
+# Set environment variables
+export COMPOSE_FILE_URL=https://raw.githubusercontent.com/meadej/SkyScan/main/docker-compose.yml
+export ENV_FILE_URL=https://raw.githubusercontent.com/meadej/SkyScan/main/.env-example
+
+echo "Installing skyscan on ${HOSTNAME} at $(pwd)"
+
+echo "Configuring Mobian base"
+bash <(curl -fsSL https://short.iqt.org/pinephonepro)
+
+
+mkdir skyscan
+cd skyscan
+mkdir data
+cd data
+curl -O https://opensky-network.org/datasets/metadata/aircraftDatabase.csv
+cd ..
+
+# Make necessary folders
+mkdir raw
+mkdir coral
+mkdir coral/plane
+mkdir coral/noplane
+mkdir coral/log
+mkdir weights
+mkdir edge
+mkdir edge/plane
+mkdir edge/noplane
+mkdir edge/log
+mkdir processed
+mkdir processed/log
+
+curl -O $COMPOSE_FILE_URL
+curl -O $ENV_FILE_URL
+
+docker-compose pull
+
+echo "Installation complete. Run `docker-compose up` to start the system"
+echo "Before running, ensure you have replaced the demo weights with your own trained weights"