From 5d3210efb2888297825550c7646591fb200654d1 Mon Sep 17 00:00:00 2001 From: meadej Date: Wed, 5 Apr 2023 18:41:44 -0400 Subject: [PATCH 1/5] Pipeline updates and documentation --- .github/workflows/README.md | 21 +++++++++++++++++++++ .github/workflows/dockerbuild.yml | 12 ++++++------ 2 files changed, 27 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/README.md diff --git a/.github/workflows/README.md b/.github/workflows/README.md new file mode 100644 index 0000000..e3d7772 --- /dev/null +++ b/.github/workflows/README.md @@ -0,0 +1,21 @@ +# SkyScan GitHub Workflows + +## Overview + +SkyScan and associated projects use a number of GitHub action workflows to verify code security, secrets integrity, and provide project building and deployment capability. These workflows and their required secrets and configuration variables are detailed below. + +For information on adding configuration variables and secrets, see the GitHub Actions documentation [here](https://docs.github.com/en/actions/learn-github-actions/variables#creating-configuration-variables-for-an-environment) and [here](https://docs.github.com/en/actions/security-guides/encrypted-secrets#creating-encrypted-secrets-for-a-repository). + +## Docker Container Building + +SkyScan uses the action outlined in `dockerbuild.yml` to automatically build Dockerfiles located in the project and push them to Docker Hub. This action is triggered everytime there is a push to the `main` branch. + +### Secrets +* `DOCKER_USERNAME` (required) - The login username for Docker Hub. +* `DOCKER_NAMESPACE` (required) - The Docker Hub account that this image will be pushed to. The `DOCKER_USERNAME` Docker Hub user must have write permissions to `DOCKER_NAMESPACE`. Oftentimes they are the same value. +* `DOCKER_TOKEN` (required) - A Docker Hub personal access token associated with the `DOCKER_USERNAME` account. + +### Configuration Variables +* `PROJECT_NAME` (required) - The overarching name of the project. +* `DOCKER_BUILD_PLATFORMS` (optional, defaults to `linux/amd64`) - A comma separated list of the target platforms the images will be built for. +* `DOCKER_BUILD_FOLDERS` (optional, defaults to `.`) - A comma separated list of folders under the main repository containing Dockerfiles to be built. \ No newline at end of file diff --git a/.github/workflows/dockerbuild.yml b/.github/workflows/dockerbuild.yml index bf17efd..d6af381 100644 --- a/.github/workflows/dockerbuild.yml +++ b/.github/workflows/dockerbuild.yml @@ -39,21 +39,21 @@ jobs: - name: Build Images env: # Every folder in the repo that has a Dockerfile within it, comma separated - DOCKERFOLDERS: "tracker,piaware,pan-tilt-pi,notebook-server,egi,axis-ptz" - PROJECT_NAME: "skyscan" + DOCKERFOLDERS: "${{ vars.DOCKER_BUILD_FOLDERS }}" + PROJECT_NAME: "${{ vars.PROJECT_NAME }}" REPO_NAME: "${{ secrets.DOCKER_NAMESPACE }}" + PLATFORMS: "${{ vars.DOCKER_BUILD_PLATFORMS }}" run: | IFS="," - read -ra ARR <<< "$DOCKERFOLDERS" + read -ra ARR <<< "${DOCKERFOLDERS:-.}" for folder in "${ARR[@]}" do IFS="/" read -ra NAMEFOLDER <<< $folder SUBNAME=${NAMEFOLDER[0]} - echo "Building $folder" - echo $SUBNAME + echo "Building $folder..." docker buildx build "$folder" --push \ --tag $REPO_NAME/$PROJECT_NAME-$SUBNAME:latest \ --tag $REPO_NAME/$PROJECT_NAME-$SUBNAME:${{ steps.get_tag.outputs.IMAGE_TAG }} \ - --platform linux/arm64,linux/amd64 + --platform "${PLATFORMS:-linux/amd64}" done From 756e558312b45f79748fe2bbaf462762859ca7ee Mon Sep 17 00:00:00 2001 From: meadej Date: Tue, 18 Apr 2023 15:31:55 -0400 Subject: [PATCH 2/5] Updates to camera module --- axis-ptz/camera.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/axis-ptz/camera.py b/axis-ptz/camera.py index cb7a982..797a483 100755 --- a/axis-ptz/camera.py +++ b/axis-ptz/camera.py @@ -619,7 +619,7 @@ def moveCamera(ip, username, password, mqtt_client): get_jpeg_request() capture_metadata = get_json_request() mqtt_client.publish( - "skyscan/captures/data", + publish_topic, json.dumps(capture_metadata), 0, False @@ -816,6 +816,7 @@ def main(): global cameraConfig global flight_topic global object_topic + global publish_topic global logging_directory global Active @@ -892,6 +893,12 @@ def main(): action="store_true", help="Verbose output" ) + parser.add_argument( + "--publish-topic", + type=str, + help="The topic to publish capture information to", + default="skyscan/captures/data" + ) parser.add_argument( "-f", "--flat-file-structure", @@ -956,6 +963,7 @@ def main(): flight_topic = args.mqtt_flight_topic object_topic = args.mqtt_object_topic + publish_topic = args.publish_topic print( "connecting to MQTT broker at " + args.mqtt_host From 47f9fd5fd176fb66ab0bf0b7a0b8be0818a9f68a Mon Sep 17 00:00:00 2001 From: meadej Date: Mon, 24 Apr 2023 10:36:30 -0400 Subject: [PATCH 3/5] Deployment additions --- docker-compose.yml | 91 +++++++++++++++++++++++++--------------------- install-skyscan.sh | 36 ++++++++++++++++++ 2 files changed, 85 insertions(+), 42 deletions(-) create mode 100644 install-skyscan.sh diff --git a/docker-compose.yml b/docker-compose.yml index 3e07b43..70d5a96 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,49 +2,62 @@ version: '3.7' services: tracker: - image: iqtlabs/skyscan-tracker - command: "./flighttracker.py -m mqtt -H piaware -l ${LAT} -L ${LONG} -a ${ALT} -P skyscan/planes/json -T skyscan/flight/json -M ${MIN_ELEVATION} -c ${CAMERA_LEAD}" + image: iqtlabs/skyscan-tracker:${SERVICES_VERSION:-latest} + command: "./flighttracker.py -m mqtt -H piaware -l ${LAT} -L ${LONG} -a ${ALT} -P skyscan/planes/json -T skyscan/${HOSTNAME}/flight/tracker/JSON -M ${MIN_ELEVATION} -c ${CAMERA_LEAD}" volumes: - ./data:/data - ports: - - 5000:5000 - depends_on: - - mqtt + env_file: + - .env + deploy: + resources: + limits: + cpus: '1.0' restart: unless-stopped axis-ptz: - image: iqtlabs/skyscan-axis-ptz - command: "./camera.py -m mqtt -t skyscan/flight/json -u ${AXIS_USERNAME} -p ${AXIS_PASSWORD} -a ${AXIS_IP} -z ${CAMERA_ZOOM} -s ${CAMERA_MOVE_SPEED} -d ${CAMERA_DELAY} --lat ${LAT} --lon ${LONG} --alt ${ALT} --roll ${ROLL} --pitch ${PITCH} --yaw ${YAW}" + image: iqtlabs/edgetech-axis-ptz-controller + command: "./camera.py --publish-topic skyscan/${HOSTNAME}/image/edgetech-axis-ptz/JSON -f -m mqtt -t skyscan/${HOSTNAME}/flight/tracker/JSON -u ${AXIS_USERNAME} -p ${AXIS_PASSWORD} -a ${AXIS_IP} -z ${CAMERA_ZOOM} -s ${CAMERA_MOVE_SPEED} -d ${CAMERA_DELAY} --lat ${LAT} --lon ${LONG} --alt ${ALT} --roll ${ROLL} --pitch ${PITCH} --yaw ${YAW}" volumes: - /flash/raw:/app/capture + - /flash/processed/log:/flash/processed/log depends_on: - mqtt - restart: unless-stopped - - piaware: - image: iqtlabs/skyscan-piaware - tty: true - container_name: piaware - devices: - - /dev/bus/usb:/dev/bus/usb - ports: - - 8080:80 - - 30003:30003 - - 30005:30005 - environment: - - TZ=${TZ} - - LAT=${LAT} - - LONG=${LONG} - - FEEDER_ID=${FEEDER_ID} - - RECEIVER_TYPE=rtlsdr - - DUMP1090_DEVICE=${RTL_DEV} + env_file: + - .env + deploy: + resources: + limits: + cpus: '1.0' restart: unless-stopped - notebook: - image: iqtlabs/skyscan-notebook-server - ports: - - "8888:8888" + coral-detect: + image: iqtlabs/skyscan-coral-detect + volumes: + - /flash/raw:/data/tosort + - /flash/coral/plane:/data/plane + - /flash/coral/noplane:/data/noplane + - /flash/coral/log:/data/log + - /flash/weights:/data/weights + deploy: + resources: + limits: + cpus: '1.0' restart: unless-stopped + + edge-detect: + image: iqtlabs/skyscan-yolo-detect + volumes: + - /flash/coral/plane:/data/tosort + - /flash/edge/plane:/data/plane + - /flash/edge/log:/data/log + - /flash/edge/noplane:/data/noplane + - /flash/weights:/data/weights + deploy: + resources: + limits: + cpus: '1.0' + env_file: + - .env depends_on: - mqtt @@ -53,14 +66,8 @@ services: ports: - "9001:9001" - "1883:1883" - restart: unless-stopped - - egi: - image: iqtlabs/skyscan-egi - entrypoint: bash -c "gpsd ${GPS_SERIAL} -F /var/run/gpsd.sock && python3 egi_mqtt.py -m mqtt -l ${LAT} -L ${LONG} -a ${ALT} -r ${ROLL} -p ${PITCH} -y ${YAW}" - devices: - - /dev/ttyACM0 - privileged: true - depends_on: - - mqtt - restart: unless-stopped + deploy: + resources: + limits: + cpus: '1.0' + restart: unless-stopped \ No newline at end of file diff --git a/install-skyscan.sh b/install-skyscan.sh new file mode 100644 index 0000000..a2fe9b7 --- /dev/null +++ b/install-skyscan.sh @@ -0,0 +1,36 @@ +# Set environment variables +export COMPOSE_FILE_URL= +export ENV_FILE_URL= + +echo "Installing skyscan on ${HOSTNAME} at $(pwd)" + +echo "Configuring Mobian base" +bash <(curl -fsSL https://short.iqt.org/pinephonepro) + +mkdir data +cd data +curl -O https://opensky-network.org/datasets/metadata/aircraftDatabase.csv +cd .. + +# Make necessary folders +mkdir /flash +mkdir /flash/raw +mkdir /flash/coral +mkdir /flash/coral/plane +mkdir /flash/coral/noplane +mkdir /flash/coral/log +mkdir /flash/weights +mkdir /flash/edge +mkdir /flash/edge/plane +mkdir /flash/edge/noplane +mkdir /flash/edge/log +mkdir /flash/processed +mkdir /flash/processed/log + +curl -O $COMPOSE_FILE_URL +curl -O $ENV_FILE_URL + +docker-compose pull + +echo "Installation complete. Run `docker-compose up` to start the system" +echo "Before running, ensure you have replaced the demo weights with your own trained weights" \ No newline at end of file From c3a51793f579dce659d510da8b9270d254dee97a Mon Sep 17 00:00:00 2001 From: meadej Date: Mon, 24 Apr 2023 10:53:14 -0400 Subject: [PATCH 4/5] Updates --- .env-example | 54 ++++++++++++++++++++++++++++++++++++++++++++++ install-skyscan.sh | 4 ++-- 2 files changed, 56 insertions(+), 2 deletions(-) create mode 100644 .env-example diff --git a/.env-example b/.env-example new file mode 100644 index 0000000..283f144 --- /dev/null +++ b/.env-example @@ -0,0 +1,54 @@ +HOSTNAME=skyscan + +#Axis-PTZ +TZ="New York" #Local timezone in "TZ database name" format. https://en.wikipedia.org/wiki/List_of_tz_database_time_zones +LAT=38.889444 # Latitude of the camera +LONG=-77.035278 # Longitude of the camera +ALT=200 # Altitude of the camera, this should be in METERS above sea level +ROLL=0 # Roll Angle of Camera Mount from Inertial +PITCH=0 # Pitch Angle of Camera Mount from Inertial +YAW=0 # Yaw Angle of Camera Mount from Inertial +MIN_ELEVATION=0 # The minimum elevation for the camera. +FEEDER_ID=long-api-key-goes-here # Your FlightAware feeder ID (required) +AXIS_USERNAME=root # The username for the Axis camera +AXIS_PASSWORD=password # The Password for the Axis camera +AXIS_IP=192.168.1.220 # The IP address of the camera +CAMERA_MOVE_SPEED=50 # The speed at which the Axis will move for Pan/Tilt (0-100) +CAMERA_DELAY=0.5 # How many seconds after issuing a Pan/Tilt command should a picture be taken +CAMERA_ZOOM=1000 # The zoom setting for the camera (0-9999) +CAMERA_LEAD=0.5 # How many seconds ahead of a plane's predicted location should the camera be positioned +RTL_DEV=1 # The device ID for the RTL-SDR - set using the rtl_eeprom program +GPS_SERIAL=/dev/ttyACM0 # GPS module serial port + +#Edge-detect +WEIGHTS=/data/weights/localizer.pt +AGNOSTIC_NMS=True +NOSAVE=True +CONF_THRES=0.25 +IMG_SIZE=640 +DEVICE=CPU +IOU_THRES=0.45 +SOURCE_DIR=/data/tosort +PLANE_DIR=/data/plane +NOPLANE_DIR=/data/noplane +LOG_DIR=/data/log +VIEW_IMG=False +SAVE_TXT=False +SAVE_JSON=True +SAVE_CONF=False +CLASSES=0 +AUGMENT=False +MQTT_IP=mqtt +MQTT_TOPIC=/skyscan/captures/data/processed +CAMERA_TOPIC=skyscan/${HOSTNAME}/image/edgetech-axis-ptz/JSON +NO_TRACE=False +UPDATE=False +OMP_NUM_THREADS=1 + +# Coral Detect Variables +CORAL_WEIGHTS=/data/weights/localizer.tflite + +# Tracker Variables +MQTT_IP=mqtt +TRACKER_INPUT_TOPIC=skyscan/${HOSTNAME}/ADSB/edgetech-pingusb/JSON +TRACKER_PUBLISH_TOPIC=skyscan/${HOSTNAME}/flight/edgetech-tracker/JSON \ No newline at end of file diff --git a/install-skyscan.sh b/install-skyscan.sh index a2fe9b7..acc68ee 100644 --- a/install-skyscan.sh +++ b/install-skyscan.sh @@ -1,6 +1,6 @@ # Set environment variables -export COMPOSE_FILE_URL= -export ENV_FILE_URL= +export COMPOSE_FILE_URL=https://raw.githubusercontent.com/meadej/SkyScan/main/docker-compose.yml +export ENV_FILE_URL=https://raw.githubusercontent.com/meadej/SkyScan/main/.env-example echo "Installing skyscan on ${HOSTNAME} at $(pwd)" From 714ce84e1024410eedb711410d0aa321562e8017 Mon Sep 17 00:00:00 2001 From: meadej Date: Mon, 24 Apr 2023 13:04:45 -0400 Subject: [PATCH 5/5] Updates --- install-skyscan.sh | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/install-skyscan.sh b/install-skyscan.sh index acc68ee..1c168e1 100644 --- a/install-skyscan.sh +++ b/install-skyscan.sh @@ -7,25 +7,27 @@ echo "Installing skyscan on ${HOSTNAME} at $(pwd)" echo "Configuring Mobian base" bash <(curl -fsSL https://short.iqt.org/pinephonepro) + +mkdir skyscan +cd skyscan mkdir data cd data curl -O https://opensky-network.org/datasets/metadata/aircraftDatabase.csv cd .. # Make necessary folders -mkdir /flash -mkdir /flash/raw -mkdir /flash/coral -mkdir /flash/coral/plane -mkdir /flash/coral/noplane -mkdir /flash/coral/log -mkdir /flash/weights -mkdir /flash/edge -mkdir /flash/edge/plane -mkdir /flash/edge/noplane -mkdir /flash/edge/log -mkdir /flash/processed -mkdir /flash/processed/log +mkdir raw +mkdir coral +mkdir coral/plane +mkdir coral/noplane +mkdir coral/log +mkdir weights +mkdir edge +mkdir edge/plane +mkdir edge/noplane +mkdir edge/log +mkdir processed +mkdir processed/log curl -O $COMPOSE_FILE_URL curl -O $ENV_FILE_URL