Skip to content
This repository was archived by the owner on Feb 24, 2025. It is now read-only.

Commit 2edba55

Browse files
miloswrathmiloswrath
committed
reworking EVERYTHING
1 parent 2cf9244 commit 2edba55

File tree

6 files changed

+336
-58
lines changed

6 files changed

+336
-58
lines changed

‎.github/workflows/main.yml

Lines changed: 232 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -1,102 +1,280 @@
1-
name: Task1 QC
1+
# .github/workflows/deploy.yml
2+
3+
name: Task1 QC and Deploy
4+
25
on:
36
push:
47
branches:
58
- main
69

10+
permissions:
11+
contents: write
12+
packages: write
13+
issues: write
14+
id-token: write
15+
pages: write
16+
717
jobs:
818
process_raw:
19+
name: Process Raw CSV Files
920
runs-on: self-hosted
1021
outputs:
11-
sub: ${{ steps.set_vars.outputs.sub }}
12-
task: ${{ steps.set_vars.outputs.task }}
13-
version: ${{ steps.set_vars.outputs.version }}
22+
sub: ${{ steps.set_vars.outputs.sub }}
23+
task: ${{ steps.set_vars.outputs.task }}
24+
version: ${{ steps.set_vars.outputs.version }}
25+
run_part: ${{ steps.set_vars.outputs.run_part }}
1426

1527
steps:
16-
- name: checkout code and return recently uploaded file in /data
28+
- name: Checkout Code
1729
uses: actions/checkout@v3
18-
- name: Get changed files
30+
31+
- name: Get Changed CSV Files
1932
run: |
20-
#!/bin/bash
33+
#!/bin/bash
34+
35+
# Get the list of CSV files changed in the last 24 hours
36+
data=$(git log --since="24 hours ago" --name-only --pretty=format: -- '*.csv' | sort | uniq)
2137
22-
# Get the list of CSV files changed in the last 24 hours
23-
data=$(git log --since="24 hours ago" --name-only --pretty=format: -- '*.csv' | sort | uniq)
38+
# Export the data variable to the environment
39+
echo "data=$data" >> $GITHUB_ENV
2440
25-
# Export the data variable to the environment
26-
echo "data=$data" >> $GITHUB_ENV
41+
# Print the changed CSV files
42+
echo "Changed CSV files in the last 24 hours: $data"
2743
28-
# Print the changed CSV files
29-
echo "Changed CSV files in the last 24 hours: $data"
3044
31-
- name: set up python
45+
- name: Install Python Dependencies
3246
run: |
3347
python -m pip install --upgrade pip
3448
35-
- name: parse raw
49+
- name: Parse Raw CSV Files
3650
id: set_vars
3751
run: |
38-
# Loop through each CSV file in $data
39-
for file in $data; do
40-
filename=$(basename "$file")
41-
IFS='_' read -r sub task version <<< "$filename"
42-
version="${version%.csv}" # Remove the .csv extension from version
43-
echo "::set-output name=sub::$sub"
44-
echo "::set-output name=task::$task"
45-
echo "::set-output name=version::$version"
46-
echo "Subject: $sub"
47-
echo "Task: $task"
48-
echo "Version: $version"
49-
done
52+
# Loop through each CSV file in $data
53+
for file in $data; do
54+
# Extract the directory and filename
55+
dir=$(dirname "$file")
56+
filename=$(basename "$file")
57+
58+
# Extract the run-* part from the directory
59+
run_part=$(basename "$dir")
60+
61+
# Split the filename into sub, task, and version
62+
IFS='_' read -r sub task version <<< "$filename"
63+
version="${version%.csv}" # Remove the .csv extension from version
64+
65+
# Set outputs
66+
echo "::set-output name=run_part::$run_part"
67+
echo "::set-output name=sub::$sub"
68+
echo "::set-output name=task::$task"
69+
echo "::set-output name=version::$version"
70+
71+
# Print the extracted values
72+
echo "Run Part: $run_part"
73+
echo "Subject: $sub"
74+
echo "Task: $task"
75+
echo "Version: $version"
76+
done
5077
5178
run_qc:
79+
name: Run Quality Control
5280
runs-on: self-hosted
5381
needs: process_raw
82+
5483
steps:
55-
- name: Checkout code
84+
- name: Checkout Code
5685
uses: actions/checkout@v3
5786

58-
- name: Debug env vars
87+
- name: Debug Environment Variables
5988
run: |
60-
echo "sub=${{ needs.process_raw.outputs.sub }}"
61-
echo "task=${{ needs.process_raw.outputs.task }}"
62-
echo "version=${{ needs.process_raw.outputs.version }}"
89+
echo "Subject: ${{ needs.process_raw.outputs.sub }}"
90+
echo "Task: ${{ needs.process_raw.outputs.task }}"
91+
echo "Version: ${{ needs.process_raw.outputs.version }}"
92+
echo "Run Part: ${{ needs.process_raw.outputs.run_part }}"
6393
64-
- name: run quality control
94+
- name: Install Python Dependencies
6595
run: |
66-
sub=${{ needs.process_raw.outputs.sub }}
67-
task=${{ needs.process_raw.outputs.task }}
68-
vers=${{ needs.process_raw.outputs.version }}
69-
for sub in ${sub}; do
96+
python -m pip install --upgrade pip
97+
98+
- name: Run Quality Control Script
99+
run: |
100+
sub=${{ needs.process_raw.outputs.sub }}
101+
task=${{ needs.process_raw.outputs.task }}
102+
vers=${{ needs.process_raw.outputs.version }}
103+
run_part=${{ needs.process_raw.outputs.run_part }}
104+
105+
for sub in ${sub}; do
70106
echo "Processing subject: $sub"
71107
for task in ${task}; do
72-
echo "Processing task: $task"
73-
for vers in ${vers}; do
108+
echo "Processing task: $task"
109+
for vers in ${vers}; do
74110
echo "Processing version: $vers"
75-
csv_file="./data/${sub}/processed/${sub}_${task}_${vers}.csv"
76-
log_file="./data/${sub}/qc_${task}_${vers}.log"
111+
csv_file="./data/${sub}/processed/${run_part}/${sub}_${task}_${vers}.csv"
112+
mkdir -p "./data/${sub}/${run_part}"
113+
log_file="./data/${sub}/${run_part}/qc_${task}_${vers}.log"
77114
echo "CSV file: $csv_file"
78115
echo "Log file: $log_file"
79116
if [ -f "$csv_file" ]; then
80-
python ./code/PCqC.py -s "$csv_file" -o "./data/${sub}/" -sub "$sub" | tee "$log_file"
81-
echo "QC for ${sub}_${task}_${vers} running"
117+
python ./code/PCqC.py -s "$csv_file" -o "./data/${sub}/${run_part}" -sub "$sub" | tee "$log_file"
118+
echo "QC for ${sub}_${task}_${vers} completed"
82119
else
83-
echo "CSV file $csv_file does not exist"
120+
echo "CSV file $csv_file does not exist"
84121
fi
85-
done
86-
done
122+
done
87123
done
124+
done
125+
126+
127+
- name: Commit and Push QC Results
128+
run: |
129+
git config --global user.name "miloswrath"
130+
git config --global user.email "[email protected]"
131+
git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/$GITHUB_REPOSITORY
132+
git add ./data/*/*/*.png
133+
git commit -m "Add QC results for subjects $(date +%Y-%m-%d)"
134+
git push origin main
135+
136+
- name: List Directory After QC
137+
run: |
138+
echo "Listing directory after running QC:"
139+
find ./data -type d
140+
find ./data -type f
141+
142+
143+
144+
add:
145+
name: Generate Jekyll Posts and Deploy
146+
runs-on: ubuntu-latest
147+
needs: [process_raw, run_qc]
88148

89-
push:
90-
runs-on: self-hosted
91-
needs: run_qc
92149
steps:
93-
- name: Commit and Push Changes
150+
# 1. Checkout the Repository
151+
152+
- name: Checkout Repository
153+
uses: actions/checkout@v4
154+
with:
155+
persist-credentials: false # Recommended for security
156+
fetch-depth: 0
157+
ref: main
158+
- name: Pull Latest Changes
159+
run: git pull origin main
160+
161+
# 4. Set Up Ruby Environment
162+
- name: Setup Ruby
163+
uses: ruby/setup-ruby@v1
164+
with:
165+
ruby-version: '3.1' # Specify your Ruby version
166+
bundler-cache: true # Caches installed gems automatically
167+
cache-version: 1 # Increment if you need to reset the cache
168+
169+
# 5. Install Ruby Dependencies
170+
- name: Install Dependencies
171+
run: bundle install
172+
173+
# 6. Generate Jekyll Posts from PNGs
174+
- name: Generate Jekyll Posts
175+
run: |
176+
set -e # Exit immediately if a command exits with a non-zero status
177+
set -x # Print commands and their arguments as they are executed
178+
179+
run_part=${{ needs.process_raw.outputs.run_part }}
180+
POSTS_DIR="_posts"
181+
mkdir -p "$POSTS_DIR" # Ensure the _posts directory exists
182+
183+
# Initialize an associative array to group images by subject
184+
declare -A subjects
185+
186+
# Get the list of PNG files added in the latest commit
187+
images=$(git diff --name-only HEAD~1 HEAD | grep '\.png$')
188+
echo "Images: $images"
189+
190+
# Iterate over each PNG file and group them by subject number
191+
for file in $images; do
192+
# Check if the file exists to avoid errors
193+
if [ ! -f "$file" ]; then
194+
continue
195+
fi
196+
197+
# Extract the subject number from the filename (assuming it's the first part before '_')
198+
filename=$(basename "$file")
199+
subject=$(echo "$filename" | awk -F_ '{print $1}')
200+
201+
# Append the filename to the subject's array
202+
subjects["$subject"]+="$file "
203+
done
204+
205+
# Generate Jekyll posts for each subject
206+
for subject in "${!subjects[@]}"; do
207+
# Define the post filename with current date and subject number
208+
timestamp=$(date +%H%M%S)
209+
post_filename="$POSTS_DIR/$(date +%Y-%m-%d)-subject-$subject-$timestamp.md"
210+
211+
# Create the Jekyll post
212+
{
213+
echo "---"
214+
echo "layout: post"
215+
echo "title: Subject $subject"
216+
echo "date: $(date +%Y-%m-%d)"
217+
echo "categories: subjects"
218+
echo "---"
219+
echo ""
220+
# Add images to the post
221+
for image in ${subjects["$subject"]}; do
222+
# Adjust the image path based on your site structure
223+
# Assuming images are served from the root
224+
echo "![]($image)"
225+
done
226+
} > "$post_filename"
227+
228+
echo "Created post: $post_filename"
229+
done
230+
# 7. List _posts Directory for Verification (Optional)
231+
- name: List _posts Directory
232+
run: |
233+
echo "Listing _posts directory:"
234+
ls -la _posts
235+
236+
# 8. Commit and Push Generated Posts
237+
- name: Commit and Push Posts
94238
run: |
95239
git config --global user.name "miloswrath"
96240
git config --global user.email "[email protected]"
97241
git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/$GITHUB_REPOSITORY
98-
git add .
99-
git commit -m "Automated commit by GitHub Actions"
100-
git push
242+
243+
# Add new posts to git
244+
git add _posts/*.md
245+
246+
# Commit changes if there are any
247+
if ! git diff --cached --exit-code > /dev/null; then
248+
git commit -m "Add new posts for subjects $(date +%Y-%m-%d)"
249+
git push origin main # Replace 'main' with your default branch if different
250+
else
251+
echo "No changes to commit."
252+
fi
253+
env:
254+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
255+
256+
# 9. Build the Jekyll Site
257+
- name: Build with Jekyll
258+
run: bundle exec jekyll build --verbose --baseurl "${{ github.event.inputs.base_path || '' }}"
101259
env:
102-
GITHUB_TOKEN: ${{ secrets.GIT_PAT }}
260+
JEKYLL_ENV: production
261+
262+
# 10. Deploy to GitHub Pages
263+
# Using GitHub's built-in Pages action
264+
- name: Configure GitHub Pages
265+
uses: actions/configure-pages@v5
266+
267+
- name: Upload Pages Artifact
268+
uses: actions/upload-pages-artifact@v1
269+
with:
270+
path: ./_site # Ensure this matches your Jekyll build output
271+
272+
- name: List _site Directory
273+
run: |
274+
echo "Listing _site directory:"
275+
ls -la _site
276+
277+
- name: Deploy to GitHub Pages
278+
uses: actions/deploy-pages@v1
279+
with:
280+
token: ${{ secrets.GITHUB_TOKEN }}

‎Gemfile

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
# Gemfile teehee with no extension teehee
2+
3+
source 'https://rubygems.org'
4+
5+
gem 'github-pages', group: :jekyll_plugins

‎_config.yml

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
title: "Boost Behavioral Data -> PC"
2+
description: "A blog showcasing various subjects post-QC."
3+
baseurl: "/boost-beh-PC" # If your site is at https://username.github.io, leave it empty. If at https://username.github.io/repo-name, set it to "/repo-name"
4+
url: "https://hbclab.github.io" # Replace with your GitHub Pages URL
5+
logo: https://avatars0.githubusercontent.com/u/24659915?s=400&u=12a4f626488fe0f692d77f355d9dd9f3e4e63f7a&v=4
6+
repository: "HBClab/boost-beh-PC" # Replace with your GitHub repository in "owner/re
7+
plugins:
8+
- jekyll-feed
9+
- jekyll-seo-tag
10+
- jekyll-github-metadata
11+
include:
12+
- data
13+
14+
theme: minima

‎_layouts/default.html

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
<!DOCTYPE html>
2+
<html lang="en">
3+
<head>
4+
<meta charset="UTF-8">
5+
<title>{{ page.title }} | {{ site.title }}</title>
6+
<meta name="viewport" content="width=device-width, initial-scale=1">
7+
<!-- Link to external CSS -->
8+
<link rel="stylesheet" href="{{ '/assets/css/style.css' | relative_url }}">
9+
</head>
10+
<body>
11+
{{ content }}
12+
</body>
13+
</html>

0 commit comments

Comments
 (0)