This repository has been archived by the owner on Feb 24, 2025. It is now read-only.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # .github/workflows/deploy.yml | |
| name: Task1 QC and Deploy | |
| on: | |
| push: | |
| branches: | |
| - main | |
| permissions: | |
| contents: write | |
| packages: write | |
| issues: write | |
| id-token: write | |
| pages: write | |
| jobs: | |
| process_raw: | |
| name: Process Raw CSV Files | |
| runs-on: self-hosted | |
| outputs: | |
| sub: ${{ steps.set_vars.outputs.sub }} | |
| task: ${{ steps.set_vars.outputs.task }} | |
| version: ${{ steps.set_vars.outputs.version }} | |
| run_part: ${{ steps.set_vars.outputs.run_part }} | |
| steps: | |
| - name: Checkout Code | |
| uses: actions/checkout@v3 | |
| - name: Get Changed CSV Files | |
| run: | | |
| #!/bin/bash | |
| # Get the list of CSV files changed in the last 24 hours | |
| data=$(git log --since="24 hours ago" --name-only --pretty=format: -- '*.csv' | sort | uniq) | |
| # Export the data variable to the environment | |
| echo "data=$data" >> $GITHUB_ENV | |
| # Print the changed CSV files | |
| echo "Changed CSV files in the last 24 hours: $data" | |
| - name: Install Python Dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| - name: Parse Raw CSV Files | |
| id: set_vars | |
| run: | | |
| # Loop through each CSV file in $data | |
| for file in $data; do | |
| # Extract the directory and filename | |
| dir=$(dirname "$file") | |
| filename=$(basename "$file") | |
| # Extract the run-* part from the directory | |
| run_part=$(basename "$dir") | |
| # Split the filename into sub, task, and version | |
| IFS='_' read -r sub task version <<< "$filename" | |
| version="${version%.csv}" # Remove the .csv extension from version | |
| # Set outputs | |
| echo "::set-output name=run_part::$run_part" | |
| echo "::set-output name=sub::$sub" | |
| echo "::set-output name=task::$task" | |
| echo "::set-output name=version::$version" | |
| # Print the extracted values | |
| echo "Run Part: $run_part" | |
| echo "Subject: $sub" | |
| echo "Task: $task" | |
| echo "Version: $version" | |
| done | |
| run_qc: | |
| name: Run Quality Control | |
| runs-on: self-hosted | |
| needs: process_raw | |
| steps: | |
| - name: Checkout Code | |
| uses: actions/checkout@v3 | |
| - name: Debug Environment Variables | |
| run: | | |
| echo "Subject: ${{ needs.process_raw.outputs.sub }}" | |
| echo "Task: ${{ needs.process_raw.outputs.task }}" | |
| echo "Version: ${{ needs.process_raw.outputs.version }}" | |
| echo "Run Part: ${{ needs.process_raw.outputs.run_part }}" | |
| - name: Install Python Dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| - name: Run Quality Control Script | |
| run: | | |
| sub=${{ needs.process_raw.outputs.sub }} | |
| task=${{ needs.process_raw.outputs.task }} | |
| vers=${{ needs.process_raw.outputs.version }} | |
| run_part=${{ needs.process_raw.outputs.run_part }} | |
| for sub in ${sub}; do | |
| echo "Processing subject: $sub" | |
| for task in ${task}; do | |
| echo "Processing task: $task" | |
| for vers in ${vers}; do | |
| echo "Processing version: $vers" | |
| csv_file="./data/${sub}/processed/${run_part}/${sub}_${task}_${vers}.csv" | |
| mkdir -p "./data/${sub}/${run_part}" | |
| log_file="./data/${sub}/${run_part}/qc_${task}_${vers}.log" | |
| echo "CSV file: $csv_file" | |
| echo "Log file: $log_file" | |
| if [ -f "$csv_file" ]; then | |
| python ./code/AFqC.py -s "$csv_file" -o "./data/${sub}/${run_part}" -sub "$sub" | tee "$log_file" | |
| echo "QC for ${sub}_${task}_${vers} completed" | |
| else | |
| echo "CSV file $csv_file does not exist" | |
| fi | |
| done | |
| done | |
| done | |
| - name: List Directory After QC | |
| run: | | |
| echo "Listing directory after running QC:" | |
| find ./data -type d | |
| find ./data -type f | |
| - name: List Generated PNGs | |
| id: list_pngs | |
| run: | | |
| # Find PNG files and replace newlines with spaces | |
| png_files=$(find ./data/*/*/*.png | tr '\n' ' ' | tr -d '\r' | sed 's/ *$//') | |
| echo "Found PNG files:" | |
| echo "$png_files" | |
| # Save the list to an environment variable and output | |
| echo "PNG_FILES=$png_files" >> $GITHUB_ENV | |
| echo "png_files=$png_files" >> $GITHUB_OUTPUT | |
| - name: Verify Files Exist Before Upload | |
| run: | | |
| for file in ${{ steps.list_pngs.outputs.png_files }}; do | |
| if [ -f "$file" ]; then | |
| echo "$file exists." | |
| else | |
| echo "$file does not exist!" | |
| exit 1 | |
| fi | |
| done | |
| - name: Upload Generated PNGs as Artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: generated-pngs | |
| path: ~/data/*/*/*.png | |
| include-hidden-files: true | |
| retention-days: 1 | |
| add: | |
| name: Generate Jekyll Posts and Deploy | |
| runs-on: ubuntu-latest | |
| needs: run_qc | |
| steps: | |
| # 1. Checkout the Repository | |
| - name: Checkout Repository | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false # Recommended for security | |
| fetch-depth: 0 # Ensure full history is fetched for git commands | |
| # 2. Download PNG Artifacts from QC Job | |
| - name: Download PNG Artifacts | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: generated-pngs | |
| path: ./downloaded-pngs # Directory where PNGs will be downloaded | |
| # 3. List Downloaded PNGs for Verification (Optional) | |
| - name: List Downloaded PNGs | |
| run: | | |
| echo "Listing downloaded PNGs:" | |
| find ./downloaded-pngs -type f -name "*.png" | |
| # 4. Set Up Ruby Environment | |
| - name: Setup Ruby | |
| uses: ruby/setup-ruby@v1 | |
| with: | |
| ruby-version: '3.1' # Specify your Ruby version | |
| bundler-cache: true # Caches installed gems automatically | |
| cache-version: 1 # Increment if you need to reset the cache | |
| # 5. Install Ruby Dependencies | |
| - name: Install Dependencies | |
| run: bundle install | |
| # 6. Generate Jekyll Posts from PNGs | |
| - name: Generate Jekyll Posts | |
| run: | | |
| POSTS_DIR="_posts" | |
| mkdir -p "$POSTS_DIR" # Ensure the _posts directory exists | |
| # Initialize an associative array to group images by subject | |
| declare -A subjects | |
| # Iterate over each PNG file and group them by subject number | |
| for file in ./downloaded-pngs/*/processed/*/*.png; do | |
| # Check if the file exists to avoid errors | |
| if [ ! -f "$file" ]; then | |
| continue | |
| fi | |
| # Extract the subject number from the filename (assuming it's the first part before '_') | |
| filename=$(basename "$file") | |
| subject=$(echo "$filename" | awk -F_ '{print $1}') | |
| # Append the filename to the subject's array | |
| subjects["$subject"]+="$file " | |
| done | |
| # Generate Jekyll posts for each subject | |
| for subject in "${!subjects[@]}"; do | |
| # Define the post filename with current date and subject number | |
| timestamp=$(date +%H%M%S) | |
| post_filename="$POSTS_DIR/$(date +%Y-%m-%d)-subject-$subject-$timestamp.md" | |
| # Create the Jekyll post | |
| { | |
| echo "---" | |
| echo "layout: post" | |
| echo "title: Subject $subject" | |
| echo "date: $(date +%Y-%m-%d)" | |
| echo "categories: subjects" | |
| echo "---" | |
| echo "" | |
| # Add images to the post | |
| for image in ${subjects["$subject"]}; do | |
| # Adjust the image path based on your site structure | |
| # Assuming images are served from the root, adjust if necessary | |
| image_relative_path=$(realpath --relative-to=./ _site $image) | |
| echo "))/$(basename "$image"))" | |
| done | |
| } > "$post_filename" | |
| echo "Created post: $post_filename" | |
| done | |
| # 7. List _posts Directory for Verification (Optional) | |
| - name: List _posts Directory | |
| run: | | |
| echo "Listing _posts directory:" | |
| ls -la _posts | |
| # 8. Commit and Push Generated Posts | |
| - name: Commit and Push Posts | |
| if: ${{ steps.Download_PNG_Artifacts.outputs.artifact == 'generated-pngs' }} && success() && success() | |
| run: | | |
| git config user.name "github-actions[bot]" | |
| git config user.email "github-actions[bot]@users.noreply.github.com" | |
| # Add new posts to git | |
| git add _posts/*.md | |
| # Commit changes if there are any | |
| if ! git diff --cached --exit-code > /dev/null; then | |
| git commit -m "Add new posts for subjects $(date +%Y-%m-%d)" | |
| git push origin main # Replace 'main' with your default branch if different | |
| else | |
| echo "No changes to commit." | |
| fi | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| # 9. Build the Jekyll Site | |
| - name: Build with Jekyll | |
| run: bundle exec jekyll build --verbose --baseurl "${{ github.event.inputs.base_path || '' }}" | |
| env: | |
| JEKYLL_ENV: production | |
| # 10. Deploy to GitHub Pages | |
| # Using GitHub's built-in Pages action | |
| - name: Configure GitHub Pages | |
| uses: actions/configure-pages@v5 | |
| - name: Upload Pages Artifact | |
| uses: actions/upload-pages-artifact@v1 | |
| with: | |
| path: ./_site # Ensure this matches your Jekyll build output | |
| - name: Deploy to GitHub Pages | |
| uses: actions/deploy-pages@v1 | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| push: | |
| name: Commit and Push Changes | |
| runs-on: self-hosted | |
| needs: add | |
| steps: | |
| - name: Checkout Repository | |
| uses: actions/checkout@v3 | |
| - name: Commit and Push Changes | |
| run: | | |
| git config --global user.name "miloswrath" | |
| git config --global user.email "[email protected]" | |
| git remote set-url origin https://x-access-token:${{ secrets.GIT_TOKEN }}@github.com/$GITHUB_REPOSITORY | |
| git add . | |
| git commit -m "Automated commit by GitHub Actions" || echo "No changes to commit." | |
| git push | |
| env: | |
| GIT_TOKEN: ${{ secrets.GIT_TOKEN }} |