main.yml

# .github/workflows/deploy.yml

name: Task1 QC and Deploy

on:
  push:
    branches:
      - main

permissions:
  contents: write
  packages: write
  issues: write
  id-token: write
  pages: write

jobs:
  process_raw:
    name: Process Raw CSV Files
    runs-on: self-hosted
    outputs:
      sub: ${{ steps.set_vars.outputs.sub }}
      task: ${{ steps.set_vars.outputs.task }}
      version: ${{ steps.set_vars.outputs.version }}
      run_part: ${{ steps.set_vars.outputs.run_part }}

    steps:
      - name: Checkout Code
        uses: actions/checkout@v3

      - name: Get Changed CSV Files
        run: |
          #!/bin/bash

          # Get the list of CSV files changed in the last 24 hours
          data=$(git log --since="24 hours ago" --name-only --pretty=format: -- '*.csv' | sort | uniq)

          # Convert the list into a multi-line format
          formatted_data=$(echo "$data" | tr ' ' '\n')

          # Export the formatted data variable to the environment
          echo "data<<EOF" >> $GITHUB_ENV
          echo "$formatted_data" >> $GITHUB_ENV
          echo "EOF" >> $GITHUB_ENV

          # Print the changed CSV files
          echo "Changed CSV files in the last 24 hours:"
          echo "$formatted_data"


      - name: Install Python Dependencies
        run: |
          python -m pip install --upgrade pip

      - name: Parse Raw CSV Files
        id: set_vars
        run: |
          # Loop through each CSV file in $data
          for file in $data; do
              # Extract the directory and filename
              dir=$(dirname "$file")
              filename=$(basename "$file")

              # Extract the run-* part from the directory
              run_part=$(basename "$dir")

              # Split the filename into sub, task, and version
              IFS='_' read -r sub task version <<< "$filename"
              version="${version%.csv}"  # Remove the .csv extension from version

              # Set outputs
              echo "::set-output name=run_part::$run_part"
              echo "::set-output name=sub::$sub"
              echo "::set-output name=task::$task"
              echo "::set-output name=version::$version"

              # Print the extracted values
              echo "Run Part: $run_part"
              echo "Subject: $sub"
              echo "Task: $task"
              echo "Version: $version"
          done

  run_qc:
    name: Run Quality Control
    runs-on: self-hosted
    needs: process_raw

    steps:
      - name: Checkout Code
        uses: actions/checkout@v3

      - name: Debug Environment Variables
        run: |
          echo "Subject: ${{ needs.process_raw.outputs.sub }}"
          echo "Task: ${{ needs.process_raw.outputs.task }}"
          echo "Version: ${{ needs.process_raw.outputs.version }}"
          echo "Run Part: ${{ needs.process_raw.outputs.run_part }}"

      - name: Install Python Dependencies
        run: |
          python -m pip install --upgrade pip

      - name: Run Quality Control Script
        env:
          TEASE: ${{ secrets.TEASE }}
        run: |
          sub=${{ needs.process_raw.outputs.sub }}
          task=${{ needs.process_raw.outputs.task }}
          vers=${{ needs.process_raw.outputs.version }}
          run_part=${{ needs.process_raw.outputs.run_part }}

          for sub in ${sub}; do
            echo "Processing subject: $sub"
            for task in ${task}; do
              echo "Processing task: $task"
              for vers in ${vers}; do
                echo "Processing version: $vers"
                csv_file="./data/${sub}/processed/${run_part}/${sub}_${task}_${vers}.csv"
                mkdir -p "./data/${sub}/${run_part}"
                log_file="./data/${sub}/${run_part}/qc_${task}_${vers}.log"
                echo "CSV file: $csv_file"
                echo "Log file: $log_file"
                if [ -f "$csv_file" ]; then
                  python ./code/NTSqC.py -s "$csv_file" -o "./data/${sub}/${run_part}" -sub "$sub" | tee "$log_file"
                  echo "QC for ${sub}_${task}_${vers} completed"
                else
                  echo "CSV file $csv_file does not exist"
                fi
              done
            done
          done


      - name: Commit and Push QC Results
        run: |
          git config --global user.name "miloswrath"
          git config --global user.email "miloswrath@users.noreply.github.com"
          git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/$GITHUB_REPOSITORY
          git add ./data/*/*/*.png
          git commit -m "Add QC results for subjects $(date +%Y-%m-%d)"
          git push origin main  

      - name: List Directory After QC
        run: |
          echo "Listing directory after running QC:"
          find ./data -type d
          find ./data -type f



  add:
    name: Generate Jekyll Posts and Deploy
    runs-on: ubuntu-latest
    needs: [process_raw, run_qc]

    steps:
      # 1. Checkout the Repository

      - name: Checkout Repository
        uses: actions/checkout@v4
        with:
          persist-credentials: false  # Recommended for security
          fetch-depth: 0
          ref: main
      - name: Pull Latest Changes
        run: git pull origin main

      # 4. Set Up Ruby Environment
      - name: Setup Ruby
        uses: ruby/setup-ruby@v1
        with:
          ruby-version: '3.1'  # Specify your Ruby version
          bundler-cache: true  # Caches installed gems automatically
          cache-version: 1  # Increment if you need to reset the cache

      # 5. Install Ruby Dependencies
      - name: Install Dependencies
        run: bundle install

      # 6. Generate Jekyll Posts from PNGs
      - name: Generate Jekyll Posts
        run: |
          set -e  # Exit immediately if a command exits with a non-zero status
          set -x  # Print commands and their arguments as they are executed
      
          run_part=${{ needs.process_raw.outputs.run_part }}
          POSTS_DIR="_posts"
          mkdir -p "$POSTS_DIR"  # Ensure the _posts directory exists
      
          # Initialize an associative array to group images by subject
          declare -A subjects
      
          # Get the list of PNG files added in the latest commit
          images=$(git diff --name-only HEAD~1 HEAD | grep '\.png$')
          echo "Images: $images"
      
          # Iterate over each PNG file and group them by subject number
          for file in $images; do
              # Check if the file exists to avoid errors
              if [ ! -f "$file" ]; then
                  continue
              fi
      
              # Extract the subject number from the filename (assuming it's the first part before '_')
              filename=$(basename "$file")
              subject=$(echo "$filename" | awk -F_ '{print $1}')
      
              # Append the filename to the subject's array
              subjects["$subject"]+="$file "
          done
      
          # Generate Jekyll posts for each subject
          for subject in "${!subjects[@]}"; do
              # Define the post filename with current date and subject number
              timestamp=$(date +%H%M%S)
              post_filename="$POSTS_DIR/$(date +%Y-%m-%d)-subject-$subject-$timestamp.md"
      
              # Create the Jekyll post
              {
                  echo "---"
                  echo "layout: post"
                  echo "title: Subject $subject"
                  echo "date: $(date +%Y-%m-%d)"
                  echo "categories: subjects"
                  echo "---"
                  echo ""
                  # Add images to the post
                  for image in ${subjects["$subject"]}; do
                      # Adjust the image path based on your site structure
                      # Assuming images are served from the root
                      echo "![]($image)"
                  done
              } > "$post_filename"
      
              echo "Created post: $post_filename"
          done
      # 7. List _posts Directory for Verification (Optional)
      - name: List _posts Directory
        run: |
          echo "Listing _posts directory:"
          ls -la _posts

      # 8. Commit and Push Generated Posts
      - name: Commit and Push Posts
        run: |
          git config --global user.name "miloswrath"
          git config --global user.email "miloswrath@users.noreply.github.com"
          git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/$GITHUB_REPOSITORY
      
          # Add new posts to git
          git add _posts/*.md
      
          # Commit changes if there are any
          if ! git diff --cached --exit-code > /dev/null; then
            git commit -m "Add new posts for subjects $(date +%Y-%m-%d)"
            git push origin main  # Replace 'main' with your default branch if different
          else
            echo "No changes to commit."
          fi
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      # 9. Build the Jekyll Site
      - name: Build with Jekyll
        run: bundle exec jekyll build --verbose --baseurl "${{ github.event.inputs.base_path || '' }}"
        env:
          JEKYLL_ENV: production

      # 10. Deploy to GitHub Pages
      # Using GitHub's built-in Pages action
      - name: Configure GitHub Pages
        uses: actions/configure-pages@v5

      - name: Upload Pages Artifact
        uses: actions/upload-pages-artifact@v1
        with:
          path: ./_site  # Ensure this matches your Jekyll build output

      - name: List _site Directory
        run: |
          echo "Listing _site directory:"
          ls -la _site

      - name: Deploy to GitHub Pages
        uses: actions/deploy-pages@v1
        with:
          token: ${{ secrets.GITHUB_TOKEN }}