main.yml

# .github/workflows/deploy.yml

name: Task1 QC and Deploy

on:
  push:
    branches:
      - main

permissions:
  contents: write
  packages: write
  issues: write
  id-token: write
  pages: write

jobs:
  process_raw:
    name: Process Raw CSV Files
    runs-on: self-hosted
    outputs:
      sub: ${{ steps.set_vars.outputs.sub }}
      task: ${{ steps.set_vars.outputs.task }}
      version: ${{ steps.set_vars.outputs.version }}
      run_part: ${{ steps.set_vars.outputs.run_part }}

    steps:
      - name: Checkout Code
        uses: actions/checkout@v3

      - name: Get Changed CSV Files
        run: |
          #!/bin/bash

          # Get the list of CSV files changed in the last 24 hours
          data=$(git log --since="24 hours ago" --name-only --pretty=format: -- '*.csv' | sort | uniq)

          # Export the data variable to the environment
          echo "data=$data" >> $GITHUB_ENV

          # Print the changed CSV files
          echo "Changed CSV files in the last 24 hours: $data"


      - name: Install Python Dependencies
        run: |
          python -m pip install --upgrade pip

      - name: Parse Raw CSV Files
        id: set_vars
        run: |
          # Loop through each CSV file in $data
          for file in $data; do
              # Extract the directory and filename
              dir=$(dirname "$file")
              filename=$(basename "$file")

              # Extract the run-* part from the directory
              run_part=$(basename "$dir")

              # Split the filename into sub, task, and version
              IFS='_' read -r sub task version <<< "$filename"
              version="${version%.csv}"  # Remove the .csv extension from version

              # Set outputs
              echo "::set-output name=run_part::$run_part"
              echo "::set-output name=sub::$sub"
              echo "::set-output name=task::$task"
              echo "::set-output name=version::$version"

              # Print the extracted values
              echo "Run Part: $run_part"
              echo "Subject: $sub"
              echo "Task: $task"
              echo "Version: $version"
          done

  run_qc:
    name: Run Quality Control
    runs-on: self-hosted
    needs: process_raw
    steps:
      - name: Checkout Code
        uses: actions/checkout@v3

      - name: Debug Environment Variables
        run: |
          echo "Subject: ${{ needs.process_raw.outputs.sub }}"
          echo "Task: ${{ needs.process_raw.outputs.task }}"
          echo "Version: ${{ needs.process_raw.outputs.version }}"
          echo "Run Part: ${{ needs.process_raw.outputs.run_part }}"

      - name: Install Python Dependencies
        run: |
          python -m pip install --upgrade pip

      - name: Run Quality Control Script
        run: |
          sub=${{ needs.process_raw.outputs.sub }}
          task=${{ needs.process_raw.outputs.task }}
          vers=${{ needs.process_raw.outputs.version }}
          run_part=${{ needs.process_raw.outputs.run_part }}

          for sub in ${sub}; do
            echo "Processing subject: $sub"
            for task in ${task}; do
              echo "Processing task: $task"
              for vers in ${vers}; do
                echo "Processing version: $vers"
                csv_file="./data/${sub}/processed/${run_part}/${sub}_${task}_${vers}.csv"
                mkdir -p "./data/${sub}/${run_part}"
                log_file="./data/${sub}/${run_part}/qc_${task}_${vers}.log"
                echo "CSV file: $csv_file"
                echo "Log file: $log_file"
                if [ -f "$csv_file" ]; then
                  python ./code/AFqC.py -s "$csv_file" -o "./data/${sub}/${run_part}" -sub "$sub" | tee "$log_file"
                  echo "QC for ${sub}_${task}_${vers} completed"
                else
                  echo "CSV file $csv_file does not exist"
                fi
              done
            done
          done


      - name: List Directory After QC
        run: |
          echo "Listing directory after running QC:"
          find ./data -type d
          find ./data -type f

      - name: List Generated PNGs
        id: list_pngs
        run: |
          # Find PNG files and replace newlines with spaces
          png_files=$(find ./data/*/*/*.png | tr '\n' ' ' | tr -d '\r' | sed 's/ *$//')
          echo "Found PNG files:"
          echo "$png_files"
          # Save the list to an environment variable and output
          echo "PNG_FILES=$png_files" >> $GITHUB_ENV
          echo "png_files=$png_files" >> $GITHUB_OUTPUT

      - name: Verify Files Exist Before Upload
        run: |
          for file in ${{ steps.list_pngs.outputs.png_files }}; do
            if [ -f "$file" ]; then
              echo "$file exists."
            else
              echo "$file does not exist!"
              exit 1
            fi
          done
        
      - name: Upload Generated PNGs as Artifacts
        uses: actions/upload-artifact@v3
        with:
          name: generated-pngs
          path: ${{ steps.list_pngs.outputs.PNG_FILES }}
          retention-days: 7

  add:
    name: Generate Jekyll Posts and Deploy
    runs-on: ubuntu-latest
    needs: run_qc

    steps:
      # 1. Checkout the Repository
      - name: Checkout Repository
        uses: actions/checkout@v4
        with:
          persist-credentials: false  # Recommended for security
          fetch-depth: 0  # Ensure full history is fetched for git commands

      # 2. Download PNG Artifacts from QC Job
      - name: Download PNG Artifacts
        uses: actions/download-artifact@v3
        with:
          name: generated-pngs
          path: ./downloaded-pngs  # Directory where PNGs will be downloaded

      # 3. List Downloaded PNGs for Verification (Optional)
      - name: List Downloaded PNGs
        run: |
          echo "Listing downloaded PNGs:"
          find ./downloaded-pngs -type f -name "*.png"

      # 4. Set Up Ruby Environment
      - name: Setup Ruby
        uses: ruby/setup-ruby@v1
        with:
          ruby-version: '3.1'  # Specify your Ruby version
          bundler-cache: true  # Caches installed gems automatically
          cache-version: 1  # Increment if you need to reset the cache

      # 5. Install Ruby Dependencies
      - name: Install Dependencies
        run: bundle install

      # 6. Generate Jekyll Posts from PNGs
      - name: Generate Jekyll Posts
        run: |
          POSTS_DIR="_posts"
          mkdir -p "$POSTS_DIR"  # Ensure the _posts directory exists

          # Initialize an associative array to group images by subject
          declare -A subjects

          # Iterate over each PNG file and group them by subject number
          for file in ./downloaded-pngs/*/processed/*/*.png; do
              # Check if the file exists to avoid errors
              if [ ! -f "$file" ]; then
                  continue
              fi

              # Extract the subject number from the filename (assuming it's the first part before '_')
              filename=$(basename "$file")
              subject=$(echo "$filename" | awk -F_ '{print $1}')

              # Append the filename to the subject's array
              subjects["$subject"]+="$file "
          done

          # Generate Jekyll posts for each subject
          for subject in "${!subjects[@]}"; do
              # Define the post filename with current date and subject number
              timestamp=$(date +%H%M%S)
              post_filename="$POSTS_DIR/$(date +%Y-%m-%d)-subject-$subject-$timestamp.md"

              # Create the Jekyll post
              {
                  echo "---"
                  echo "layout: post"
                  echo "title: Subject $subject"
                  echo "date: $(date +%Y-%m-%d)"
                  echo "categories: subjects"
                  echo "---"
                  echo ""
                  # Add images to the post
                  for image in ${subjects["$subject"]}; do
                      # Adjust the image path based on your site structure
                      # Assuming images are served from the root, adjust if necessary
                      image_relative_path=$(realpath --relative-to=./ _site $image)
                      echo "![](/downloaded-pngs/${subject}/processed/$(basename $(dirname $(dirname $image)))/$(basename "$image"))"
                  done
              } > "$post_filename"

              echo "Created post: $post_filename"
          done

      # 7. List _posts Directory for Verification (Optional)
      - name: List _posts Directory
        run: |
          echo "Listing _posts directory:"
          ls -la _posts

      # 8. Commit and Push Generated Posts
      - name: Commit and Push Posts
        if: ${{ steps.Download_PNG_Artifacts.outputs.artifact == 'generated-pngs' }} && success() && success()
        run: |
          git config user.name "github-actions[bot]"
          git config user.email "github-actions[bot]@users.noreply.github.com"

          # Add new posts to git
          git add _posts/*.md

          # Commit changes if there are any
          if ! git diff --cached --exit-code > /dev/null; then
            git commit -m "Add new posts for subjects $(date +%Y-%m-%d)"
            git push origin main  # Replace 'main' with your default branch if different
          else
            echo "No changes to commit."
          fi

        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      # 9. Build the Jekyll Site
      - name: Build with Jekyll
        run: bundle exec jekyll build --verbose --baseurl "${{ github.event.inputs.base_path || '' }}"
        env:
          JEKYLL_ENV: production

      # 10. Deploy to GitHub Pages
      # Using GitHub's built-in Pages action
      - name: Configure GitHub Pages
        uses: actions/configure-pages@v5

      - name: Upload Pages Artifact
        uses: actions/upload-pages-artifact@v1
        with:
          path: ./_site  # Ensure this matches your Jekyll build output

      - name: Deploy to GitHub Pages
        uses: actions/deploy-pages@v1
        with:
          token: ${{ secrets.GITHUB_TOKEN }}

  push:
    name: Commit and Push Changes
    runs-on: self-hosted
    needs: add
    steps:
      - name: Checkout Repository
        uses: actions/checkout@v3

      - name: Commit and Push Changes
        run: |
          git config --global user.name "miloswrath"
          git config --global user.email "miloswrath@users.noreply.github.com"
          git remote set-url origin https://x-access-token:${{ secrets.GIT_TOKEN }}@github.com/$GITHUB_REPOSITORY
          git add .
          git commit -m "Automated commit by GitHub Actions" || echo "No changes to commit."
          git push
        env:
          GIT_TOKEN: ${{ secrets.GIT_TOKEN }}