main.yml

name: Task1 QC
on:
  push:
    branches:
      - main

jobs:
  process_raw:
    runs-on: self-hosted
    outputs:
        sub: ${{ steps.set_vars.outputs.sub }}
        task: ${{ steps.set_vars.outputs.task }}
        version: ${{ steps.set_vars.outputs.version }}

    steps:
      - name: checkout code and return recently uploaded file in /data
        uses: actions/checkout@v3
      - name: Get changed files
        run: |
            #!/bin/bash

            # Get the list of CSV files changed in the last 24 hours
            data=$(git log --since="24 hours ago" --name-only --pretty=format: -- '*.csv' | sort | uniq)

            # Export the data variable to the environment
            echo "data=$data" >> $GITHUB_ENV

            # Print the changed CSV files
            echo "Changed CSV files in the last 24 hours: $data"

      - name: set up python
        run: |
          python -m pip install --upgrade pip
          

      - name: parse raw
        id: set_vars
        run: |
            # Loop through each CSV file in $data
            for file in $data; do
                filename=$(basename "$file")
                IFS='_' read -r sub task version <<< "$filename"
                version="${version%.csv}"  # Remove the .csv extension from version
                echo "::set-output name=sub::$sub"
                echo "::set-output name=task::$task"
                echo "::set-output name=version::$version"
                echo "Subject: $sub"
                echo "Task: $task"
                echo "Version: $version"
            done

  run_qc:
    runs-on: self-hosted
    needs: process_raw
    steps:
      - name: Checkout code
        uses: actions/checkout@v3

      - name: Debug env vars
        run: |
            echo "sub=${{ needs.process_raw.outputs.sub }}"
            echo "task=${{ needs.process_raw.outputs.task }}"
            echo "version=${{ needs.process_raw.outputs.version }}"

      - name: run quality control
        run: |
            sub=${{ needs.process_raw.outputs.sub }}
            task=${{ needs.process_raw.outputs.task }}
            vers=${{ needs.process_raw.outputs.version }}
            for sub in ${sub}; do
            echo "Processing subject: $sub"
            for task in ${task}; do
                echo "Processing task: $task"
                for vers in ${vers}; do
                echo "Processing version: $vers"
                csv_file="./data/${sub}/processed/${sub}_${task}_${vers}.csv"
                log_file="./data/${sub}/qc_${task}_${vers}.log"
                echo "CSV file: $csv_file"
                echo "Log file: $log_file"
                if [ -f "$csv_file" ]; then
                    python ./code/AFqC.py -s "$csv_file" -o "./data/${sub}/" -sub "$sub" | tee "$log_file"
                    echo "QC for ${sub}_${task}_${vers} running"
                else
                    echo "CSV file $csv_file does not exist"
                fi
                done
            done
            done
  add:

    concurrency:
      group: "pages"
      cancel-in-progress: false
    runs-on: self-hosted
    needs: run_qc
    steps:
      - name: find new png files
        run: |
            png_files=$(git diff --name-only --relative --diff-filter=A -- '*.png')
            echo png_files << GITHUB_ENV


      - name: Add _posts
        run: |
          POSTS_DIR="_posts"
          # Loop through the images and group by subject number
          declare -A subjects

          for file in $png_files; do
              # Extract the subject number (assuming it's the first part of the filename before '_')
              subject=$(echo "$file" | awk -F_ '{print $1}')
              
              # Append the filename to the subject's array
              subjects["$subject"]+="$file "
          done
          
          # Generate Jekyll posts
          for subject in "${!subjects[@]}"; do
              # Define the post filename (e.g., using current date and subject number)
              post_filename="$POSTS_DIR/$(date +%Y-%m-%d)-subject-$subject.md"

              # Create the Jekyll post
              echo "---" > "$post_filename"
              echo "layout: post" >> "$post_filename"
              echo "title: Subject $subject" >> "$post_filename"
              echo "date: $(date +%Y-%m-%d)" >> "$post_filename"
              echo "categories: subjects" >> "$post_filename"
              echo "---" >> "$post_filename"
              
              # Add images to the post
              for image in ${subjects["$subject"]}; do
                  echo "![]($image)" >> "$post_filename"
              done
          done

      
      - name: checkout
        uses: actions/checkout@v4
      
      - name: Setup Ruby
        uses: ruby/setup-ruby@8575951200e472d5f2d95c625da0c7bec8217c42 # v1.161.0
        with:
          ruby-version: '3.1' # Not needed with a .ruby-version file
          bundler-cache: true # runs 'bundle install' and caches installed gems automatically
          cache-version: 0 # Increment this number if you need to re-download cached gems
      - name: Setup Pages
        id: pages
        uses: actions/configure-pages@v5
      - name: Build with Jekyll
        # Outputs to the './_site' directory by default
        run: bundle exec jekyll build --baseurl "${{ steps.pages.outputs.base_path }}"
        env:
          JEKYLL_ENV: production
      - name: Upload artifact
        # Automatically uploads an artifact from the './_site' directory by default
        uses: actions/upload-pages-artifact@v3

  push:
    runs-on: self-hosted
    needs: run_qc
    steps:
      - name: Commit and Push Changes
        run: |
          git config --global user.name "miloswrath"
          git config --global user.email "miloswrath@users.noreply.github.com"
          git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/$GITHUB_REPOSITORY
          git add .
          git commit -m "Automated commit by GitHub Actions"
          git push
        env:
            GITHUB_TOKEN: ${{ secrets.GIT_PAT }}