release-new-data.yml

name: Convert CSV to JSON for Database updates and release to Azure DB

on:
  push:
    paths:
      - 'db/**'
    branches:
      - main
  pull_request:
    paths:
      - 'db/**'

jobs:
  convert:
    runs-on: ubuntu-latest

    steps:
    # Checkout project
    - name: Checkout repository
      uses: actions/checkout@v4

    - name: Set up Python
      uses: actions/setup-python@v5
      with:
        python-version: '3.x'

    - name: Install pandas
        run: pip install pandas


    - name: Convert CSV to JSON
        run: |
          import os
          import pandas as pd

          os.makedirs('db/json', exist_ok=True)
          csv_files = [f for f in os.listdir('db/csv') if f.endswith('.csv')]

          for file in csv_files:
              df = pd.read_csv(f'db/csv/{file}')
              json_path = f'db/json/{file.replace(".csv", ".json")}'
              df.to_json(json_path, orient='records', lines=True)
          print("Conversion complete.")

    - name: Upload JSON files as artifacts
      uses: actions/upload-artifact@v4
      with:
        name: json-files
        path: db/json/