Merge branch 'fix-docker-volumes' of https://github.com/MateuszKikmun…

…ter/lotr-api into fix-docker-volumes

.github/workflows/build.yml

@@ -1,8 +1,12 @@
-name: Test and Build
+name: Test and Build Merge Requests
 
 on:
   push:
+    paths-ignore:
+      - 'db/**'
   pull_request:
+    paths-ignore:
+      - 'db/**'
 
 jobs:
   test-and-build:
@@ -11,15 +15,16 @@ jobs:
     # Define Node versions to run CI/CD on
     strategy:
       matrix:
-        node-version: [18.x, 20.x, 21.x]
+        node-version: [21.x]
 
     steps:
     # Checkout project
-    - uses: actions/checkout@v3
+    - name: Checkout repository
+      uses: actions/checkout@v4
 
     # Setup Node
-    - name: Use Node.js ${{ matrix.node-version }}
-      uses: actions/setup-node@v3
+    - name: Use Node.js ${{ matrix.node-version }} for frontend
+      uses: actions/setup-node@v4
       with:
         node-version: ${{ matrix.node-version }}
 

.github/workflows/data-upgrade.yml

@@ -0,0 +1,38 @@
+name: Convert CSV to JSON for Database updates
+
+on:
+  push:
+    paths:
+      - 'db/**'
+  pull_request:
+    paths:
+      - 'db/**'
+
+jobs:
+  convert:
+    runs-on: ubuntu-latest
+
+    steps:
+    # Checkout project
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.x'
+
+    - name: Install pandas
+      run: pip install pandas
+
+    - name: Install regex
+      run: pip install regex
+
+    - name: Convert CSV to JSON
+      run: python db/convert-csv-to-json.py
+
+    - name: Upload JSON files as artifacts
+      uses: actions/upload-artifact@v4
+      with:
+        name: json-files
+        path: db/json/

.github/workflows/release-new-data.yml

@@ -0,0 +1,73 @@
+name: Convert CSV to JSON for Database updates and release to Azure DB
+
+on:
+  push:
+    paths:
+      - 'db/**'
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+    paths:
+      - 'db/**'
+
+jobs:
+  convert:
+    runs-on: ubuntu-latest
+
+    steps:
+    # Checkout project
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.x'
+
+    - name: Install pandas
+      run: pip install pandas
+
+    - name: Install regex
+      run: pip install regex
+
+    - name: Convert CSV to JSON
+      run: python db/convert-csv-to-json.py
+
+    - name: Upload JSON files as artifacts
+      uses: actions/upload-artifact@v4
+      with:
+        name: json-files
+        path: db/json/
+
+  mongoimport:
+    needs: convert
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        mongodb-version: ['6.0']
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Download JSON files as artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: json-files
+          path: db/json/
+
+      - name: Install MongoDB Tools
+        run: |
+          wget -qO - https://www.mongodb.org/static/pgp/server-5.0.asc | sudo apt-key add -
+          echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu $(lsb_release -sc)/mongodb-org/5.0 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-5.0.list
+          sudo apt-get update
+          sudo apt-get install -y mongodb-database-tools
+
+      - name: Import to MongoDB
+        run: |
+          chmod +x ./db/import_json_to_mongo.sh
+          ./db/import_json_to_mongo.sh
+        shell: bash
+        env:
+          MONGODB_URI: ${{ secrets.MONGODB_URI }}       

.github/workflows/release.yml

@@ -0,0 +1,105 @@
+name: Build and deploy Express.js backend and React frontend as Azure Web App
+
+on:
+  push:
+    paths-ignore:
+      - 'db/**'
+    branches:
+      - main
+  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        node-version: [21.x]
+
+    steps:
+    # Checkout project
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+    # Setup Node
+    - name: Use Node.js ${{ matrix.node-version }} for frontend
+      uses: actions/setup-node@v4
+      with:
+        node-version: ${{ matrix.node-version }}
+
+    # FRONTEND #
+    # Install Packages
+    - name: Install Dependencies for Frontend
+      run: npm ci
+      working-directory: frontend
+
+    # Include whenver tests are added
+    # - name: npm run test for frontend
+    #   run: npm run test --if-present
+    #   working-directory: frontend
+
+    - name: npm run build for frontend
+      run: npm run build
+      working-directory: frontend
+
+    # working-directory: frontend
+    # with:
+    #   node-version: '18.x'
+    #   app_location: "/frontend"
+    #   output_location: "build" 
+
+    # Copy build directory to backend
+    - name: Copy frontend build to backend
+      run: cp -R ./frontend/build/* ./backend/__BUILD/
+
+    # BACKEND #
+    # Install Packages
+    - name: Install Dependencies for Backend
+      run: |
+        npm ci
+      working-directory: ./backend
+
+    # Run Jest tests
+    - name: Run Tests
+      working-directory: ./backend
+      run: npm run test --if-present
+
+    # Compile TS
+    - name: Compile TS
+      working-directory: ./backend
+      run: npm run compile
+
+    - name: Zip artifact for deployment
+      run: zip release.zip ./* -r
+      working-directory: ./backend
+
+    - name: Upload artifact for deployment job
+      uses: actions/upload-artifact@v3
+      with:
+        name: node-app
+        path: backend/release.zip
+
+  deploy:
+    runs-on: ubuntu-latest
+    needs: build
+    environment:
+      name: 'Production'
+      url: ${{ steps.deploy-to-webapp.outputs.webapp-url }}
+
+    steps:
+      - name: Download artifact from build job
+        uses: actions/download-artifact@v3
+        with:
+          name: node-app
+
+      - name: Unzip artifact for deployment
+        run: unzip release.zip
+
+      - name: 'Deploy to Azure Web App'
+        id: deploy-to-webapp
+        uses: azure/webapps-deploy@v2
+        with:
+          app-name: 'lotr-backend'
+          slot-name: 'Production'
+          package: .
+          publish-profile: ${{ secrets.AZUREAPPSERVICE_PUBLISHPROFILE_755D03D3D66E4CFEA1B7360C17082E2D }}

.gitignore

@@ -3,6 +3,7 @@ db/bson/user.bson
 db/bson/user.metadata.json
 db/init-prod.js
 db/db
+db/bson-for-azure
 sample-app/node_modules
 docker-compose.yml
 .idea

backend/.gitignore

@@ -1,3 +1,2 @@
 node_modules
-__BUILD
 .env

backend/package.json

@@ -7,8 +7,9 @@
     "test": "jest",
     "compile": "npx tsc",
     "format": "prettier --write \"**/*.ts\"",
-    "prod": "NODE_ENV=production node server.js",
-    "prod_win": "set NODE_ENV=production&&node server.js"
+    "prod": "NODE_ENV=production node dist/server.js",
+    "prod_win": "set NODE_ENV=production&&node dist/server.js",
+    "start": "NODE_ENV=production node dist/server.js"
   },
   "keywords": [
     "the lord of the rings",
@@ -67,4 +68,4 @@
     "ts-node": "^10.9.1",
     "typescript": "^5.3.2"
   }
-}
+}

backend/server.ts

@@ -27,7 +27,7 @@ app.use(express.json());
 app.use(express.urlencoded({ extended: false }));
 app.use(cors());
 
-app.use(express.static(path.join(__dirname, '/__BUILD'))); // React build
+app.use(express.static(path.join(__dirname, '/../__BUILD'))); // React build
 
 const server_port = process.env.PORT || 3001;
 
@@ -100,13 +100,13 @@ app.use((req, res, next) => {
 	}
 });
 
-app.use('/v2/', apiLimiter); 
+app.use('/v2/', apiLimiter);
 app.use('/v2', apiRoutes);
 app.use('/auth', authRoutes);
 
 // Handles React frontend requests
 app.get('*', (req, res) => {
-	res.sendFile(path.join(__dirname + '/__BUILD/index.html'));
+	res.sendFile(path.join(__dirname + '/../__BUILD/index.html'));
 });
 
 async function start() {

db/convert-csv-to-json.py

@@ -0,0 +1,32 @@
+import os
+import pandas as pd
+import json
+import regex as re
+
+def transform_objectid(text):
+    """Replace MongoDB ObjectId references to proper JSON format."""
+    # Use non-capturing group and directly format the string with $oid.
+    pattern = r'ObjectId\(([^)]+)\)'
+    replacements = re.findall(pattern, text)
+    for r in replacements:
+        text = text.replace(f'ObjectId({r})', f'{{"$oid": "{r}"}}')
+    return text
+
+def main():
+    os.makedirs('db/json', exist_ok=True)  # Ensure the directory for JSON files exists
+    csv_files = [f for f in os.listdir('db/csv') if f.endswith('.csv')]
+
+    for file in csv_files:
+        df = pd.read_csv(f'db/csv/{file}')
+        # Transform all string columns that may contain ObjectId references
+        for column in df.select_dtypes(include=['object']):
+            df[column] = df[column].apply(lambda x: transform_objectid(str(x)) if pd.notna(x) else x)
+        # Convert transformed string JSON to actual JSON objects
+        for column in df.select_dtypes(include=['object']):
+            df[column] = df[column].apply(lambda x: json.loads(x) if pd.notna(x) and x.startswith('{') else x)
+        # Save each dataframe as a JSON file with all objects in a single array
+        json_path = f'db/json/{file.replace(".csv", ".json")}'
+        df.to_json(json_path, orient='records', indent=4)
+
+if __name__ == "__main__":
+    main()