diff --git a/.github/workflows/azure-static-web-apps-zealous-stone-0723df810.yml b/.github/workflows/azure-static-web-apps-zealous-stone-0723df810.yml deleted file mode 100644 index 76b8e794..00000000 --- a/.github/workflows/azure-static-web-apps-zealous-stone-0723df810.yml +++ /dev/null @@ -1,46 +0,0 @@ -name: Azure Static Web Apps CI/CD - -on: - push: - branches: - - main - pull_request: - types: [opened, synchronize, reopened, closed] - branches: - - main - -jobs: - build_and_deploy_job: - if: github.event_name == 'push' || (github.event_name == 'pull_request' && github.event.action != 'closed') - runs-on: ubuntu-latest - name: Build and Deploy Job - steps: - - uses: actions/checkout@v3 - with: - submodules: true - lfs: false - - name: Build And Deploy - id: builddeploy - uses: Azure/static-web-apps-deploy@v1 - with: - azure_static_web_apps_api_token: ${{ secrets.AZURE_STATIC_WEB_APPS_API_TOKEN_ZEALOUS_STONE_0723DF810 }} - repo_token: ${{ secrets.GITHUB_TOKEN }} # Used for Github integrations (i.e. PR comments) - action: "upload" - ###### Repository/Build Configurations - These values can be configured to match your app requirements. ###### - # For more information regarding Static Web App workflow configurations, please visit: https://aka.ms/swaworkflowconfig - app_location: "/frontend" # App source code path - api_location: "" # Api source code path - optional - output_location: "build" # Built app content directory - optional - ###### End of Repository/Build Configurations ###### - - close_pull_request_job: - if: github.event_name == 'pull_request' && github.event.action == 'closed' - runs-on: ubuntu-latest - name: Close Pull Request Job - steps: - - name: Close Pull Request - id: closepullrequest - uses: Azure/static-web-apps-deploy@v1 - with: - azure_static_web_apps_api_token: ${{ secrets.AZURE_STATIC_WEB_APPS_API_TOKEN_ZEALOUS_STONE_0723DF810 }} - action: "close" diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e0b9954f..9b436428 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,8 +1,12 @@ -name: Test and Build +name: Test and Build Merge Requests on: push: + paths-ignore: + - 'db/**' pull_request: + paths-ignore: + - 'db/**' jobs: test-and-build: @@ -11,15 +15,16 @@ jobs: # Define Node versions to run CI/CD on strategy: matrix: - node-version: [18.x, 20.x, 21.x] + node-version: [21.x] steps: # Checkout project - - uses: actions/checkout@v3 + - name: Checkout repository + uses: actions/checkout@v4 # Setup Node - - name: Use Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v3 + - name: Use Node.js ${{ matrix.node-version }} for frontend + uses: actions/setup-node@v4 with: node-version: ${{ matrix.node-version }} diff --git a/.github/workflows/data-upgrade.yml b/.github/workflows/data-upgrade.yml new file mode 100644 index 00000000..e32994f2 --- /dev/null +++ b/.github/workflows/data-upgrade.yml @@ -0,0 +1,38 @@ +name: Convert CSV to JSON for Database updates + +on: + push: + paths: + - 'db/**' + pull_request: + paths: + - 'db/**' + +jobs: + convert: + runs-on: ubuntu-latest + + steps: + # Checkout project + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.x' + + - name: Install pandas + run: pip install pandas + + - name: Install regex + run: pip install regex + + - name: Convert CSV to JSON + run: python db/convert-csv-to-json.py + + - name: Upload JSON files as artifacts + uses: actions/upload-artifact@v4 + with: + name: json-files + path: db/json/ \ No newline at end of file diff --git a/.github/workflows/release-new-data.yml b/.github/workflows/release-new-data.yml new file mode 100644 index 00000000..5bb2fb9f --- /dev/null +++ b/.github/workflows/release-new-data.yml @@ -0,0 +1,73 @@ +name: Convert CSV to JSON for Database updates and release to Azure DB + +on: + push: + paths: + - 'db/**' + branches: + - main + pull_request: + branches: + - main + paths: + - 'db/**' + +jobs: + convert: + runs-on: ubuntu-latest + + steps: + # Checkout project + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.x' + + - name: Install pandas + run: pip install pandas + + - name: Install regex + run: pip install regex + + - name: Convert CSV to JSON + run: python db/convert-csv-to-json.py + + - name: Upload JSON files as artifacts + uses: actions/upload-artifact@v4 + with: + name: json-files + path: db/json/ + + mongoimport: + needs: convert + runs-on: ubuntu-latest + strategy: + matrix: + mongodb-version: ['6.0'] + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Download JSON files as artifacts + uses: actions/download-artifact@v4 + with: + name: json-files + path: db/json/ + + - name: Install MongoDB Tools + run: | + wget -qO - https://www.mongodb.org/static/pgp/server-5.0.asc | sudo apt-key add - + echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu $(lsb_release -sc)/mongodb-org/5.0 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-5.0.list + sudo apt-get update + sudo apt-get install -y mongodb-database-tools + + - name: Import to MongoDB + run: | + chmod +x ./db/import_json_to_mongo.sh + ./db/import_json_to_mongo.sh + shell: bash + env: + MONGODB_URI: ${{ secrets.MONGODB_URI }} \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..20d5c00d --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,105 @@ +name: Build and deploy Express.js backend and React frontend as Azure Web App + +on: + push: + paths-ignore: + - 'db/**' + branches: + - main + workflow_dispatch: + +jobs: + build: + runs-on: ubuntu-latest + + strategy: + matrix: + node-version: [21.x] + + steps: + # Checkout project + - name: Checkout repository + uses: actions/checkout@v4 + + # Setup Node + - name: Use Node.js ${{ matrix.node-version }} for frontend + uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node-version }} + + # FRONTEND # + # Install Packages + - name: Install Dependencies for Frontend + run: npm ci + working-directory: frontend + + # Include whenver tests are added + # - name: npm run test for frontend + # run: npm run test --if-present + # working-directory: frontend + + - name: npm run build for frontend + run: npm run build + working-directory: frontend + + # working-directory: frontend + # with: + # node-version: '18.x' + # app_location: "/frontend" + # output_location: "build" + + # Copy build directory to backend + - name: Copy frontend build to backend + run: cp -R ./frontend/build/* ./backend/__BUILD/ + + # BACKEND # + # Install Packages + - name: Install Dependencies for Backend + run: | + npm ci + working-directory: ./backend + + # Run Jest tests + - name: Run Tests + working-directory: ./backend + run: npm run test --if-present + + # Compile TS + - name: Compile TS + working-directory: ./backend + run: npm run compile + + - name: Zip artifact for deployment + run: zip release.zip ./* -r + working-directory: ./backend + + - name: Upload artifact for deployment job + uses: actions/upload-artifact@v3 + with: + name: node-app + path: backend/release.zip + + deploy: + runs-on: ubuntu-latest + needs: build + environment: + name: 'Production' + url: ${{ steps.deploy-to-webapp.outputs.webapp-url }} + + steps: + - name: Download artifact from build job + uses: actions/download-artifact@v3 + with: + name: node-app + + - name: Unzip artifact for deployment + run: unzip release.zip + + - name: 'Deploy to Azure Web App' + id: deploy-to-webapp + uses: azure/webapps-deploy@v2 + with: + app-name: 'lotr-backend' + slot-name: 'Production' + package: . + publish-profile: ${{ secrets.AZUREAPPSERVICE_PUBLISHPROFILE_755D03D3D66E4CFEA1B7360C17082E2D }} diff --git a/.gitignore b/.gitignore index 9b3f0442..d0435b00 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ db/bson/user.bson db/bson/user.metadata.json db/init-prod.js db/db +db/bson-for-azure sample-app/node_modules docker-compose.yml .idea diff --git a/backend/.gitignore b/backend/.gitignore index d09c770e..1dcef2d9 100644 --- a/backend/.gitignore +++ b/backend/.gitignore @@ -1,3 +1,2 @@ node_modules -__BUILD .env \ No newline at end of file diff --git a/backend/__BUILD/test.txt b/backend/__BUILD/test.txt new file mode 100644 index 00000000..e69de29b diff --git a/backend/package.json b/backend/package.json index f7caaf2e..0771c7ca 100644 --- a/backend/package.json +++ b/backend/package.json @@ -7,8 +7,9 @@ "test": "jest", "compile": "npx tsc", "format": "prettier --write \"**/*.ts\"", - "prod": "NODE_ENV=production node server.js", - "prod_win": "set NODE_ENV=production&&node server.js" + "prod": "NODE_ENV=production node dist/server.js", + "prod_win": "set NODE_ENV=production&&node dist/server.js", + "start": "NODE_ENV=production node dist/server.js" }, "keywords": [ "the lord of the rings", @@ -67,4 +68,4 @@ "ts-node": "^10.9.1", "typescript": "^5.3.2" } -} +} \ No newline at end of file diff --git a/backend/server.ts b/backend/server.ts index 15cb27fb..7af6d723 100644 --- a/backend/server.ts +++ b/backend/server.ts @@ -27,7 +27,7 @@ app.use(express.json()); app.use(express.urlencoded({ extended: false })); app.use(cors()); -app.use(express.static(path.join(__dirname, '/__BUILD'))); // React build +app.use(express.static(path.join(__dirname, '/../__BUILD'))); // React build const server_port = process.env.PORT || 3001; @@ -100,13 +100,13 @@ app.use((req, res, next) => { } }); -app.use('/v2/', apiLimiter); +app.use('/v2/', apiLimiter); app.use('/v2', apiRoutes); app.use('/auth', authRoutes); // Handles React frontend requests app.get('*', (req, res) => { - res.sendFile(path.join(__dirname + '/__BUILD/index.html')); + res.sendFile(path.join(__dirname + '/../__BUILD/index.html')); }); async function start() { diff --git a/db/convert-csv-to-json.py b/db/convert-csv-to-json.py new file mode 100644 index 00000000..327e6dda --- /dev/null +++ b/db/convert-csv-to-json.py @@ -0,0 +1,32 @@ +import os +import pandas as pd +import json +import regex as re + +def transform_objectid(text): + """Replace MongoDB ObjectId references to proper JSON format.""" + # Use non-capturing group and directly format the string with $oid. + pattern = r'ObjectId\(([^)]+)\)' + replacements = re.findall(pattern, text) + for r in replacements: + text = text.replace(f'ObjectId({r})', f'{{"$oid": "{r}"}}') + return text + +def main(): + os.makedirs('db/json', exist_ok=True) # Ensure the directory for JSON files exists + csv_files = [f for f in os.listdir('db/csv') if f.endswith('.csv')] + + for file in csv_files: + df = pd.read_csv(f'db/csv/{file}') + # Transform all string columns that may contain ObjectId references + for column in df.select_dtypes(include=['object']): + df[column] = df[column].apply(lambda x: transform_objectid(str(x)) if pd.notna(x) else x) + # Convert transformed string JSON to actual JSON objects + for column in df.select_dtypes(include=['object']): + df[column] = df[column].apply(lambda x: json.loads(x) if pd.notna(x) and x.startswith('{') else x) + # Save each dataframe as a JSON file with all objects in a single array + json_path = f'db/json/{file.replace(".csv", ".json")}' + df.to_json(json_path, orient='records', indent=4) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/db/csv/quotes.csv b/db/csv/quotes.csv index 22db7e72..5295bd6f 100644 --- a/db/csv/quotes.csv +++ b/db/csv/quotes.csv @@ -1,5 +1,5 @@ dialog,movie,character,_id -Deagol!,ObjectId(5cd95395de30eff6ebccde5d),ObjectId(5cd99d4bde30eff6ebccfe9e),ObjectId(5cd96e05de30eff6ebcce7e9) +Deagol!!,ObjectId(5cd95395de30eff6ebccde5d),ObjectId(5cd99d4bde30eff6ebccfe9e),ObjectId(5cd96e05de30eff6ebcce7e9) Deagol!,ObjectId(5cd95395de30eff6ebccde5d),ObjectId(5cd99d4bde30eff6ebccfe9e),ObjectId(5cd96e05de30eff6ebcce7ea) Deagol!,ObjectId(5cd95395de30eff6ebccde5d),ObjectId(5cd99d4bde30eff6ebccfe9e),ObjectId(5cd96e05de30eff6ebcce7eb) Give us that! Deagol my love,ObjectId(5cd95395de30eff6ebccde5d),ObjectId(5cd99d4bde30eff6ebccfe9e),ObjectId(5cd96e05de30eff6ebcce7ec) diff --git a/db/import_json_to_mongo.sh b/db/import_json_to_mongo.sh new file mode 100644 index 00000000..204bc519 --- /dev/null +++ b/db/import_json_to_mongo.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +# Script to import JSON files to MongoDB collections +for file in db/json/*.json; do + # Extract the collection name from the filename + collection=$(basename "$file" .json) + echo "Importing $file to collection $collection" + # Run mongoimport command + mongoimport --type json --uri "$MONGODB_URI" --collection $collection --file "$file" --drop --maintainInsertionOrder --jsonArray +done \ No newline at end of file diff --git a/frontend/src/components/Footer.tsx b/frontend/src/components/Footer.tsx index e6887b40..420a54ef 100644 --- a/frontend/src/components/Footer.tsx +++ b/frontend/src/components/Footer.tsx @@ -6,7 +6,7 @@ const Footer: React.FC = () => {


All we have to decide is what to do with the time that is given to us.{" "} - Built with ♥ in 2020
+ Built with ♥ 2019-{new Date().getFullYear()}

); diff --git a/frontend/src/helpers/api.ts b/frontend/src/helpers/api.ts index b5661c29..3f9cba6f 100644 --- a/frontend/src/helpers/api.ts +++ b/frontend/src/helpers/api.ts @@ -4,7 +4,9 @@ let host: string; if (process.env.NODE_ENV === "development") { host = "http://localhost:3001"; } else { - host = "https://the-one-api.dev"; + host = `${window.location.protocol}//${window.location.hostname}${ + window.location.port ? ":" + window.location.port : "" + }`; } interface RequestOptions { method: string; diff --git a/frontend/src/index.css b/frontend/src/index.css index b4faf5dd..6766cf80 100644 --- a/frontend/src/index.css +++ b/frontend/src/index.css @@ -1,21 +1,20 @@ @charset "UTF-8"; -html, body { +html, +body { max-width: 100%; overflow-x: hidden; - } .card { /* margin-bottom: 3rem; */ - border: 1rem solid var(--card-border-color) + border: 1rem solid var(--card-border-color); } .logo { - font-size: 1.0rem !important; + font-size: 1rem !important; padding: 25px !important; /* font-family: 'Just Another Hand', cursive; */ color: var(--a-visited-color) !important; - } table { @@ -24,15 +23,14 @@ table { .logo a:hover { text-decoration: none; - color: var(--card-border-color) + color: var(--card-border-color); } a:hover { text-decoration: none; - color: var(--header-fore-color) + color: var(--header-fore-color); } - header { color: var(--card-border-color); background-color: var(--fore-color) !important; @@ -40,7 +38,6 @@ header { overflow-x: hidden; } - .drawer-toggle:before { color: var(--card-border-color); font-size: 80px; @@ -55,18 +52,39 @@ header { } } -p, li, ul, th, td, tr, em, a, b, input, label, button { - font-family: 'Work Sans', sans-serif !important; +p, +li, +ul, +th, +td, +tr, +em, +a, +b, +input, +label, +button { + font-family: "Work Sans", sans-serif !important; color: #4c4c4c; font-size: 16px; line-height: 2.1rem; } @media screen and (min-width: 1000px) { - p, li, ul, th, td, tr, em, a, b, input, label, button { + p, + li, + ul, + th, + td, + tr, + em, + a, + b, + input, + label, + button { font-size: 20px; line-height: 2.8rem; - } } @@ -87,7 +105,6 @@ h1 { } } - .subtitle { margin-top: -1.7rem; margin-left: 1rem; @@ -97,7 +114,6 @@ h1 { @media screen and (max-width: 576px) { .subtitle { font-size: 35px; - } } @@ -110,7 +126,6 @@ h2 { h2 { font-size: 28px; line-height: 3.5rem; - } } @@ -118,15 +133,16 @@ h3 { font-size: 36px !important; } - h2 > smaller { top: -1rem; font-size: 1.5rem !important; - } -h1, h2, h3, .subtitle { - font-family: 'Martel', serif; +h1, +h2, +h3, +.subtitle { + font-family: "Martel", serif; } .container { @@ -135,9 +151,8 @@ h1, h2, h3, .subtitle { padding: 0; } #content-wrap { - padding-bottom: 4rem; + padding-bottom: 4rem; margin-bottom: 70px; - } #flexy { @@ -155,25 +170,33 @@ footer { text-align: center; bottom: 0; width: 100%; - height: 3rem; + height: 4rem; } -footer > p, footer b { +footer > p, +footer b { font-size: 20px; line-height: 35px; color: var(--fore-color) !important; } .hr-like { - border: none; - border-top: 1px solid transparent; - background-image: linear-gradient(to right, transparent, var(--border-color) 20%, var(--border-color) 80%, transparent); - height: 1px; - margin: 1em 0; + border: none; + border-top: 1px solid transparent; + background-image: linear-gradient( + to right, + transparent, + var(--border-color) 20%, + var(--border-color) 80%, + transparent + ); + height: 1px; + margin: 1em 0; } @media screen and (max-width: 1000px) { - footer > p, footer b { + footer > p, + footer b { font-size: small; line-height: 25px; } @@ -181,15 +204,12 @@ footer > p, footer b { #drawer { border: 1rem solid var(--card-border-color); - } #drawer a { font-size: 20px !important; - } - nav { line-height: 0.2rem; border: unset; @@ -206,7 +226,8 @@ nav { margin-bottom: 3rem; } -nav a, nav a:visited { +nav a, +nav a:visited { color: unset; } diff --git a/frontend/src/pages/About.tsx b/frontend/src/pages/About.tsx index 1c045575..9f04af09 100644 --- a/frontend/src/pages/About.tsx +++ b/frontend/src/pages/About.tsx @@ -11,27 +11,21 @@ const About: React.FC = () => {


- The one API to rule them all is a project by{" "} + The one API to rule them all is a non-commercial, + open-source project maintained by{" "} Rike {" "} - started during the{" "} + and{" "} - #100DaysOfCode challenge - {" "} - and is brought to you via{" "} - - DigitalOcean + Mateusz - . You have some questions or suggestions? Find me on Twitter ( + , which has been around since 2019. If you have any quick questions or + suggestions, feel free to reach out to Rike on X ( { Express.js {" "} -{" "} + + Azure + {" "} + -{" "} { {" "} and more.

- It could not have been made without the scraping work of Moko Sharma - ( + This project would not have been possible without the invaluable + scraping contributions of Moko Sharma ( { > Kaggle - ) . + ).

@@ -152,7 +154,7 @@ const About: React.FC = () => {

Have a look into some great projects based on this API. You would like to get listed?{" "} - Drop me a line. + Drop Rike a line.

); -} +}; export default About; diff --git a/frontend/src/pages/Account.tsx b/frontend/src/pages/Account.tsx index d83151d9..55d19a5b 100644 --- a/frontend/src/pages/Account.tsx +++ b/frontend/src/pages/Account.tsx @@ -44,6 +44,12 @@ const Account: React.FC = () => { (Include this in your API calls!)

+

+ + Not sure how to do this? Please refer to the documentation's{" "} + auth part! + +

diff --git a/frontend/src/pages/Documentation.tsx b/frontend/src/pages/Documentation.tsx index a77f5e92..c0e02005 100644 --- a/frontend/src/pages/Documentation.tsx +++ b/frontend/src/pages/Documentation.tsx @@ -12,10 +12,15 @@ const Documentation: React.FC = () => { What is an API?
  • - I'm a total newbie to REST APIs. Is there a beginner-friendly introduction? + + I'm a total newbie to REST APIs. Is there a beginner-friendly + introduction? +
  • - Which data does the "one API to rule them all" provide? + + Which data does the "one API to rule them all" provide? +
  • What about response formats and authentication? @@ -38,7 +43,11 @@ const Documentation: React.FC = () => { expect in a graphical user interface like a website.{" "} I took that great definition from{" "} - + this {" "} very understandable article. Please refer to it for more @@ -49,16 +58,30 @@ const Documentation: React.FC = () => {
    -

    I'm a total newbie to REST APIs. Is there a beginner-friendly introduction?

    +

    + I'm a total newbie to REST APIs. Is there a beginner-friendly + introduction? +

    - Yes, I wrote a blog post that explains in detail how REST APIs work and why you would use - them at all. It also covers authentication, JSON handling and a sample React app. {' '} - + Yes, I wrote a blog post that explains in detail how REST APIs work + and why you would use them at all. It also covers authentication, + JSON handling and a sample React app.{" "} + This is the link to the blog post - and + {" "} + and{" "} + here you will find the according code base - for the - sample React app. + {" "} + for the sample React app.

    @@ -90,7 +113,11 @@ const Documentation: React.FC = () => { You need to send the access key as a bearer token in every request you make to the api. Bearer tokens must be included in the authorization header ( - + More information on authorization headers? ) in the following format: @@ -119,9 +146,9 @@ const Documentation: React.FC = () => { - - - + + + @@ -144,7 +171,7 @@ const Documentation: React.FC = () => { -
    EndpointResponseToken requiredEndpointResponseToken required
    Request all chapters of one specific book +
    @@ -231,8 +258,8 @@ const Documentation: React.FC = () => { - - + + @@ -260,7 +287,7 @@ const Documentation: React.FC = () => {
    OptionExampleOptionExample
    - + @@ -281,8 +308,8 @@ const Documentation: React.FC = () => {
    ExamplesExamples
    - - + + @@ -330,6 +357,6 @@ const Documentation: React.FC = () => { ); -} +}; export default Documentation; diff --git a/frontend/src/pages/Home.tsx b/frontend/src/pages/Home.tsx index 37558cf0..e7c83bf3 100644 --- a/frontend/src/pages/Home.tsx +++ b/frontend/src/pages/Home.tsx @@ -12,19 +12,20 @@ const Home: React.FC = () => {

    What is this?

    - Mellon, this is the one API ( + Mellon, this is the one API ( - What the hell is an Application Programming Interface? + wondering what an Application Programming Interface is? - ) to rule them all. No really, it serves your needs - regarding data about The Lord of the Rings, the epic books by - J. R. R. Tolkien and the official movie adaptions by Peter Jackson. + ) to rule them all, that caters to all your needs + concerning data from The Lord of the Rings, the iconic books + by J.R.R. Tolkien and their official film adaptations directed by + Peter Jackson.

    - There are many endpoints available, but you need to{" "} - sign up to obtain an access key. Get a - glimpse into the documentation to check - out all accessible datasets. + The API offers numerous endpoints. To access them, please{" "} + sign up for an access key. We recommend + reviewing the documentation to explore + the extensive datasets available.

    @@ -32,24 +33,21 @@ const Home: React.FC = () => {

    What's new?

    - Version 2.0 is out! (August 2020) Please update your API - calls from{" "} -
    - {" "} - » https://the-one-api.herokuapp.com/v1/ -
    - to -
    - » https://the-one-api.dev/v2/. -

    - We moved from Heroku to DigitalOcean, added some features and - reimplemented a lot of backend code. You can now contribute to this - Open Source Project on Github!{" "} + To explore everything happening with this project, feel free to + visit our{" "} + + Github + {" "} + Page. We warmly invite you to consider contributing as well.

    ); -} +}; export default Home;
    OptionExampleOptionExample