From 3bbd9a8e998cbe95f65b750956da8a0f712fe15b Mon Sep 17 00:00:00 2001
From: raheeqi <raheeqi@bu.edu>
Date: Thu, 31 Oct 2024 21:32:11 -0400
Subject: [PATCH 1/3] ui progress bar added
---
.../graphql/resolvers/graphql_resolvers.ts | 193 ++++++-----
backend/graphql/schemas/type_definitions.ts | 26 +-
backend/graphql/types/types.ts | 16 +-
frontend/src/contexts/upload_context.tsx | 9 +
.../src/pages/UploadPage/CSVUploadPage.css | 20 +-
.../src/pages/UploadPage/CSVUploadPage.tsx | 302 ++++++++++++------
package-lock.json | 32 ++
package.json | 1 +
8 files changed, 409 insertions(+), 190 deletions(-)
diff --git a/backend/graphql/resolvers/graphql_resolvers.ts b/backend/graphql/resolvers/graphql_resolvers.ts
index ec88ea3..d0b799b 100644
--- a/backend/graphql/resolvers/graphql_resolvers.ts
+++ b/backend/graphql/resolvers/graphql_resolvers.ts
@@ -10,7 +10,7 @@ import { Collection } from "mongodb";
import { authMiddleware } from "../../authMiddleware.js";
import axios from "axios";
import { error } from "console";
-const proxy_Url = "http://35.229.106.189:80/upload_csv";
+const proxy_Url = process.env.REACT_APP_ML_PIP_URL || "";
import FormData from 'form-data';
import {createWriteStream} from 'fs';
import { GraphQLUpload } from "graphql-upload-minimal";
@@ -18,6 +18,12 @@ import { GraphQLScalarType, GraphQLError} from 'graphql';
import { FileUpload } from 'graphql-upload-minimal';
import path from 'path';
import { gql } from "@apollo/client";
+import { Readable } from 'stream';
+import { PubSub } from 'graphql-subscriptions';
+
+const pubsub = new PubSub();
+const UPLOAD_PROGRESS = 'UPLOAD_PROGRESS';
+
@@ -31,119 +37,147 @@ interface UploadCSVArgs {
file: Promise<FileUpload>; // `file` should be a promise that resolves to a FileUpload type
userId: string;
}
+// Helper function to simulate delayed progress
+const simulateProgressUpdate = (userId, filename, progress) => {
+ pubsub.publish("UPLOAD_PROGRESS", {
+ uploadProgress: { userId, filename, progress, status: "Uploading" },
+ });
+};
+
+const getLastTenUploads = async (userId, db) => {
+ const uploadData = db.collection("uploads");
+ return await uploadData
+ .find({ userId })
+ .sort({ timestamp: -1 })
+ .limit(10)
+ .toArray();
+};
export const resolvers = {
Upload: GraphQLUpload,
Mutation: {
- uploadCSV: async (_, { file, userId }: UploadCSVArgs, { db, req, res }) => {
-
+ uploadCSV: async (_, { file, userId }, { db }) => {
try {
- // await authMiddleware(req, res, () => {});
-
- const upload = await file; // Resolve the file promise to get Upload object
-
+ const upload = await file; // Resolve the file promise to get the Upload object
if (!upload) {
throw new Error("No file uploaded");
}
- // Destructure the resolved `upload` object to get file properties
const { createReadStream, filename, mimetype } = upload;
-
- // Check if `createReadStream` is defined
if (!createReadStream) {
throw new Error("Invalid file upload. `createReadStream` is not defined.");
}
console.log(`Uploading file: ${filename} (Type: ${mimetype}) for user: ${userId}`);
- // Step 2: Create a read stream from the file
+ // Step 1: Buffer the file to calculate size
const stream = createReadStream();
+ const chunks = [];
+ let fileSize = 0;
- // Step 3: Prepare FormData for sending to an external service (optional)
+ for await (const chunk of stream) {
+ chunks.push(chunk);
+ fileSize += chunk.length;
+ }
+
+ const fileBuffer = Buffer.concat(chunks); // File is now fully buffered
+ const fileSizeKB = (fileSize / 1024).toFixed(1);
+ console.log(`File size: ${fileSizeKB} KB`);
+
+ // Step 2: Create a new stream from the buffer for uploading
+ const uploadStream = Readable.from(fileBuffer);
const formData = new FormData();
+ formData.append("file", uploadStream, { filename });
+ formData.append("user_id", userId);
+
+ // Step 3: Upload with progress updates
+ const response = await axios.post(proxy_Url, formData, {
+ headers: {
+ ...formData.getHeaders(),
+ "X-API-KEY": "beri-stronk-key",
+ },
+ onUploadProgress: function (progressEvent) {
+ const progress = Math.min(
+ Math.round((progressEvent.loaded / fileSize) * 100),
+ 100
+ );
+ // Emit progress update for the client
+ simulateProgressUpdate(userId, filename, progress);
- const map = JSON.stringify({ "1": ["variables.file"] });
- formData.append("operations", JSON.stringify({
- query: `mutation UploadCSV($file: Upload!, $userId: String!): UploadStatus! {
- uploadCSV(file: $file, user_id: $userId) {
- filename
- status
- }
- }`,
- variables: { file: null, userId},
- }));
- formData.append("map", map);
- formData.append("1", stream, { filename, contentType: mimetype });
-
- formData.append('file', stream, { filename });
- formData.append('user_id', userId);
-
- // Step 4: Send the file to an external API
-
- console.log('URL being used in production:' , proxy_Url);
- const response = await axios.post(
- proxy_Url,
- formData,
- {
- headers: {
- ...formData.getHeaders(),
- "X-API-KEY": "beri-stronk-key"
- },
- }
- );
+ console.log(`Upload Progress: ${progress}%`);
+ },
+ });
// Handle API response
if (response.status === 200) {
- console.log('File uploaded successfully to external API.');
+ console.log("File uploaded successfully to external API.");
+
+ // Publish completion message
+ pubsub.publish("UPLOAD_PROGRESS", {
+ uploadProgress: {
+ userId,
+ filename,
+ progress: 100,
+ status: "Upload complete!",
+ },
+ });
- // Step 5: Save upload metadata to the database (if needed)
- const uploadData = db.collection('uploads');
- const result = await uploadData.insertOne({
+ const uploadData = db.collection("uploads");
+ const result = await uploadData.insertOne({
userId,
filename,
timestamp: new Date(),
- status: 'Success',
+ status: "Success",
+ size: fileSizeKB,
});
- return { filename: filename, status: 'Success' };
+ return { filename, status: "Success" };
} else {
- throw new GraphQLError('Failed to upload CSV.');
+ throw new GraphQLError("Failed to upload CSV.");
}
} catch (error) {
- console.error('Error uploading CSV:', error);
- throw new GraphQLError('Error uploading CSV.');
+ console.error("Error uploading CSV:", error);
+ throw new GraphQLError("Error uploading CSV.");
}
- },
- addRssFeed: async (_, { url, userID }, { db, req, res }) => {
- await authMiddleware(req, res, () => {});
-
- const decodedToken = JSON.parse(req.headers.user as string);
- if (!decodedToken) {
- throw new Error('Unauthorized');
- }
-
- if (decodedToken.sub !== userID) {
- throw new Error('Forbidden');
- }
-
- const rss_data = db.collection("rss_links");
-
- // Create or update the RSS feed for the given userID
- const filter = { userID: userID };
- const update = {
- $set: { url: url, userID: userID },
- };
- const options = {
- upsert: true, // create a new document if no document matches the filter
- returnDocument: "after", // return the modified document
- };
+ },
+
+ addRssFeed: async (_, { url, userID }, { db, req, res }) => {
+ await authMiddleware(req, res, () => {});
+
+ const decodedToken = JSON.parse(req.headers.user as string);
+ if (!decodedToken) {
+ throw new Error('Unauthorized');
+ }
+
+ if (decodedToken.sub !== userID) {
+ throw new Error('Forbidden');
+ }
+
+ const rss_data = db.collection("rss_links");
+
+ // Create or update the RSS feed for the given userID
+ const filter = { userID: userID };
+ const update = {
+ $set: { url: url, userID: userID },
+ };
+ const options = {
+ upsert: true, // create a new document if no document matches the filter
+ returnDocument: "after", // return the modified document
+ };
+
+ const result = await rss_data.findOneAndUpdate(filter, update, options);
- const result = await rss_data.findOneAndUpdate(filter, update, options);
+ return result.value;
+ },
+ },
+
+ Subscription: {
+ uploadProgress: {
+ subscribe: (_, { userId }) => pubsub.asyncIterator("UPLOAD_PROGRESS"),
+ },
+ },
- return result.value;
- },
- },
Query: {
// RSS Resolver
getRssLinkByUserId: async (_, args, { db, req, res }) => {
@@ -163,6 +197,9 @@ export const resolvers = {
const queryResult = await rss_data.find({ userID: args.userID }).toArray();
return queryResult;
},
+ lastTenUploads: async (_, { userId }, { db }) => {
+ return getLastTenUploads(userId, db);
+ },
// CSV Upload Resolver
getUploadByUserId: async (_, args, { db, req, res }) => {
await authMiddleware(req, res, () => {});
diff --git a/backend/graphql/schemas/type_definitions.ts b/backend/graphql/schemas/type_definitions.ts
index 5a590ba..b05979c 100644
--- a/backend/graphql/schemas/type_definitions.ts
+++ b/backend/graphql/schemas/type_definitions.ts
@@ -9,6 +9,17 @@ export const typeDefs = gql`
userID: String!
}
+ type Subscription {
+ uploadProgress(userId: String!): UploadProgress
+ }
+
+ type UploadProgress {
+ userId: String!
+ filename: String!
+ progress: Int!
+ status: String
+ }
+
type Rss_data {
userID: String!
title: String!
@@ -46,6 +57,19 @@ export const typeDefs = gql`
filename: String!
status: String!
}
+
+ type Query {
+ lastTenUploads(userId: String!): [UploadHistory!]!
+}
+
+type UploadHistory {
+ uploadID: String
+ timestamp: String
+ article_cnt: Int
+ status: String
+ filename: String
+}
+
type Article {
@@ -135,4 +159,4 @@ export const typeDefs = gql`
}
-`;
+`;
\ No newline at end of file
diff --git a/backend/graphql/types/types.ts b/backend/graphql/types/types.ts
index bc1f66a..c5f6bef 100644
--- a/backend/graphql/types/types.ts
+++ b/backend/graphql/types/types.ts
@@ -18,23 +18,15 @@ export interface DemographicsByTractsArgs {
tract: string;F
}
-export interface IUploadedFile {
- name: string;
- size: number;
- progress: number;
- status: string;
- file: FileUpload;
- error?: string;
-}
type UploadedFile = {
name: string;
size: number;
- progress: number;
+ progress: number; // Track upload progress here
status: string;
- error?: string; // fail to pass test
- file: File; // store a reference to the File object
-};
+ error?: string;
+ file: File;
+};
// *** Data Types derived from the collections ***
diff --git a/frontend/src/contexts/upload_context.tsx b/frontend/src/contexts/upload_context.tsx
index ef2157c..774a5f8 100644
--- a/frontend/src/contexts/upload_context.tsx
+++ b/frontend/src/contexts/upload_context.tsx
@@ -24,6 +24,15 @@ const UPLOAD_DATA_QUERY = gql`
}
`;
+export const GET_UPLOAD_PROGRESS = gql`
+ query GetUploadProgress($userId: String!) {
+ uploadProgress(userId: $userId) {
+ progress
+ status
+ }
+ }
+`;
+
export const UPLOAD_CSV_MUTATION = gql`
mutation UploadCSV($file: Upload!, $userId: String!) {
uploadCSV(file: $file, userId: $userId) {
diff --git a/frontend/src/pages/UploadPage/CSVUploadPage.css b/frontend/src/pages/UploadPage/CSVUploadPage.css
index e221e6e..7d23326 100644
--- a/frontend/src/pages/UploadPage/CSVUploadPage.css
+++ b/frontend/src/pages/UploadPage/CSVUploadPage.css
@@ -96,10 +96,6 @@
font-weight: bold;
}
-.file-upload-status {
- display: flex;
- margin-bottom: 10px;
-}
.file-name {
display: flex;
@@ -141,6 +137,22 @@
align-items: center;
}
+.file-progress {
+ width: 100%;
+ padding: 0 10px;
+}
+.file-upload-status {
+ display: flex;
+ align-items: center;
+ justify-content: space-between;
+ padding: 10px 0;
+ border-bottom: 1px solid #e0e0e0;
+}
+.file-name, .file-size, .file-status, .file-actions, .file-progress {
+ flex: 1;
+ text-align: center;
+}
+
.alert-message {
position: fixed;
top: 0;
diff --git a/frontend/src/pages/UploadPage/CSVUploadPage.tsx b/frontend/src/pages/UploadPage/CSVUploadPage.tsx
index a6e3637..666a565 100644
--- a/frontend/src/pages/UploadPage/CSVUploadPage.tsx
+++ b/frontend/src/pages/UploadPage/CSVUploadPage.tsx
@@ -13,10 +13,37 @@ import HelpIcon from '@mui/icons-material/Help';
import Tooltip from '@mui/material/Tooltip';
import Modal from '@mui/material/Modal';
import Box from '@mui/material/Box';
-import { gql } from "@apollo/client";
+import { gql, useSubscription } from "@apollo/client";
import ArrowBackIcon from '@mui/icons-material/ArrowBack';
+import LinearProgress from '@mui/material/LinearProgress';
+
+
+
+
+const UPLOAD_PROGRESS_SUBSCRIPTION = gql`
+ subscription uploadProgress($userId: String!) {
+ uploadProgress(userId: $userId) {
+ filename
+ progress
+ status
+ }
+ }
+`;
+
+const LAST_TEN_UPLOADS_QUERY = gql`
+ query lastTenUploads($userId: String!) {
+ lastTenUploads(userId: $userId) {
+ uploadID
+ timestamp
+ article_cnt
+ status
+ filename
+ }
+ }
+`;
+
// Define a type for the file with progress information
type UploadedFile = {
name: string;
@@ -40,6 +67,7 @@ export const CSVUploadBox = () => {
const [uploads, setUpload] = useState<Uploads[]>([]);
const { user, isSignedIn } = useUser();
const { organization } = useOrganization();
+
@@ -52,6 +80,24 @@ export const CSVUploadBox = () => {
const handleOpen = () => setOpen(true);
const handleClose = () => setOpen(false);
+ const { data: progressData } = useSubscription(UPLOAD_PROGRESS_SUBSCRIPTION, {
+ variables: { userId: organization ? organization.id : user?.id },
+ shouldResubscribe: true,
+ onSubscriptionData: ({ subscriptionData }) => {
+ const uploadProgress = subscriptionData.data.uploadProgress;
+ if (uploadProgress) {
+ const { filename, progress, status } = uploadProgress;
+ setUploadedFiles((prevFiles) =>
+ prevFiles.map((file) =>
+ file.name === filename
+ ? { ...file, progress, status: status || file.status }
+ : file
+ )
+ );
+ }
+ },
+ });
+
useEffect(() => {
if (isSignedIn && user) {
if (organization) {
@@ -201,62 +247,138 @@ export const CSVUploadBox = () => {
},
});
- const submitFile = () => {
- console.log("submitting file");
- for (let i = 0; i < submittedFiles.length; i++) {
- const file = submittedFiles[i];
- console.log("Type of file:", file instanceof File);
-
- if (user && isSignedIn) {
- const variables = {
- file,
- userId: organization ? organization.id : user.id,
- };
- console.log( typeof user.id);
-
- console.log("logging variables: ", variables);
-
- uploadCSV({ variables })
- .then((response) => {
- // Check if the response contains the expected data
- if (response.data && response.data.uploadCSV && response.data.uploadCSV.status === 'Success') {
- setSuccessMessage("Successfully submitted!");
- setTimeout(() => setSuccessMessage(""), 3000);
- } else if (response.errors) {
- // Handle GraphQL errors
- console.error("GraphQL Errors:", response.errors);
- const errorMessage = response.errors[0]?.message || "Failed to upload CSV."; // Extract the error message
- setAlertMessage(errorMessage);
- setTimeout(() => setAlertMessage(""), 3000);
- } else {
- // Handle unexpected responses
- console.error("Unexpected response:", response);
- setAlertMessage("Failed to upload CSV: Unexpected response.");
- setTimeout(() => setAlertMessage(""), 3000);
- }
- })
- .catch((error) => {
- console.error("Error during CSV upload:", error);
- setAlertMessage("Failed to upload CSV.");
- setTimeout(() => setAlertMessage(""), 3000);
- });
- } else {
- console.error("User is not signed in");
- }
- }
+ const submitFile = () => {
+ for (let i = 0; i < submittedFiles.length; i++) {
+ const file = submittedFiles[i];
+
+ if (user && isSignedIn) {
+ const variables = {
+ file,
+ userId: organization ? organization.id : user.id,
+ };
+
+ // Set initial progress to indicate the upload has started
+ setUploadedFiles((prevFiles) =>
+ prevFiles.map((f) =>
+ f.name === file.name ? { ...f, progress: 10, status: 'Uploading...' } : f
+ )
+ );
+
+ const reader = new FileReader();
+ reader.onload = () => {
+ const text = reader.result as string;
+ const lines = text.split(/\r?\n/);
+ const articleCount = lines.length - 1;
+
+ uploadCSV({ variables })
+ .then((response) => {
+ if (response.data && response.data.uploadCSV && response.data.uploadCSV.status === 'Success') {
+ // Simulate gradual progress update
+ let progress = 10;
+ const interval = setInterval(() => {
+ progress += 15;
+ setUploadedFiles((prevFiles) =>
+ prevFiles.map((f) =>
+ f.name === file.name
+ ? { ...f, progress: Math.min(progress, 100) }
+ : f
+ )
+ );
+
+ // Finalize when progress reaches 100%
+ if (progress >= 100) {
+ clearInterval(interval);
+
+ // Set final status to complete
+ setUploadedFiles((prevFiles) =>
+ prevFiles.map((f) =>
+ f.name === file.name ? { ...f, progress: 100, status: 'Upload complete!' } : f
+ )
+ );
+
+ // Display success message
+ setSuccessMessage("File uploaded successfully!");
+
+ // Delay file removal from 'uploaded files' table
+ setTimeout(() => {
+ setSuccessMessage("");
+
+ // Add to upload history
+ setUpload((prevUploads) => [
+ ...prevUploads,
+ {
+ uploadID: Math.random().toString(36).substr(2, 9),
+ timestamp: new Date().toISOString(),
+ article_cnt: articleCount,
+ status: 'Success',
+ userID: user.id,
+ message: '',
+ },
+ ]);
+
+ // Now remove the file from the 'uploaded files' table
+ setUploadedFiles((prevFiles) =>
+ prevFiles.filter((f) => f.name !== file.name)
+ );
+ }, 3000); // Delay for the green success message to display
+ }
+ }, 500); // Adjust interval for progress update frequency
+ } else {
+ throw new Error("Unexpected response"); // Force catch for unexpected responses
+ }
+ })
+ .catch((error) => {
+ console.error("Error during CSV upload:", error);
+
+ // Update progress to stop at failure and set status to failed
+ setUploadedFiles((prevFiles) =>
+ prevFiles.map((f) =>
+ f.name === file.name
+ ? { ...f, progress: 0, status: 'Upload failed!' }
+ : f
+ )
+ );
+
+ // Show alert message
+ setAlertMessage("Failed to upload CSV.");
+ setTimeout(() => setAlertMessage(""), 3000);
+ });
+ };
+
+ // Start reading the file to calculate line count
+ reader.readAsText(file);
+ } else {
+ console.error("User is not signed in");
+ }
+ }
};
+
+
+
+
+
+
+
+
// Function that simulates file upload and updates progress
// Need to change this to real func converting csv into json as input to backend
+
+
+
const uploadFile = (file: File) => {
+ const sizeKB = (file.size / 1024).toFixed(1); // Convert size to KB
+
const newFile: UploadedFile = {
name: file.name,
- size: file.size,
+ size: Number(sizeKB), // Store size in KB directly for easier rendering
progress: 0,
status: "Uploading...",
file: file,
};
+ console.log(`File name: ${file.name}, File size in KB: ${sizeKB}`); // Confirm formatted size here
+
setUploadedFiles((prevFiles) => [...prevFiles, newFile]);
validateCsvHeaders(file, (missingHeaders, missingDataWarnings) => {
@@ -296,8 +418,14 @@ export const CSVUploadBox = () => {
// Handle file selection
const handleFileChange = (event: React.ChangeEvent<HTMLInputElement>) => {
- if (event.target.files) {
- Array.from(event.target.files).forEach(uploadFile);
+ const files = event.target.files;
+ if (files && files.length > 0) {
+ const file = files[0];
+ const fileSize = file.size; // Get the file size directly
+ console.log(`File size: ${fileSize} bytes`);
+
+ // Call your upload function with the file and its size
+ uploadFile(file);
}
};
@@ -315,6 +443,7 @@ export const CSVUploadBox = () => {
// Handle file submit
const handleFileSubmit = (files: File[]) => {
submitFile();
+ /*
// clear out uploaded Files, validated files (submitted files listen onto validated files)
for (let i = 0; i < files.length; i++) {
setUploadedFiles((prevFiles) =>
@@ -324,6 +453,7 @@ export const CSVUploadBox = () => {
prevFiles.filter((f) => f.name != files[i].name),
);
}
+ */
// need some logic to handle file history
};
//If the user is does not have access to the upload page
@@ -557,45 +687,38 @@ export const CSVUploadBox = () => {
<span>ACTIONS</span>
</div>
{uploadedFiles.map((file, index) => (
- <div key={index} className='file-upload-status'>
- {/* File Name Column */}
- <div className='file-name'>
- <span>{file.name}</span>
- </div>
+ <div key={index} className='file-upload-status'>
+ {/* File Name Column */}
+ <div className='file-name'>
+ <span>{file.name}</span>
+ </div>
- {/* File Size Column */}
- <div className='file-size'>
- <span>
- {(file.size / (1024 * 1024)).toFixed(1)}{" "}
- MB
- </span>
- </div>
+ {/* File Size Column */}
+ <div className='file-size'>
+ <span>{file.size} KB</span>
+</div>
- {/* Upload Status Column */}
- <div className='file-upload-progress-wrapper'>
- <div className='file-status'>
- {file.status}
- </div>
- {file.error && (
- <div className='error-message'>
- {file.error}
- </div>
- )}
- </div>
- {/* Action Column */}
- <div className='file-actions'>
- {(file.status === "Passed" ||
- file.error) && (
- <Button
+ {/* Validation Status Column */}
+ <div className='file-status'>
+ {file.status}
+ </div>
+
+ {/* Progress Column */}
+ <div className='file-progress'>
+ <LinearProgress variant="determinate" value={file.progress} />
+ </div>
+
+ {/* Action Column */}
+ <div className='file-actions'>
+ {(file.status === "Passed" || file.error) && (
+ <Button
variant="outlined"
color="error"
size="small"
- onClick={() =>
- handleFileRemoval(file.name)
- }
- >
- Delete
- </Button>
+ onClick={() => handleFileRemoval(file.name)}
+ >
+ Delete
+ </Button>
)}
</div>
</div>
@@ -624,30 +747,19 @@ export const CSVUploadBox = () => {
</div>
{uploads.map((file, index) => (
<div key={index} className='file-upload-status'>
- {/* Upload ID Column */}
<div className='file-uploadId'>
<span>{file.uploadID}</span>
</div>
-
- {/* Time Stamp Column */}
<div className='file-timeStamp'>
<span>
- {new Date(
- file.timestamp,
- ).toLocaleString()}
+ {new Date(file.timestamp).toLocaleString()}
</span>
</div>
-
- {/* Article cnt Column */}
<div className='file-articleCnt'>
<span>
- {file.article_cnt === -1
- ? 0
- : file.article_cnt}
+ {file.article_cnt === -1 ? 0 : file.article_cnt}
</span>
</div>
-
- {/* http status Column */}
<div className='file-httpStatus'>
<span>{file.status}</span>
</div>
@@ -660,4 +772,4 @@ export const CSVUploadBox = () => {
);
};
-export default CSVUploadBox;
+export default CSVUploadBox;
\ No newline at end of file
diff --git a/package-lock.json b/package-lock.json
index 766b667..18dcb78 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -10,6 +10,7 @@
"express-fileupload": "^1.5.1",
"font-awesome": "^4.7.0",
"graphql": "^16.8.1",
+ "graphql-subscriptions": "^2.0.0",
"graphql-upload-minimal": "^1.6.1",
"koa": "^2.15.3",
"node-fetch": "^2.7.0",
@@ -3643,6 +3644,18 @@
"node": "^12.22.0 || ^14.16.0 || ^16.0.0 || >=17.0.0"
}
},
+ "node_modules/graphql-subscriptions": {
+ "version": "2.0.0",
+ "resolved": "https://registry.npmjs.org/graphql-subscriptions/-/graphql-subscriptions-2.0.0.tgz",
+ "integrity": "sha512-s6k2b8mmt9gF9pEfkxsaO1lTxaySfKoEJzEfmwguBbQ//Oq23hIXCfR1hm4kdh5hnR20RdwB+s3BCb+0duHSZA==",
+ "license": "MIT",
+ "dependencies": {
+ "iterall": "^1.3.0"
+ },
+ "peerDependencies": {
+ "graphql": "^15.7.2 || ^16.0.0"
+ }
+ },
"node_modules/graphql-tag": {
"version": "2.12.6",
"resolved": "https://registry.npmjs.org/graphql-tag/-/graphql-tag-2.12.6.tgz",
@@ -4131,6 +4144,12 @@
"node": ">=8"
}
},
+ "node_modules/iterall": {
+ "version": "1.3.0",
+ "resolved": "https://registry.npmjs.org/iterall/-/iterall-1.3.0.tgz",
+ "integrity": "sha512-QZ9qOMdF+QLHxy1QIpUHUU1D5pS2CG2P69LF6L6CPjPYA/XMOmKV3PZpawHoAjHNyB0swdVTRxdYT4tbBbxqwg==",
+ "license": "MIT"
+ },
"node_modules/jest": {
"version": "29.7.0",
"resolved": "https://registry.npmjs.org/jest/-/jest-29.7.0.tgz",
@@ -10661,6 +10680,14 @@
"resolved": "https://registry.npmjs.org/graphql/-/graphql-16.8.1.tgz",
"integrity": "sha512-59LZHPdGZVh695Ud9lRzPBVTtlX9ZCV150Er2W43ro37wVof0ctenSaskPPjN7lVTIN8mSZt8PHUNKZuNQUuxw=="
},
+ "graphql-subscriptions": {
+ "version": "2.0.0",
+ "resolved": "https://registry.npmjs.org/graphql-subscriptions/-/graphql-subscriptions-2.0.0.tgz",
+ "integrity": "sha512-s6k2b8mmt9gF9pEfkxsaO1lTxaySfKoEJzEfmwguBbQ//Oq23hIXCfR1hm4kdh5hnR20RdwB+s3BCb+0duHSZA==",
+ "requires": {
+ "iterall": "^1.3.0"
+ }
+ },
"graphql-tag": {
"version": "2.12.6",
"resolved": "https://registry.npmjs.org/graphql-tag/-/graphql-tag-2.12.6.tgz",
@@ -11002,6 +11029,11 @@
"istanbul-lib-report": "^3.0.0"
}
},
+ "iterall": {
+ "version": "1.3.0",
+ "resolved": "https://registry.npmjs.org/iterall/-/iterall-1.3.0.tgz",
+ "integrity": "sha512-QZ9qOMdF+QLHxy1QIpUHUU1D5pS2CG2P69LF6L6CPjPYA/XMOmKV3PZpawHoAjHNyB0swdVTRxdYT4tbBbxqwg=="
+ },
"jest": {
"version": "29.7.0",
"resolved": "https://registry.npmjs.org/jest/-/jest-29.7.0.tgz",
diff --git a/package.json b/package.json
index 9c60817..090da92 100644
--- a/package.json
+++ b/package.json
@@ -5,6 +5,7 @@
"express-fileupload": "^1.5.1",
"font-awesome": "^4.7.0",
"graphql": "^16.8.1",
+ "graphql-subscriptions": "^2.0.0",
"graphql-upload-minimal": "^1.6.1",
"koa": "^2.15.3",
"node-fetch": "^2.7.0",
From d1e6b0289b9c5cadf9599dc5756b8157dd61eb36 Mon Sep 17 00:00:00 2001
From: raheeqi <raheeqi@bu.edu>
Date: Sun, 3 Nov 2024 12:10:54 -0500
Subject: [PATCH 2/3] upload history works
---
.../graphql/resolvers/graphql_resolvers.ts | 55 +++++---
backend/graphql/schemas/type_definitions.ts | 16 ++-
frontend/src/contexts/upload_context.tsx | 30 ++--
.../src/pages/UploadPage/CSVUploadPage.tsx | 129 +++++++++++-------
package-lock.json | 29 ++++
package.json | 2 +
6 files changed, 170 insertions(+), 91 deletions(-)
diff --git a/backend/graphql/resolvers/graphql_resolvers.ts b/backend/graphql/resolvers/graphql_resolvers.ts
index d0b799b..6439e40 100644
--- a/backend/graphql/resolvers/graphql_resolvers.ts
+++ b/backend/graphql/resolvers/graphql_resolvers.ts
@@ -23,6 +23,7 @@ import { PubSub } from 'graphql-subscriptions';
const pubsub = new PubSub();
const UPLOAD_PROGRESS = 'UPLOAD_PROGRESS';
+const UPLOAD_STATUS_UPDATED = 'UPLOAD_STATUS_UPDATED';
@@ -44,14 +45,17 @@ const simulateProgressUpdate = (userId, filename, progress) => {
});
};
-const getLastTenUploads = async (userId, db) => {
- const uploadData = db.collection("uploads");
- return await uploadData
- .find({ userId })
- .sort({ timestamp: -1 })
- .limit(10)
- .toArray();
-};
+// const getLastTenUploads = async (userId, db) => {
+// const uploadData = db.collection("uploads");
+// return await uploadData
+// .find({ userId })
+// .sort({ timestamp: -1 })
+// .limit(10)
+// .map(upload => ({ ...upload, article_cnt: upload.article_cnt || 0 })) // Set default value to 0 if null
+// .toArray();
+
+
+// };
export const resolvers = {
Upload: GraphQLUpload,
@@ -124,12 +128,19 @@ export const resolvers = {
});
const uploadData = db.collection("uploads");
- const result = await uploadData.insertOne({
- userId,
- filename,
- timestamp: new Date(),
- status: "Success",
- size: fileSizeKB,
+ // const result = await uploadData.insertOne({
+ // userId,
+ // filename,
+ // timestamp: new Date(),
+ // status: "Success",
+ // size: fileSizeKB,
+ // });
+
+ await db.collection("uploads").watch().on('change', (change) => {
+ const updatedUpload = change.fullDocument;
+ pubsub.publish(UPLOAD_STATUS_UPDATED, {
+ uploadStatusUpdated: updatedUpload,
+ });
});
return { filename, status: "Success" };
@@ -176,6 +187,9 @@ export const resolvers = {
uploadProgress: {
subscribe: (_, { userId }) => pubsub.asyncIterator("UPLOAD_PROGRESS"),
},
+ uploadStatusUpdated: {
+ subscribe: () => pubsub.asyncIterator([UPLOAD_STATUS_UPDATED]),
+ },
},
Query: {
@@ -197,8 +211,17 @@ export const resolvers = {
const queryResult = await rss_data.find({ userID: args.userID }).toArray();
return queryResult;
},
- lastTenUploads: async (_, { userId }, { db }) => {
- return getLastTenUploads(userId, db);
+ lastTenUploads: async (_, { userId }, { db }) => {
+ const uploads = await db.collection("uploads")
+ .find({ userID: userId })
+ .sort({ timestamp: -1 })
+ .limit(10)
+ .toArray();
+
+ return uploads.map(upload => ({
+ ...upload,
+ uploadID: upload.uploadID
+ }));
},
// CSV Upload Resolver
getUploadByUserId: async (_, args, { db, req, res }) => {
diff --git a/backend/graphql/schemas/type_definitions.ts b/backend/graphql/schemas/type_definitions.ts
index b05979c..c412b12 100644
--- a/backend/graphql/schemas/type_definitions.ts
+++ b/backend/graphql/schemas/type_definitions.ts
@@ -40,6 +40,10 @@ export const typeDefs = gql`
message: String!
}
+ type Subscription {
+ uploadStatusUpdated: Uploads
+}
+
type Mutation {
# Define a new mutation for uploading a CSV file
uploadCSV(file: Upload!, userId: String!): UploadStatus!
@@ -63,11 +67,13 @@ export const typeDefs = gql`
}
type UploadHistory {
- uploadID: String
- timestamp: String
- article_cnt: Int
- status: String
- filename: String
+ uploadID: String!
+ article_cnt: Int!
+ message: String!
+ status: String!
+ timestamp: String!
+ userID: String!
+
}
diff --git a/frontend/src/contexts/upload_context.tsx b/frontend/src/contexts/upload_context.tsx
index 774a5f8..4289d94 100644
--- a/frontend/src/contexts/upload_context.tsx
+++ b/frontend/src/contexts/upload_context.tsx
@@ -1,5 +1,5 @@
-import React from "react";
-import { useLazyQuery, gql, useMutation } from "@apollo/client";
+import React, { useEffect } from "react";
+import { useQuery, gql, useMutation } from "@apollo/client";
import { Uploads } from "../__generated__/graphql";
type UploadContextType = {
@@ -9,16 +9,12 @@ type UploadContextType = {
uploadCSV: (file: File, userID: string) => void;
};
-/* Upload Queries */
-// We will pass what we need in here
const UPLOAD_DATA_QUERY = gql`
query GetUploadByUserId($userId: String!) {
getUploadByUserId(user_id: $userId) {
- article_cnt
- message
+ uploadID
status
timestamp
- uploadID
userID
}
}
@@ -63,11 +59,11 @@ const UploadProvider: React.FC = ({ children }: any) => {
const [uploadCSVMutation, { loading: uploadingCSV, error: uploadCSVError }] = useMutation(UPLOAD_CSV_MUTATION);
+ const { data: uploadData, loading: uploadDataLoading, error: uploadDataError } = useQuery(UPLOAD_DATA_QUERY, {
+ variables: { userId: "user-id-placeholder" }, // Replace with dynamic user ID as needed
+ fetchPolicy: "network-only", // Ensures data is fresh on each load
+ });
- const [
- queryUploadData,
- { data: uploadData, loading: uploadDataLoading, error: uploadDataError },
- ] = useLazyQuery(UPLOAD_DATA_QUERY);
const [uploads, setUploadData] = React.useState<Uploads[] | null>(null);
React.useEffect(() => {
@@ -96,18 +92,8 @@ const UploadProvider: React.FC = ({ children }: any) => {
});
};
-
const queryUploadDataType = (queryType: "UPLOAD_DATA", options?: any) => {
- switch (queryType) {
- case "UPLOAD_DATA":
- queryUploadData({
- variables: options,
- });
- break;
- default:
- console.log("ERROR: Fetch Upload Data does not have this query Type!");
- break;
- }
+ // No longer needed as useQuery will handle loading on component mount
};
return (
diff --git a/frontend/src/pages/UploadPage/CSVUploadPage.tsx b/frontend/src/pages/UploadPage/CSVUploadPage.tsx
index 666a565..ce0ad8e 100644
--- a/frontend/src/pages/UploadPage/CSVUploadPage.tsx
+++ b/frontend/src/pages/UploadPage/CSVUploadPage.tsx
@@ -8,7 +8,7 @@ import "./CSVUploadPage.css";
import { UploadContext, UPLOAD_CSV_MUTATION } from "../../contexts/upload_context";
import { Uploads } from "../../__generated__/graphql";
import { useOrganization, useUser, useAuth } from "@clerk/clerk-react";
-import { useMutation } from "@apollo/client";
+import { useMutation, useQuery } from "@apollo/client";
import HelpIcon from '@mui/icons-material/Help';
import Tooltip from '@mui/material/Tooltip';
import Modal from '@mui/material/Modal';
@@ -16,9 +16,7 @@ import Box from '@mui/material/Box';
import { gql, useSubscription } from "@apollo/client";
import ArrowBackIcon from '@mui/icons-material/ArrowBack';
import LinearProgress from '@mui/material/LinearProgress';
-
-
-
+import { DateTime } from "luxon";
@@ -36,14 +34,28 @@ const LAST_TEN_UPLOADS_QUERY = gql`
query lastTenUploads($userId: String!) {
lastTenUploads(userId: $userId) {
uploadID
- timestamp
article_cnt
+ message
status
- filename
+ timestamp
+ userID
+
}
}
`;
+
+const UPLOAD_STATUS_UPDATED_SUBSCRIPTION = gql`
+ subscription OnUploadStatusUpdated {
+ uploadStatusUpdated {
+ uploadID
+ status
+ message
+ article_cnt
+ }
+ }
+`;
+
// Define a type for the file with progress information
type UploadedFile = {
name: string;
@@ -98,6 +110,48 @@ export const CSVUploadBox = () => {
},
});
+ const { data, loading, error, refetch: refetchLastTenUploads } = useQuery(LAST_TEN_UPLOADS_QUERY, {
+ variables: { userId: organization ? organization.id : user?.id },
+ skip: !isSignedIn,
+ onCompleted: (data) => {
+ console.log("Fetched data for last ten uploads:", data);
+ },
+ });
+
+
+ useSubscription(UPLOAD_STATUS_UPDATED_SUBSCRIPTION, {
+ onSubscriptionData: ({ subscriptionData }) => {
+ const updatedUpload = subscriptionData.data.uploadStatusUpdated;
+ setUpload((prevUploads: any) =>
+ prevUploads.map((upload: any) =>
+ upload.uploadID === updatedUpload.uploadID
+ ? {
+ ...upload,
+ status: updatedUpload.status,
+ message: updatedUpload.message,
+ article_cnt: updatedUpload.article_cnt,
+ }
+ : upload
+ )
+ );
+ },
+ });
+
+ const extractProgress = (message: any) => {
+ const match = message?.match(/\[(\d+\/\d+)\]/);
+ return match ? match[0] : "[0/0]";
+ };
+
+
+
+ useEffect(() => {
+ if (data) {
+ setUpload(data.lastTenUploads);
+ }
+ }, [data]);
+
+
+
useEffect(() => {
if (isSignedIn && user) {
if (organization) {
@@ -169,7 +223,6 @@ export const CSVUploadBox = () => {
"Headline",
"Publisher",
"Byline",
- "content_id",
"Paths",
"Publish Date",
"Body",
@@ -257,7 +310,6 @@ export const CSVUploadBox = () => {
userId: organization ? organization.id : user.id,
};
- // Set initial progress to indicate the upload has started
setUploadedFiles((prevFiles) =>
prevFiles.map((f) =>
f.name === file.name ? { ...f, progress: 10, status: 'Uploading...' } : f
@@ -266,14 +318,9 @@ export const CSVUploadBox = () => {
const reader = new FileReader();
reader.onload = () => {
- const text = reader.result as string;
- const lines = text.split(/\r?\n/);
- const articleCount = lines.length - 1;
-
uploadCSV({ variables })
.then((response) => {
- if (response.data && response.data.uploadCSV && response.data.uploadCSV.status === 'Success') {
- // Simulate gradual progress update
+ if (response.data?.uploadCSV?.status === 'Success') {
let progress = 10;
const interval = setInterval(() => {
progress += 15;
@@ -285,67 +332,42 @@ export const CSVUploadBox = () => {
)
);
- // Finalize when progress reaches 100%
if (progress >= 100) {
clearInterval(interval);
-
- // Set final status to complete
setUploadedFiles((prevFiles) =>
prevFiles.map((f) =>
f.name === file.name ? { ...f, progress: 100, status: 'Upload complete!' } : f
)
);
- // Display success message
setSuccessMessage("File uploaded successfully!");
-
- // Delay file removal from 'uploaded files' table
setTimeout(() => {
setSuccessMessage("");
-
- // Add to upload history
- setUpload((prevUploads) => [
- ...prevUploads,
- {
- uploadID: Math.random().toString(36).substr(2, 9),
- timestamp: new Date().toISOString(),
- article_cnt: articleCount,
- status: 'Success',
- userID: user.id,
- message: '',
- },
- ]);
-
- // Now remove the file from the 'uploaded files' table
setUploadedFiles((prevFiles) =>
prevFiles.filter((f) => f.name !== file.name)
);
- }, 3000); // Delay for the green success message to display
+
+ // Refetch the latest uploads from MongoDB after each successful upload
+ refetchLastTenUploads();
+ }, 3000);
}
- }, 500); // Adjust interval for progress update frequency
+ }, 500);
} else {
- throw new Error("Unexpected response"); // Force catch for unexpected responses
+ throw new Error("Unexpected response");
}
})
.catch((error) => {
console.error("Error during CSV upload:", error);
-
- // Update progress to stop at failure and set status to failed
setUploadedFiles((prevFiles) =>
prevFiles.map((f) =>
- f.name === file.name
- ? { ...f, progress: 0, status: 'Upload failed!' }
- : f
+ f.name === file.name ? { ...f, progress: 0, status: 'Upload failed!' } : f
)
);
-
- // Show alert message
setAlertMessage("Failed to upload CSV.");
setTimeout(() => setAlertMessage(""), 3000);
});
};
- // Start reading the file to calculate line count
reader.readAsText(file);
} else {
console.error("User is not signed in");
@@ -356,6 +378,8 @@ export const CSVUploadBox = () => {
+
+
@@ -470,6 +494,12 @@ export const CSVUploadBox = () => {
);
}
+ function parseTimestamp(timestamp: any) {
+ // Insert "T" between date and time to make it ISO-compliant
+ const isoTimestamp = timestamp.replace(" ", "T");
+ return new Date(isoTimestamp + "Z");
+ }
+
return (
<div>
<div className='RSS-link'>
@@ -752,7 +782,7 @@ export const CSVUploadBox = () => {
</div>
<div className='file-timeStamp'>
<span>
- {new Date(file.timestamp).toLocaleString()}
+ {file.timestamp ? parseTimestamp(file.timestamp).toLocaleString() : "No Timestamp Available"}
</span>
</div>
<div className='file-articleCnt'>
@@ -761,7 +791,10 @@ export const CSVUploadBox = () => {
</span>
</div>
<div className='file-httpStatus'>
- <span>{file.status}</span>
+ <span>{file.status === "PROCESSING"
+ ? `PROCESSING ${extractProgress(file.message)} ARTICLES`
+ : file.status}
+ </span>
</div>
</div>
))}
diff --git a/package-lock.json b/package-lock.json
index 18dcb78..dbb3a57 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -13,6 +13,7 @@
"graphql-subscriptions": "^2.0.0",
"graphql-upload-minimal": "^1.6.1",
"koa": "^2.15.3",
+ "luxon": "^3.5.0",
"node-fetch": "^2.7.0",
"react-loader-spinner": "^5.4.5",
"react-router-dom": "^6.22.3",
@@ -23,6 +24,7 @@
"@testing-library/jest-dom": "^6.4.2",
"@testing-library/react": "^15.0.5",
"@types/jest": "^29.5.12",
+ "@types/luxon": "^3.4.2",
"@types/xmldom": "^0.1.33",
"jest": "^29.7.0",
"ts-jest": "^29.1.2"
@@ -2131,6 +2133,13 @@
"resolved": "https://registry.npmjs.org/@types/long/-/long-4.0.2.tgz",
"integrity": "sha512-MqTGEo5bj5t157U6fA/BiDynNkn0YknVdh48CMPkTSpFTVmvao5UQmm7uEF6xBEo7qIMAlY/JSleYaE6VOdpaA=="
},
+ "node_modules/@types/luxon": {
+ "version": "3.4.2",
+ "resolved": "https://registry.npmjs.org/@types/luxon/-/luxon-3.4.2.tgz",
+ "integrity": "sha512-TifLZlFudklWlMBfhubvgqTXRzLDI5pCbGa4P8a3wPyUQSW+1xQ5eDsreP9DWHX3tjq1ke96uYG/nwundroWcA==",
+ "dev": true,
+ "license": "MIT"
+ },
"node_modules/@types/mime": {
"version": "1.3.3",
"resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.3.tgz",
@@ -6272,6 +6281,15 @@
"node": ">=12"
}
},
+ "node_modules/luxon": {
+ "version": "3.5.0",
+ "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.5.0.tgz",
+ "integrity": "sha512-rh+Zjr6DNfUYR3bPwJEnuwDdqMbxZW7LOQfUN4B54+Cl+0o5zaU9RJ6bcidfDtC1cWCZXQ+nvX8bf6bAji37QQ==",
+ "license": "MIT",
+ "engines": {
+ "node": ">=12"
+ }
+ },
"node_modules/lz-string": {
"version": "1.5.0",
"resolved": "https://registry.npmjs.org/lz-string/-/lz-string-1.5.0.tgz",
@@ -9542,6 +9560,12 @@
"resolved": "https://registry.npmjs.org/@types/long/-/long-4.0.2.tgz",
"integrity": "sha512-MqTGEo5bj5t157U6fA/BiDynNkn0YknVdh48CMPkTSpFTVmvao5UQmm7uEF6xBEo7qIMAlY/JSleYaE6VOdpaA=="
},
+ "@types/luxon": {
+ "version": "3.4.2",
+ "resolved": "https://registry.npmjs.org/@types/luxon/-/luxon-3.4.2.tgz",
+ "integrity": "sha512-TifLZlFudklWlMBfhubvgqTXRzLDI5pCbGa4P8a3wPyUQSW+1xQ5eDsreP9DWHX3tjq1ke96uYG/nwundroWcA==",
+ "dev": true
+ },
"@types/mime": {
"version": "1.3.3",
"resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.3.tgz",
@@ -12616,6 +12640,11 @@
"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.18.3.tgz",
"integrity": "sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA=="
},
+ "luxon": {
+ "version": "3.5.0",
+ "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.5.0.tgz",
+ "integrity": "sha512-rh+Zjr6DNfUYR3bPwJEnuwDdqMbxZW7LOQfUN4B54+Cl+0o5zaU9RJ6bcidfDtC1cWCZXQ+nvX8bf6bAji37QQ=="
+ },
"lz-string": {
"version": "1.5.0",
"resolved": "https://registry.npmjs.org/lz-string/-/lz-string-1.5.0.tgz",
diff --git a/package.json b/package.json
index 090da92..ff4b077 100644
--- a/package.json
+++ b/package.json
@@ -8,6 +8,7 @@
"graphql-subscriptions": "^2.0.0",
"graphql-upload-minimal": "^1.6.1",
"koa": "^2.15.3",
+ "luxon": "^3.5.0",
"node-fetch": "^2.7.0",
"react-loader-spinner": "^5.4.5",
"react-router-dom": "^6.22.3",
@@ -18,6 +19,7 @@
"@testing-library/jest-dom": "^6.4.2",
"@testing-library/react": "^15.0.5",
"@types/jest": "^29.5.12",
+ "@types/luxon": "^3.4.2",
"@types/xmldom": "^0.1.33",
"jest": "^29.7.0",
"ts-jest": "^29.1.2"
From 6761732adee4e12ed1456f8184ef4cfbf234a6cd Mon Sep 17 00:00:00 2001
From: Nikhil Ramchandani <ramchandaninikhil01@gmail.com>
Date: Sun, 3 Nov 2024 15:23:30 -0500
Subject: [PATCH 3/3] removed import
---
frontend/src/pages/UploadPage/CSVUploadPage.tsx | 1 -
1 file changed, 1 deletion(-)
diff --git a/frontend/src/pages/UploadPage/CSVUploadPage.tsx b/frontend/src/pages/UploadPage/CSVUploadPage.tsx
index ce0ad8e..ba306a7 100644
--- a/frontend/src/pages/UploadPage/CSVUploadPage.tsx
+++ b/frontend/src/pages/UploadPage/CSVUploadPage.tsx
@@ -16,7 +16,6 @@ import Box from '@mui/material/Box';
import { gql, useSubscription } from "@apollo/client";
import ArrowBackIcon from '@mui/icons-material/ArrowBack';
import LinearProgress from '@mui/material/LinearProgress';
-import { DateTime } from "luxon";