Fix Manual Syncing Tool - (#127)

disqus/rest-api/class-disqus-rest-api.php

@@ -700,7 +700,6 @@ private function update_comment_from_post( $post, $comments ) {
         // Remove non-updating fields.
         unset( $comment_data['comment_meta'] );
         unset( $comment_data['comment_agent'] );
-        unset( $comment_data['comment_parent'] );
         unset( $comment_data['comment_type'] );
         unset( $comment_data['comment_date_gmt'] );
         unset( $comment_data['comment_post_ID'] );
@@ -763,11 +762,9 @@ private function comment_data_from_post( $post ) {
             update_post_meta( $wp_post_id, 'dsq_thread_id', $thread['id'] );
         }
 
-        if ( null === $wp_post_id || false == $wp_post_id ) {
-            throw new Exception( 'No post found associated with the thread.' );
-        }
-
         // Find the parent comment, if any.
+        // To simplify our syncing process and prevent syncing errors,
+        // still sync the comment even if we don't have its parent comment synced.
         $parent = 0;
         if ( null !== $post['parent'] ) {
             $parent_comment_query = new WP_Comment_Query( array(
@@ -777,9 +774,7 @@ private function comment_data_from_post( $post ) {
             ) );
             $parent_comments = $parent_comment_query->comments;
 
-            if ( empty( $parent_comments ) ) {
-                throw new Exception( 'This comment\'s parent has not been synced yet.' );
-            } else {
+            if ( $parent_comments ) {
                 $parent = $parent_comments[0]->comment_ID;
             }
         }

frontend/src/ts/DisqusApi.ts

@@ -54,6 +54,7 @@ export class DisqusApi {
             `end=${endDate.toISOString()}`,
             `forum=${this.forum}`,
             'related=thread',
+            'order=asc',
             `limit=${Math.min(Math.max(limit, 1), 100)}`,
             `cursor=${cursor || ''}`,
         ].join('&');

frontend/src/ts/WordPressRestApi.ts

@@ -121,7 +121,7 @@ export class WordPressRestApi {
                         status: XHR.status,
                         statusText: XHR.statusText
                     });
-                    console.error('Error', XHR.statusText);
+                    console.error('Error Status: ', XHR.statusText, '100 character server response preview: ', XHR.responseText.substring(0, 100));
                 }
             };
             XHR.send(data);

frontend/src/ts/components/ManualSyncForm.tsx

@@ -11,7 +11,7 @@ const ManualSyncForm = (props: IFormProps) => {
         >
             <h4>{__('Manually Sync Comments')}</h4>
             <p className='description'>
-                {__('Select a time range to sync past comments. Date ranges are limited to a maximum of 12 months.')}
+                {__('Select a time range to sync past comments. Date ranges can go up to a maximum of 5 years.')}
             </p>
             <table className='form-table'>
                 <tbody>
@@ -30,7 +30,7 @@ const ManualSyncForm = (props: IFormProps) => {
                                 value={props.data.manualSyncRangeStart}
                                 onChange={props.onDateSelectorInputchange.bind(null, 'manualSyncRangeStart')}
                                 max={props.data.manualSyncRangeEnd}
-                                min={moment(props.data.manualSyncRangeEnd).subtract(12, 'months').format('YYYY-MM-DD')}
+                                min={moment(props.data.manualSyncRangeEnd).subtract(60, 'months').format('YYYY-MM-DD')}
                                 disabled={props.data.isManualSyncRunning}
                             />
                             <p className='description'>

frontend/src/ts/containers/mapDispatchToProps.ts

@@ -49,12 +49,11 @@ const valueFromInput = (element: HTMLInputElement): string => {
 let syncedComments = 0;
 let totalSyncedComments = 0;
 
-const syncComments = async (commentQueue: any[], dispatch: Redux.Dispatch<Redux.Action>) => {
+const syncComments = async (commentQueue: any[], dispatch: Redux.Dispatch<Redux.Action>, commentType: String) => {
     // We need to throttle the amount of parallel sync requests that we make
     // because large forums could be syncing thousands of comments
     const maxParallelRequests = 100;
     const parallelRequests: Promise <void> [] = [];
-
     for (let comment of commentQueue) {
         // Make the sync request and add its promise to the queue,
         // and remove it from queue when complete
@@ -76,6 +75,8 @@ const syncComments = async (commentQueue: any[], dispatch: Redux.Dispatch<Redux.
         ).then(() => {
             parallelRequests.splice(parallelRequests.indexOf(requestPromise), 1);
             syncedComments += 1;
+        }).catch (() => {
+            console.error('could not sync comment: ', comment);
         });
         parallelRequests.push(requestPromise);
 
@@ -85,12 +86,14 @@ const syncComments = async (commentQueue: any[], dispatch: Redux.Dispatch<Redux.
             await Promise.race(parallelRequests);
         }
     }
-    // Wait for all of the requests to finish
     Promise.all(parallelRequests).then(() => {
-        dispatch(setValueAction('isManualSyncRunning', false));
+        // We sync parent comments first then child comments immediately after, so we only update the state of the sync after child comments
+        if (commentType === 'childComments') {
+            dispatch(setValueAction('isManualSyncRunning', false));
+        }
         dispatch(updateSyncStatusAction({
             is_manual: true,
-            progress_message: `Complete (${syncedComments} of ${totalSyncedComments})`,
+            progress_message: `Completed (${syncedComments} of ${totalSyncedComments})`,
         }));
     });
 }
@@ -151,7 +154,8 @@ const mapDispatchToProps = (dispatch: Redux.Dispatch<Redux.Action>) => {
             const endDate: Moment = getDateFromInput(rangeEndInput);
 
             // Create a queue of comments within the provided date-range from the Disqus API
-            let commentQueue: any[] = [];
+            const parentCommentList: any[] = [];
+            const childCommentList: any[] = [];
             const getDisqusComments = async (cursor: string = '') => {
                 return new Promise((resolve, reject) => {
                     DisqusApi.instance.listPostsForForum(cursor, startDate, endDate, 100, async (xhr: Event) => {
@@ -171,33 +175,43 @@ const mapDispatchToProps = (dispatch: Redux.Dispatch<Redux.Action>) => {
                                 last_message: null,
                             }));
                         }
-
+                        
                         const pendingComments = disqusData.response;
                         totalSyncedComments += pendingComments.length;
 
                         pendingComments.forEach((comment: any) => {
-                            commentQueue.push(comment);
+                            if (comment.parent) {
+                                childCommentList.push(comment);
+                            } else {
+                                parentCommentList.push(comment);
+                            }
                         });
 
                         const nextCursor = disqusData.cursor;
                         if (nextCursor && nextCursor.hasNext) {
                             await getDisqusComments(nextCursor.next);
                         }
-                        resolve(commentQueue);
+                        resolve([parentCommentList, childCommentList]);
                     });
                 });
             };
 
             dispatch(setValueAction('isManualSyncRunning', true));
-            getDisqusComments().then((commentQueue: any[]) => {
-                syncComments(commentQueue, dispatch);
-            }).catch((err: any) => {
-                dispatch(setMessageAction({
-                    onDismiss: handleClearMessage,
-                    text: (err && err.response) || 'Error connecting to the Disqus API',
-                    type: 'error',
-                }));
-            });
+            getDisqusComments()
+                .then((commentQueue: any[]) => {
+                    syncComments(commentQueue[0], dispatch, 'parentComments');
+                    return commentQueue;
+                })
+                .then((commentQueue: any[]) => {
+                    syncComments(commentQueue[1], dispatch, 'childComments');
+                })
+                .catch((err: any) => {
+                    dispatch(setMessageAction({
+                        onDismiss: handleClearMessage,
+                        text: (err && err.response) || 'Error connecting to the Disqus API',
+                        type: 'error',
+                    }));
+                });
         },
         onSubmitSiteForm: (event: React.SyntheticEvent<HTMLFormElement>) => {
             event.preventDefault();