Skip to content

Commit

Permalink
grepper form enhancements, incl. progress reports
Browse files Browse the repository at this point in the history 
1. Add a progress bar for parsing of JSON documents
    (but not *yet* for reading of files from the hard drive)
    when at least 50MB of combined text (or at least 16 files with at least 8MB of combined text)
    need to be parsed
2. Make it so hitting Enter in the `Previously viewed directories...` box searches the text of that box
    if it's a valid directory
3. Catch OutOfMemoryExceptions when formatting JSON, to keep Notepad++ from crashing
@molsonkiko
molsonkiko committed Jul 2, 2024
1 parent 3317a81 commit 9bc10cc
Showing 9 changed files with 258 additions and 134 deletions.
11 changes: 11 additions & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
@@ -49,6 +49,17 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
- Since v7.0, holding down `Enter` in a multiline textbox (like the [tree viewer query box](/docs/README.md#remespath)) only adds one newline when the key is lifted.
- Maybe use pre-7.1 (dictionary-based rather than indicator-based) [selection remembering](/docs/README.md#working-with-selections) for Notepad++ 8.5.5 and earlier? Indicators are risky with those older NPP's because of the lack of `NPPM_ALLOCATEINDICATOR`.

## [8.1.0] - (UNRELEASED) YYYY-MM-DD

### Added

1. [Progress reporting](/docs/README.md#reporting-progress-when-parsing-large-amounts-of-json) with the JSON from files and APIs form (henceforth the `grepper form`).
2. In the `grepper form`, pressing `Enter` inside the `Previously viewed directories...` box causes the current text of the box to be searched, assuming that it is a valid directory.

### Fixed

1. If there would be an `OutOfMemoryException` due to running out of memory while formatting JSON (a likely occurrence when using the `grepper form`), that error is caught and reported with a message box, rather than potentially causing Notepad++ to crash.

## [8.0.0] - 2024-06-29

### Added
72 changes: 71 additions & 1 deletion JsonToolsNppPlugin/Forms/GrepperForm.cs
View file
Original file line number Diff line number Diff line change
@@ -19,13 +19,18 @@ public partial class GrepperForm : Form
private readonly FileInfo directoriesVisitedFile;
private readonly FileInfo urlsQueriedFile;
private List<string> urlsQueried;
// fields related to progress reporting
private const int CHECKPOINT_COUNT = 25;
private string bufferOpenBeforeProgressReport;
private string progressReportBufferName;
private static object progressReportLock = new object();

public GrepperForm()
{
InitializeComponent();
NppFormHelper.RegisterFormIfModeless(this, false);
FormStyle.ApplyStyle(this, Main.settings.use_npp_styling);
grepper = new JsonGrepper(Main.JsonParserFromSettings());
grepper = new JsonGrepper(Main.JsonParserFromSettings(), true, CHECKPOINT_COUNT, CreateProgressReportBuffer, ReportJsonParsingProgress, AfterProgressReport);
tv = null;
filesFound = new HashSet<string>();
var configSubdir = Path.Combine(Npp.notepad.GetConfigDirectory(), Main.PluginName);
@@ -160,6 +165,64 @@ private void SearchDirectoriesButton_Click(object sender, EventArgs e)
AddFilesToFilesFound();
}

/// <summary>
/// Looks like this:
/// <code>
/// JSON from files and APIs - JSON parsing progress
/// |======== |
/// 0.301 MB of 1.25 MB parsed
/// </code>
/// </summary>
/// <param name="checkPointsFinished"></param>
/// <returns></returns>
private static string ProgressBarText(int checkPointsFinished, int lengthParsedSoFar, int totalLengthToParse)
{
return "JSON from files and APIs - JSON parsing progress\r\n|" +
ArgFunction.StrMulHelper("=", checkPointsFinished) + ArgFunction.StrMulHelper(" ", CHECKPOINT_COUNT - checkPointsFinished) + "|\r\n" +
$"{Math.Round(lengthParsedSoFar / 1e6, 3)} MB of {Math.Round(totalLengthToParse / 1e6, 3)} MB parsed" +
(lengthParsedSoFar == totalLengthToParse ? "\r\nPARSING COMPLETE!" : "");
}

private void CreateProgressReportBuffer(int totalLengthToParse)
{
bufferOpenBeforeProgressReport = Npp.notepad.GetCurrentFilePath();
Npp.notepad.FileNew();
Npp.notepad.SetCurrentBufferInternalName("JSON parsing progress report");
progressReportBufferName = Npp.notepad.GetCurrentFilePath();
Npp.editor.SetText(ProgressBarText(0, 0, totalLengthToParse));
Npp.editor.GrabFocus();
}

private void ReportJsonParsingProgress(int lengthParsedSoFar, int totalLengthToParse)
{
lock (progressReportLock)
{
// we need to do a / (b / c) rather than a * c / b, because:
// a * (c / b) will be 0 because c is less than b
// (a * c) / b could easily cause overflow, because a * c might be larger than int32.Max
int checkPointsFinished = lengthParsedSoFar / (totalLengthToParse / CHECKPOINT_COUNT);
if (Npp.notepad.GetCurrentFilePath() == progressReportBufferName)
{
Npp.editor.SetText(ProgressBarText(checkPointsFinished, lengthParsedSoFar, totalLengthToParse));
}
}
}

private void AfterProgressReport()
{
//// navigate to whatever buffer was open before the progress report started
//if (progressReportBufferName != null)
//{
// if (Npp.notepad.GetCurrentFilePath() == progressReportBufferName
// && bufferOpenBeforeProgressReport != null
// && Npp.notepad.GetOpenFileNames().Contains(bufferOpenBeforeProgressReport))
// {
// Npp.notepad.OpenFile(bufferOpenBeforeProgressReport);
// }
//}
ViewResultsButton.Focus();
}

private void AddDirectoryToDirectoriesVisited(string rootDir)
{
if (!Directory.Exists(rootDir) || DirectoriesVisitedBox.Items.IndexOf(rootDir) >= 0)
@@ -292,6 +355,13 @@ protected override bool ProcessDialogKey(Keys keyData)
private void GrepperForm_KeyUp(object sender, KeyEventArgs e)
{
NppFormHelper.GenericKeyUpHandler(this, sender, e, false);
// Enter in the DirectoriesVisitedBox selects the entered directory
if (e.KeyCode == Keys.Enter &&
sender is ComboBox cbx && cbx.Name == "DirectoriesVisitedBox"
&& Directory.Exists(cbx.Text))
{
SearchDirectoriesButton.PerformClick();
}
//if (e.Alt)
//{
// switch (e.KeyCode)
139 changes: 61 additions & 78 deletions JsonToolsNppPlugin/JSONTools/JsonGrepper.cs
View file
Original file line number Diff line number Diff line change
@@ -6,7 +6,6 @@
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
using System.Windows.Forms;
using JSON_Tools.Utils;

namespace JSON_Tools.JSON_Tools
@@ -56,9 +55,10 @@ public class JsonGrepper
/// </summary>
private const int PROGRESS_REPORT_TEXT_MIN_TOT_LENGTH = 8_000_000;
/// <summary>
/// when reporting progress using a progress bar, the progress bar is only updated this many times.
/// If the total combined length of all files to be parsed is at least this great,<br></br>
/// show a progress bar <i>even if there are fewer than <see cref="PROGRESS_REPORT_FILE_MIN_COUNT"/> files.</i>
/// </summary>
private const int NUM_PROGRESS_REPORT_CHECKPOINTS = 25;
private const int PROGRESS_REPORT_TEXT_MIN_TOT_LENGTH_IF_LT_MINCOUNT_FILES = 50_000_000;
/// <summary>
/// maps filenames and urls to parsed JSON
/// </summary>
@@ -70,20 +70,29 @@ public class JsonGrepper
/// <summary>
/// the combined length of all files to be parsed
/// </summary>
private int totLengthToParse;
//private int totLengthAlreadyParsed;
//private int checkPointLength;
//private bool reportProgress;
//private Form progressBarForm;
//private ProgressBar progressBar;
///// <summary>
///// could be used to enable the user to cancel grepping by clicking a button
///// when the progress bar pops up.
///// </summary>
//private CancellationToken cancellationToken;
//private int nextCheckPoint;
private int totalLengthToParse;

public JsonGrepper(JsonParser jsonParser = null)
public delegate void ProgressReportSetup(int totalLengthToParse);
public delegate void ProgressReportCallback(int lengthParsedSoFar, int totalLengthToParse);
// constructor fields related to progress reporting
private bool reportProgress;
private ProgressReportSetup progressReportSetup;
private ProgressReportCallback progressReportCallback;
private Action progressReportTeardown;
// derived fields related to progress reporting
private int totLengthAlreadyParsed;
private int progressReportCheckpoints;
private int checkPointLength;
private int nextCheckPoint;

/// <summary></summary>
/// <param name="jsonParser">the parser to use to parse API responses and grepped files</param>
/// <param name="reportProgress">whether to report progress</param>
/// <param name="progressReportCheckpoints">How many times to report progress</param>
/// <param name="progressReportSetup">Anything that must be done before progress reporting starts</param>
/// <param name="progressReportCallback">A function that is called at each progress report checkpoint</param>
/// <param name="progressReportTeardown">Anything that must be done after progress reporting is complete</param>
public JsonGrepper(JsonParser jsonParser = null, bool reportProgress = false, int progressReportCheckpoints = -1, ProgressReportSetup progressReportSetup = null, ProgressReportCallback progressReportCallback = null, Action progressReportTeardown = null)
{
fnameStrings = new Dictionary<string, string>();
fnameJsons = new JObject();
@@ -100,6 +109,12 @@ public JsonGrepper(JsonParser jsonParser = null)
this.jsonParser.throwIfLogged = true;
// security protocol addresses issue: https://learn.microsoft.com/en-us/answers/questions/173758/the-request-was-aborted-could-not-create-ssltls-se.html
ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12;
// configure progress reporting
this.reportProgress = reportProgress;
this.progressReportCheckpoints = progressReportCheckpoints;
this.progressReportSetup = progressReportSetup;
this.progressReportCallback = progressReportCallback;
this.progressReportTeardown = progressReportTeardown;
}

/// <summary>
@@ -119,7 +134,7 @@ private void ReadJsonFiles(string rootDir, bool recursive, params string[] searc
dirInfo = new DirectoryInfo(rootDir);
}
catch { return; }
totLengthToParse = 0;
totalLengthToParse = 0;
SearchOption searchOption = recursive ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly;
foreach (string searchPattern in searchPatterns)
{
@@ -131,16 +146,16 @@ private void ReadJsonFiles(string rootDir, bool recursive, params string[] searc
using (var fp = fileInfo.OpenText())
{
string text = fp.ReadToEnd();
totLengthToParse += text.Length;
TooMuchTextToParseException.ThrowIfTooMuchText(totLengthToParse);
totalLengthToParse += text.Length;
TooMuchTextToParseException.ThrowIfTooMuchText(totalLengthToParse);
fnameStrings[fname] = text;
}
}
}
}
}

private (string fname, JNode parsedOrError, bool error) ParseOrGetError(string fname, string jsonStr, JsonParser jsonParserTemplate)
private (string fname, JNode parsedOrError, bool error) ParseOrGetError(string fname, string jsonStr, JsonParser jsonParserTemplate, bool shouldReportProgress)
{
JNode parsedOrError;
bool error = false;
@@ -154,19 +169,19 @@ private void ReadJsonFiles(string rootDir, bool recursive, params string[] searc
error = true;
parsedOrError = new JNode(ex.ToString());
}
//if (reportProgress)
//{
// // Update the progress bar if the next checkpoint has been reached, then advance the checkpoint.
// // Otherwise, keep moving toward the next checkpoint.
// int alreadyParsed = Interlocked.Add(ref totLengthAlreadyParsed, jsonStr.Length);
// if (alreadyParsed >= nextCheckPoint)
// {
// int newNextCheckPoint = alreadyParsed + checkPointLength;
// newNextCheckPoint = newNextCheckPoint > totLengthToParse ? totLengthToParse : newNextCheckPoint;
// Interlocked.Exchange(ref nextCheckPoint, newNextCheckPoint);
// progressBar.Invoke(new Action(() => { progressBar.Value = alreadyParsed; }));
// }
//}
if (shouldReportProgress && !(progressReportCallback is null))
{
// Update the progress bar if the next checkpoint has been reached, then advance the checkpoint.
// Otherwise, keep moving toward the next checkpoint.
int alreadyParsed = Interlocked.Add(ref totLengthAlreadyParsed, jsonStr.Length);
if (alreadyParsed >= nextCheckPoint)
{
progressReportCallback(alreadyParsed, totalLengthToParse);
int newNextCheckPoint = alreadyParsed + checkPointLength;
newNextCheckPoint = newNextCheckPoint > totalLengthToParse ? totalLengthToParse : newNextCheckPoint;
Interlocked.Exchange(ref nextCheckPoint, newNextCheckPoint);
}
}
return (fname, parsedOrError, error);
}

@@ -176,49 +191,24 @@ private void ReadJsonFiles(string rootDir, bool recursive, params string[] searc
/// </summary>
private void ParseJsonStringsThreaded()
{
bool shouldReportProgress = reportProgress
&& (totalLengthToParse >= PROGRESS_REPORT_TEXT_MIN_TOT_LENGTH_IF_LT_MINCOUNT_FILES
|| (totalLengthToParse >= PROGRESS_REPORT_TEXT_MIN_TOT_LENGTH && fnameStrings.Count >= PROGRESS_REPORT_FILE_MIN_COUNT));
totLengthAlreadyParsed = 0;
checkPointLength = totalLengthToParse / progressReportCheckpoints;
string[] fnames = fnameStrings.Keys.ToArray();
if (shouldReportProgress)
{
progressReportSetup?.Invoke(totalLengthToParse);
progressReportCallback(0, totalLengthToParse);
}
var results = fnames
.Select(fname => ParseOrGetError(fname, fnameStrings[fname], jsonParser))
.Select(fname => ParseOrGetError(fname, fnameStrings[fname], jsonParser, shouldReportProgress))
.AsParallel()
.OrderBy(x => x.fname)
.ToArray();
//// below lines are for progress reporting using a visual bar.
//reportProgress = totLengthToParse >= PROGRESS_REPORT_TEXT_MIN_TOT_LENGTH && fnameStrings.Count >= PROGRESS_REPORT_FILE_MIN_COUNT;
//progressBarForm = null;
//if (reportProgress)
//{
// progressBar = new ProgressBar
// {
// Name = "progress",
// Minimum = 0,
// Maximum = totLengthToParse,
// Style = ProgressBarStyle.Blocks,
// Left = 20,
// Width = 450,
// Top = 200,
// Height = 50,
// };
// string totLengthToParseMB = (totLengthToParse / 1e6).ToString("F3", JNode.DOT_DECIMAL_SEP);
// Label label = new Label
// {
// Name = "title",
// Text = "All JSON documents have been read into memory.\r\n" +
// $"Now parsing {fnameStrings.Count} documents with combined length of about {totLengthToParseMB} MB.",
// TextAlign=ContentAlignment.TopCenter,
// Top = 20,
// AutoSize = true,
// };
// progressBarForm = new Form
// {
// Text = "JSON parsing in progress",
// Controls = { label, progressBar },
// Width=500,
// Height=300,
// };
// progressBarForm.Show();
//}
//totLengthAlreadyParsed = 0;
//checkPointLength = totLengthToParse / NUM_PROGRESS_REPORT_CHECKPOINTS;
if (shouldReportProgress)
progressReportTeardown?.Invoke();
foreach ((string fname, JNode parsedOrError, bool error) in results)
{
if (error)
@@ -227,13 +217,6 @@ private void ParseJsonStringsThreaded()
fnameJsons[fname] = parsedOrError;
}
fnameStrings.Clear(); // don't need the strings anymore, only the JSON
//if (reportProgress)
//{
// progressBarForm.Close();
// progressBarForm.Dispose();
// progressBarForm = null;
// progressBar = null;
//}
}

/// <summary>
4 changes: 4 additions & 0 deletions JsonToolsNppPlugin/JSONTools/RemesPath.cs
View file
Original file line number Diff line number Diff line change
@@ -2313,6 +2313,10 @@ public static string PrettifyException(Exception ex)
{
return $"DSON dump error: {dde.Message}";
}
if (ex is OutOfMemoryException)
{
return $"System.OutOfMemoryException (JsonTools ran out of memory)";
}
string exstr = ex.ToString();
Match isCast = CAST_REGEX.Match(exstr);
if (isCast.Success)
13 changes: 11 additions & 2 deletions JsonToolsNppPlugin/Main.cs
View file
Original file line number Diff line number Diff line change
@@ -381,6 +381,8 @@ public static void OnNotification(ScNotification notification)
if (TryGetInfoForFile(bufferModified, out info) && !(info.tv is null) && !info.tv.IsDisposed)
info.tv.shouldRefresh = true;
break;
// the user changed their native language preference

//if (code > int.MaxValue) // windows messages
//{
// int wm = -(int)code;
@@ -974,8 +976,15 @@ public static void CompressJson()
}
else
{
string newText = formatter(json);
Npp.editor.SetText(newText);
try
{
string newText = formatter(json);
Npp.editor.SetText(newText);
}
catch (Exception ex)
{
MessageBox.Show($"While attempting to reformat the file's JSON, got an error:\r\n{RemesParser.PrettifyException(ex)}");
}
}
info = AddJsonForFile(activeFname, json);
Npp.RemoveTrailingSOH();
4 changes: 2 additions & 2 deletions JsonToolsNppPlugin/Properties/AssemblyInfo.cs
View file
Original file line number Diff line number Diff line change
@@ -28,5 +28,5 @@
// Build Number
// Revision
//
[assembly: AssemblyVersion("8.0.0.0")]
[assembly: AssemblyFileVersion("8.0.0.0")]
[assembly: AssemblyVersion("8.0.0.1")]
[assembly: AssemblyFileVersion("8.0.0.1")]
Loading

0 comments on commit 9bc10cc

Please sign in to comment.