Skip to content

Commit

Permalink
Merge pull request #106 from MichaelPesce/issue-99
Browse files Browse the repository at this point in the history
Add logging panel
MichaelPesce authored Jan 24, 2024
2 parents 6237972 + 03a5ffe commit f58d2a0
Showing 10 changed files with 465 additions and 30 deletions.
9 changes: 7 additions & 2 deletions backend/app/internal/flowsheet_manager.py
Original file line number Diff line number Diff line change
@@ -76,6 +76,7 @@ def __init__(self, **kwargs):
"""
self.app_settings = AppSettings(**kwargs)
self._objs, self._flowsheets = {}, {}
self.startup_time = time.time()

# Add custom flowsheets path to the system path
self.custom_flowsheets_path = Path.home() / ".watertap" / "custom_flowsheets"
@@ -202,7 +203,7 @@ def get_diagram(self, id_: str) -> bytes:

data = b""
info = self.get_info(id_)
_log.info(f"inide get diagram:: info is - {info}")
# _log.info(f"inside get diagram:: info is - {info}")
if info.custom:
# do this
data_path = (
@@ -531,14 +532,18 @@ def add_custom_flowsheets(self):
for f in files:
if "_ui.py" in f:
try:
_log.info(f"attempting to add custom flowsheet module: {f}")
_log.info(f"adding imported flowsheet module: {f}")
module_name = f.replace(".py", "")
custom_module = importlib.import_module(module_name)
fsi = self._get_flowsheet_interface(custom_module)
self.add_flowsheet_interface(module_name, fsi, custom=True)
except Exception as e:
_log.error(f"unable to add flowsheet module: {e}")

def get_logs_path(self):
"""Return logs path."""
return self.app_settings.log_dir

@staticmethod
def _get_flowsheet_interface(module: ModuleType) -> Optional[FlowsheetInterface]:
"""Get a a flowsheet interface for module.
46 changes: 46 additions & 0 deletions backend/app/internal/log_parser.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
import time

def parse_logs(logs_path, time_since):
"""
Assume a log format of:
"[%(levelname)s] %(asctime)s %(name)s (%(filename)s:%(lineno)s): %(message)s"
"""
result = []
log_entries = []
log_file = open(logs_path, 'r')
all_logs = log_file.read()
log_file.close()
logs = all_logs.split('\n[')
for line in logs:
try:
log_split = line.split(' ')
log_time = log_split[1:3]
log_time_string = f'{log_time[0]} {log_time[1]}'.split(',')[0]
stripped_time = time.strptime(log_time_string, "%Y-%m-%d %H:%M:%S")
asctime = time.mktime(stripped_time)
if asctime > time_since:
result.append(line)
log_level = line.split(']')[0]
log_name = log_split[3]
log_file_lineno = log_split[4]
log_file = log_file_lineno.split(":")[0]
log_lineno = log_file_lineno.split(":")[1]
log_message = line.split(log_file_lineno)[1]
if len(log_file) > 0:
log_file = log_file[1:]
if len(log_lineno) > 0:
log_lineno = log_lineno[:-1]
if len(log_message) > 0:
log_message = log_message[1:]
log_entry = {
"log_time": asctime,
"log_level": log_level,
"log_name": log_name,
"log_file": log_file,
"log_lineno": log_lineno,
"log_message": log_message,
}
log_entries.append(log_entry)
except Exception as e:
print(f'unable to parse log line: {e}')
return log_entries
2 changes: 1 addition & 1 deletion backend/app/internal/settings.py
Original file line number Diff line number Diff line change
@@ -31,7 +31,7 @@ def validate_log_dir(cls, v):
v.mkdir(parents=True, exist_ok=True)

loggingFormat = "[%(levelname)s] %(asctime)s %(name)s (%(filename)s:%(lineno)s): %(message)s"
loggingFileHandler = logging.handlers.RotatingFileHandler(v / "ui_backend_logs.log", backupCount=2, maxBytes=5000000)
loggingFileHandler = logging.handlers.RotatingFileHandler(v / "watertap-ui_backend_logs.log", backupCount=2, maxBytes=5000000)
logging.basicConfig(level=logging.INFO, format=loggingFormat, handlers=[loggingFileHandler])
return v

41 changes: 32 additions & 9 deletions backend/app/routers/flowsheets.py
Original file line number Diff line number Diff line change
@@ -2,7 +2,6 @@
Handle flowsheet-related API requests from web client.
"""
# stdlib
import csv
import io
import aiofiles
from pathlib import Path
@@ -15,16 +14,19 @@
import pandas as pd
from pydantic import BaseModel
from pydantic.error_wrappers import ValidationError
import re

# package-local
from app.internal.flowsheet_manager import FlowsheetManager, FlowsheetInfo
from app.internal.parameter_sweep import run_parameter_sweep
from app.internal.log_parser import parse_logs
from watertap.ui.fsapi import FlowsheetInterface, FlowsheetExport
import idaes.logger as idaeslog

CURRENT = "current"

_log = idaeslog.getLogger(__name__)
_solver_log = idaeslog.getLogger(__name__+'.solver')

router = APIRouter(
prefix="/flowsheets",
@@ -126,10 +128,12 @@ async def solve(flowsheet_id: str, request: Request):

# run solve
try:
flowsheet.solve()
with idaeslog.solver_log(_log, level=idaeslog.INFO) as slc:
flowsheet.solve()
# set last run in tiny db
flowsheet_manager.set_last_run(info.id_)
except Exception as err:
_log.error(f"Solve failed: {err}")
raise HTTPException(500, detail=f"Solve failed: {err}")
return flowsheet.fs_exp

@@ -163,10 +167,11 @@ async def sweep(flowsheet_id: str, request: Request):
info.updated(built=True)

_log.info("trying to sweep")
results_table = run_parameter_sweep(
flowsheet=flowsheet,
info=info,
)
with idaeslog.solver_log(_log, level=idaeslog.INFO) as slc:
results_table = run_parameter_sweep(
flowsheet=flowsheet,
info=info,
)
flowsheet.fs_exp.sweep_results = results_table
# set last run in tiny db
flowsheet_manager.set_last_run(info.id_)
@@ -264,11 +269,9 @@ async def upload_flowsheet(files: List[UploadFile]) -> str:
try:
# get file contents
new_files = []

print("trying to read files with aiofiles")
for file in files:
# for file in files:
print(file.filename)
_log.info(f'reading {file.filename}')
new_files.append(file.filename)
if "_ui.py" in file.filename:
new_id = file.filename.replace(".py", "")
@@ -472,3 +475,23 @@ async def download_sweep(flowsheet_id: str) -> Path:
df.to_csv(path, index=False)
# # User can now download the contents of that file
return path

@router.get("/get_logs")
async def get_logs() -> List[str]:
"""Get backend logs.
Returns:
Logs formatted as a list
"""
logs_path = flowsheet_manager.get_logs_path() / "watertap-ui_backend_logs.log"
return parse_logs(logs_path, flowsheet_manager.startup_time)

@router.post("/download_logs", response_class=FileResponse)
async def download_logs() -> Path:
"""Download full backend logs.
Returns:
Log file
"""
logs_path = flowsheet_manager.get_logs_path() / "watertap-ui_backend_logs.log"
return logs_path
23 changes: 23 additions & 0 deletions electron/ui/package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions electron/ui/package.json
Original file line number Diff line number Diff line change
@@ -23,6 +23,7 @@
"react": "^18.1.0",
"react-dom": "^18.1.0",
"react-drag-drop-files": "^2.3.10",
"react-draggable": "^4.4.6",
"react-plotly.js": "^2.6.0",
"react-plotlyjs": "^0.4.4",
"react-router-dom": "^6.3.0",
31 changes: 19 additions & 12 deletions electron/ui/public/main.js
Original file line number Diff line number Diff line change
@@ -17,8 +17,16 @@ let uiReady = false
const serverURL = `http://localhost:${PY_PORT}`
const uiURL = `http://localhost:${UI_PORT}`

log.transports.file.resolvePath = () => path.join(__dirname, '/logsmain.log');
if(isDev) {
log.transports.file.resolvePath = () => path.join(__dirname, '/logsdev.log');
} else {
log.transports.file.resolvePath = () => path.join(__dirname, '/logsmain.log');
}

log.transports.file.level = "info";
// log.transports.console.format = '{h}:{i}:{s} {text}'
log.transports.console.format = '{text}'
log.transports.file.format = '{text}'

exports.log = (entry) => log.info(entry)

@@ -62,6 +70,7 @@ function createWindow() {
}

console.log("storing user preferences in: ",app.getPath('userData'));
log.info("storing user preferences in: ",app.getPath('userData'))

// save size of window when resized
win.on("resized", () => saveBounds(win.getSize()));
@@ -85,22 +94,20 @@ const installExtensions = () => {
]
);

log.info("installation started");
console.log("installation started");
log.info("installing idaes extensions");
console.log("installing idaes extensions");

var scriptOutput = "";
installationProcess.stdout.setEncoding('utf8');
installationProcess.stdout.on('data', function(data) {
// console.log('stdout: ' + data);
log.info('stdout: ' + data);
log.info(data);
data=data.toString();
scriptOutput+=data;
});

installationProcess.stderr.setEncoding('utf8');
installationProcess.stderr.on('data', function(data) {
// console.log('stderr: ' + data);
log.info('stderr: ' + data);
log.info(data);
data=data.toString();
scriptOutput+=data;
});
@@ -124,7 +131,7 @@ const startServer = () => {
{
cwd: '../backend/app'
}
);
);
// log.info("Python process started in dev mode");
// console.log("Python process started in dev mode");
} else {
@@ -139,15 +146,15 @@ const startServer = () => {
backendProcess.stdout.setEncoding('utf8');
backendProcess.stdout.on('data', function(data) {
console.log('stdout: ' + data);
log.info('stdout: ' + data);
log.info(data);
data=data.toString();
scriptOutput+=data;
});

backendProcess.stderr.setEncoding('utf8');
backendProcess.stderr.on('data', function(data) {
console.log('stderr: ' + data);
log.info('stderr: ' + data);
log.info(data);
data=data.toString();
scriptOutput+=data;
});
@@ -176,8 +183,8 @@ app.whenReady().then(() => {
let serverProcess
let installationProcess = installExtensions()
installationProcess.on('exit', code => {
log.info('installation exit code is', code)
console.log('installation exit code is', code)
// log.info('installation exit code is', code)
// console.log('installation exit code is', code)
log.info('starting server')
console.log('starting server')
serverProcess = startServer()
36 changes: 31 additions & 5 deletions electron/ui/src/components/Boilerplate/Header/Header.js
Original file line number Diff line number Diff line change
@@ -1,14 +1,31 @@
import './Header.css';
import React from 'react';
import logo from "../../../assets/nawi-logo-color.png";
import LoggingPanel from '../../LoggingPanel/LoggingPanel';
import { useNavigate } from "react-router-dom";
import Button from '@mui/material/Button';

import { Button, Menu, MenuItem, IconButton } from '@mui/material';
import ListIcon from '@mui/icons-material/List';

export default function Header(props) {
let navigate = useNavigate();
const [ showLogs, setShowLogs ] = React.useState(false)
const [ actionsList, setActionsList ] = React.useState(false)
const [ anchorEl, setAnchorEl ] = React.useState(null);

const handleNavigateHome = () => {
// setActionsList(!actionsList)
navigate("/flowsheets", {replace: true})
}

const handleShowLogs = () => {
setShowLogs(!showLogs)
setActionsList(false)
}

const handleShowActions = (event) => {
setActionsList(!actionsList)
setAnchorEl(event.currentTarget);
}
return (
props.show &&
<div id="Header">
@@ -20,11 +37,20 @@ export default function Header(props) {
<div id="titlebar-name">
WaterTAP
</div>
<div className="right" >
<Button style={{ color:"white" }} onClick={handleNavigateHome}>Return to list page</Button>
<div className="right" >
<IconButton style={{ color:"white" }} onClick={handleShowActions}><ListIcon/></IconButton>
<Menu
id="actions-list"
anchorEl={anchorEl}
open={actionsList}
onClose={() => setActionsList(false)}
>
<MenuItem onClick={handleShowLogs}>View Logs</MenuItem>
<MenuItem onClick={handleNavigateHome}>Return to list page</MenuItem>
</Menu>
</div>
</div>

<LoggingPanel open={showLogs} onClose={handleShowLogs}/>
</div>
);
}
290 changes: 290 additions & 0 deletions electron/ui/src/components/LoggingPanel/LoggingPanel.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,290 @@
import {useEffect, useState, useRef } from 'react';
import { InputAdornment, TextField, IconButton, Tooltip, MenuItem, Checkbox, ListItemText, Menu } from '@mui/material';
import { Dialog, DialogTitle, DialogContent, DialogContentText, DialogActions, Typography } from '@mui/material';
import CloseIcon from '@mui/icons-material/Close';
import { getLogs, downloadLogs } from '../../services/flowsheet.service';
import Draggable from 'react-draggable';
import FullscreenIcon from '@mui/icons-material/Fullscreen';
import FullscreenExitIcon from '@mui/icons-material/FullscreenExit';
import DownloadIcon from '@mui/icons-material/Download';
import SearchIcon from '@mui/icons-material/Search';
import FilterListIcon from '@mui/icons-material/FilterList';


export default function LoggingPanel(props) {
const { open, onClose } = props;
const [ logData, setLogData ] = useState([])
const [ dialogHeight, setDialogHeight ] = useState('60vh')
const [ dialogWidth, setDialogWidth ] = useState('60vw')
const [ fullscreen, setFullscreen] = useState(false)
const [ searchTerm, setSearchTerm ] = useState("")
const [ filters, setFilters ] = useState(["DEBUG", "INFO", "WARNING", "ERROR"])
const [ showFilters, setShowFilters ] = useState(false)
const [ anchorEl, setAnchorEl ] = useState(null);
const divRef = useRef(null);

useEffect(() => {
if (open)(
getLogs()
.then(response => response.json())
.then((data) => {
console.log('got logs: ')
setLogData(data)
// console.log(data)
// window.scrollTo(0, document.body.scrollHeight);
})
)

},[props])

useEffect(() => {
if (open) {
divRef.current.scrollIntoView({ behavior: 'smooth' });
}

},[logData])



const styles = {
dialogTitle: {
backgroundColor: "black",
color: "white",
},
dialogContent: {
backgroundColor: "black",
color: "white",
},
dialogContentText: {
backgroundColor: "black",
color: "white",
// overflowX: "auto"
},
dialog: {
// maxWidth: "80vw",
},
dialogPaper: {
minHeight: dialogHeight,
maxHeight: dialogHeight,
minWidth: dialogWidth,
maxWidth: dialogWidth,
},
DEBUG: {
color: "#3B90FF",
},
INFO: {
color: "#28FF24",
},
WARNING: {
color: "#FFF42C",
},
ERROR: {
color: "#FF042E",
}
}

const handleClose = () => {
setSearchTerm("")
setShowFilters(false)
setFilters(["DEBUG", "INFO", "WARNING", "ERROR"])
if(fullscreen) handleFullscreen()
onClose()
};

const handleFullscreen = () => {
if (fullscreen) {
setDialogHeight('60vh')
setDialogWidth('60vw')
} else {
setDialogHeight('100vh')
setDialogWidth('100vw')
}
setFullscreen(!fullscreen)
}

const handleDownloadLogs = () => {
downloadLogs().then(response => response.blob())
.then((data) => {
console.log(data)
let logsUrl = window.URL.createObjectURL(data);
let tempLink = document.createElement('a');
tempLink.href = logsUrl;
tempLink.setAttribute('download', 'watertap-ui-logs.log');
tempLink.click();
})
};

const handleShowLogFilters = (event) => {
setShowFilters(!showFilters)
setAnchorEl(event.currentTarget);
}

const handleFilter = (level) => {
let tempFilters = [...filters]
const index = tempFilters.indexOf(level);
if (index > -1) {
tempFilters.splice(index, 1);
} else {
tempFilters.push(level)
}
setFilters(tempFilters)
}

const getTextColor = (line) => {
if (line.includes('ERROR')) return styles.ERROR.color
else if (line.includes('INFO')) return styles.INFO.color
else if (line.includes('DEBUG')) return styles.DEBUG.color
else if (line.includes('WARNING')) return styles.WARNING.color
else return "white"
}

const descriptionElementRef = useRef(null);
useEffect(() => {
if (open) {
const { current: descriptionElement } = descriptionElementRef;
if (descriptionElement !== null) {
descriptionElement.focus();
}
}
}, [open]);

return (
<Draggable handle="#console-dialog">
<Dialog
open={open}
onClose={handleClose}
scroll={"paper"}
aria-labelledby="console-dialog"
aria-describedby="console-dialog-description"
PaperProps={{
sx: styles.dialogPaper
}}
BackdropProps={{
sx: {backgroundColor: "transparent"}
}}
>
<DialogTitle id="dialog-title" style={styles.dialogTitle}>Backend Logs</DialogTitle>

<TextField id={'searchBar'}
label={'Search'}
variant="outlined"
size="small"
value={searchTerm}
onChange={(event) => setSearchTerm(event.target.value)}
sx={{
position: 'absolute',
right: 160,
top: 12,
color: "white",
backgroundColor: "#292f30",
borderRadius: 10,
input: { color: 'white' },
}}
InputProps={{
startAdornment: (
<InputAdornment position="start" sx={{color: "white"}}>
<SearchIcon />
</InputAdornment>
),
}}
/>
<IconButton
aria-label="close"
onClick={handleShowLogFilters}
sx={{
position: 'absolute',
right: 120,
top: 8,
color: "white",
}}
>
<FilterListIcon/>
</IconButton>
<Menu
id="log-filter"
anchorEl={anchorEl}
open={showFilters}
onClose={() => setShowFilters(false)}
sx={{
"& .MuiPaper-root": {
backgroundColor: "#292f30"
}
}}
>
{["DEBUG", "INFO", "WARNING", "ERROR"].map((loglevel, idx) => (
<MenuItem key={loglevel} value={loglevel} onClick={() => handleFilter(loglevel)} sx={{color: "white"}}>
<Checkbox
checked={filters.includes(loglevel)}
sx={{
color: styles[loglevel].color,
'&.Mui-checked': {
color: styles[loglevel].color,
},
}}
/>
<ListItemText primary={loglevel} />
</MenuItem>
))}
</Menu>

<Tooltip title={"Download full logs"}>
<IconButton
aria-label="close"
onClick={handleDownloadLogs}
sx={{
position: 'absolute',
right: 80,
top: 8,
color: "white",
}}
>
<DownloadIcon/>
</IconButton>
</Tooltip>
<IconButton
aria-label="close"
onClick={handleFullscreen}
sx={{
position: 'absolute',
right: 40,
top: 8,
color: "white",
}}
>
{fullscreen ? <FullscreenExitIcon/> : <FullscreenIcon />}
</IconButton>
<IconButton
aria-label="close"
onClick={handleClose}
sx={{
position: 'absolute',
right: 0,
top: 8,
color: "white",
}}
>
<CloseIcon />
</IconButton>
<DialogContent style={styles.dialogContent} dividers={true}>
<DialogContentText
id="scroll-dialog-description"
ref={descriptionElementRef}
tabIndex={-1}
style={styles.dialogContentText}
aria-labelledby="console-dialog-content-text"
component="span"
>
{logData.map((line, idx) => {
if (line.log_message.toLowerCase().includes(searchTerm.toLowerCase()) && filters.includes(line.log_level)) {
return <Typography style={{color: getTextColor(line.log_level), overflowWrap: "break-word"}} key={idx}>[{line.log_level}] {line.log_name}: {line.log_message}</Typography>
}

})}
<div id="bottom-div" ref={divRef} ></div>
</DialogContentText>
</DialogContent>
</Dialog>
</Draggable>
)
}

16 changes: 15 additions & 1 deletion electron/ui/src/services/flowsheet.service.js
Original file line number Diff line number Diff line change
@@ -36,4 +36,18 @@ export const selectOption = (id, data) => {
mode: 'cors',
body: JSON.stringify(data)
});
};
};

export const getLogs = () => {
return fetch('http://localhost:8001/flowsheets/get_logs', {
method: 'GET',
mode: 'cors'
});
}

export const downloadLogs = () => {
return fetch('http://localhost:8001/flowsheets/download_logs', {
method: 'POST',
mode: 'cors'
});
}

0 comments on commit f58d2a0

Please sign in to comment.