Adds new endpoint for jenkins jobs (#17)

* Adds new endpoint for jenkins jobs Adds new tab for Jenkins jobs Abstracts logic to reuse methods when fetching data from ES Signed-off-by: Vicente Zepeda Mas <[email protected]> * Dropping errored runs from the results Signed-off-by: Vicente Zepeda Mas <[email protected]> --------- Signed-off-by: Vicente Zepeda Mas <[email protected]>
cloud-bulldozer · Sep 1, 2023 · 0b6b7ae · 0b6b7ae
1 parent e40aa43
commit 0b6b7ae
Show file tree

Hide file tree

Showing 10 changed files with 121 additions and 82 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1 +1,2 @@
 .idea
+backend/ocpperf.toml
diff --git a/backend/app/api/api.py b/backend/app/api/api.py
@@ -3,8 +3,10 @@
 from app.api.endpoints import results
 from app.api.endpoints import airflow
 from app.api.endpoints import jobs
+from app.api.endpoints import jenkins
 
 api_router = APIRouter()
 api_router.include_router(results.router, tags=['perfscale'])
 api_router.include_router(airflow.router, tags=['perfscale'])
 api_router.include_router(jobs.router, tags=['perfscale'])
+api_router.include_router(jenkins.router, tags=['perfscale'])
diff --git a/backend/app/api/endpoints/common.py b/backend/app/api/endpoints/common.py
@@ -0,0 +1,77 @@
+from datetime import datetime,timedelta
+import trio
+import semver
+import pandas as pd
+from app.services.search import ElasticService
+
+from app.async_util import trio_run_with_asyncio
+
+
+async def getData(type):
+    query = {
+  "query": {
+    "bool": {
+      "must": [
+        {
+          "query_string": {
+            "query": "ciSystem == " + type
+          }
+        },
+        {
+          "range": {
+            "timestamp": {
+              "gte": "now-10d"
+            }
+          }
+        }
+      ]
+    }
+  }
+}
+
+    es = ElasticService()
+    response = await es.post(query)
+    await es.close()
+    tasks = [item['_source'] for item in response["hits"]["hits"]]
+    jobs = pd.json_normalize(tasks)
+    cleanJobs = jobs[jobs['platform'] != ""]
+
+    drop = ['endDate','clusterName','upstreamJobBuild','executionDate',
+            'startDate','buildTag','upstreamJob','releaseStream','timestamp',
+            'jobDuration','networkType','workerNodesType','infraNodesType','masterNodesType',
+            'workerNodesCount','infraNodesCount','masterNodesCount']
+    cleanJobs = cleanJobs.drop(columns=drop)
+    cleanJobs['shortVersion'] = cleanJobs['ocpVersion'].str.slice(0,4)
+
+    df = {'response': group_by_platform(cleanJobs)}
+
+    return df
+
+def group_by_platform(data_frame: pd.DataFrame):
+    return [
+        get_table(group[0], group[1].drop(columns=['platform']))
+        for group in data_frame.groupby(by=['platform'])
+    ]
+
+
+def get_table(title: str, data_frame: pd.DataFrame):
+    return {
+        'title': title,
+        'data' : get_framelist(data_frame)
+    }
+
+def get_framelist(data_frame: pd.DataFrame):
+    return [
+        get_frame(group[0], (group[1].drop(
+            columns=['nodeName', 'ocpVersion'])))
+        for group in data_frame.groupby(by=['shortVersion'])
+    ]
+
+
+def get_frame(title: str, data_frame: pd.DataFrame):
+    return {
+        'version'   : title.title(),
+        'cloud_data': data_frame.values.tolist(),
+        'columns'   : [name.replace('_', ' ').title()
+                       for name in data_frame.columns.tolist()]
+    }
diff --git a/backend/app/api/endpoints/jenkins.py b/backend/app/api/endpoints/jenkins.py
@@ -0,0 +1,11 @@
+from .common import getData
+from fastapi import APIRouter
+
+from app.async_util import trio_run_with_asyncio
+
+router = APIRouter()
+
+@router.post('/api/jenkins')
+@router.get('/api/jenkins')
+async def jobs():
+    return await getData("JENKINS")
diff --git a/backend/app/api/endpoints/jobs.py b/backend/app/api/endpoints/jobs.py
@@ -1,10 +1,5 @@
-from datetime import datetime,timedelta
-import trio
-import semver
-import pprint
-import pandas as pd
+from .common import getData
 from fastapi import APIRouter
-from app.services.search import ElasticService
 
 from app.async_util import trio_run_with_asyncio
 
@@ -13,74 +8,4 @@
 @router.post('/api/jobs')
 @router.get('/api/jobs')
 async def jobs():
-    query = {
-  "query": {
-    "bool": {
-      "must": [
-        {
-          "query_string": {
-            "query": "ciSystem == PROW || ciSystem == JENKINS"
-          }
-        },
-        {
-          "range": {
-            "timestamp": {
-              "gte": "now-10d"
-            }
-          }
-        }
-      ]
-    }
-  }
-}
-
-    es = ElasticService()
-    response = await es.post(query)
-    await es.close()
-    tasks = [item['_source'] for item in response["hits"]["hits"]]
-    jobs = pd.json_normalize(tasks)
-
-
-    drop = ['endDate','clusterName','upstreamJobBuild','executionDate',
-            'startDate','buildTag','upstreamJob','releaseStream','timestamp',
-            'jobDuration','networkType','workerNodesType','infraNodesType','masterNodesType',
-            'workerNodesCount','infraNodesCount','masterNodesCount']
-    jobs = jobs.drop(columns=drop)
-    jobs['shortVersion'] = jobs['ocpVersion'].str.slice(0,4)
-
-    pprint.pprint(jobs)
-
-    cloud_data = []
-
-    df = {'response': group_by_platform(jobs) }
-
-    return df
-
-def group_by_platform(data_frame: pd.DataFrame):
-    return [
-        get_table(group[0], group[1].drop(columns=['platform']))
-        for group in data_frame.groupby(by=['platform'])
-    ]
-
-
-def get_table(title: str, data_frame: pd.DataFrame):
-    return {
-        'title': title,
-        'data' : get_framelist(data_frame)
-    }
-
-def get_framelist(data_frame: pd.DataFrame):
-    return [
-        get_frame(group[0], (group[1].drop(
-            columns=['nodeName', 'ocpVersion'])))
-        for group in data_frame.groupby(by=['shortVersion'])
-    ]
-
-
-def get_frame(title: str, data_frame: pd.DataFrame):
-    return {
-        'version'   : title.title(),
-        'cloud_data': data_frame.values.tolist(),
-        'columns'   : [name.replace('_', ' ').title()
-                       for name in data_frame.columns.tolist()]
-    }
+    return await getData("PROW")
diff --git a/frontend/src/App.js b/frontend/src/App.js
@@ -7,6 +7,7 @@ import {
 } from "react-router-dom";
 import PlatformView from './components/Platform/PlatformView';
 import JobView from './components/Platform/JobView';
+import JenkinsView from './components/Platform/JenkinsView';
 import { Nav, NavItem, NavList, PageHeader } from '@patternfly/react-core';
 import {
   Page,
@@ -32,6 +33,9 @@ export default function App() {
         <NavItem>
           <Link to="/jobs">Prow</Link>
         </NavItem>
+        <NavItem>
+          <Link to="/jenkins">Jenkins</Link>
+        </NavItem>
         <NavItem>
           <Link to="/platform">Airflow</Link>
         </NavItem>
@@ -61,6 +65,9 @@ export default function App() {
               <Route path="/jobs">
                 <JobView />
               </Route>
+              <Route path="/jenkins">
+                <JenkinsView />
+              </Route>
               <Route path="/platform">
                 <PlatformView />
               </Route>

diff --git a/frontend/src/components/Platform/JenkinsView.js b/frontend/src/components/Platform/JenkinsView.js
@@ -0,0 +1,18 @@
+import './PlatformView.css';
+import "@patternfly/react-core/dist/styles/base.css";
+import PlatformTabs from './Tabs/Platform';
+import useESPerfData from '../../hooks/useESPerfData';
+import React from 'react';
+
+
+export default function JenkinsView() {
+  const perfData = useESPerfData("jenkins")
+  return (
+    <>
+    <div className="PlatformView">
+     <PlatformTabs id="PlatformTabs" data={perfData} />
+     </div>
+    </>
+  );
+}
+
diff --git a/frontend/src/components/Platform/JobView.js b/frontend/src/components/Platform/JobView.js
@@ -1,12 +1,12 @@
 import './PlatformView.css';
 import "@patternfly/react-core/dist/styles/base.css";
 import PlatformTabs from './Tabs/Platform';
-import useProwPerfData from '../../hooks/useProwPerfData';
+import useESPerfData from '../../hooks/useESPerfData';
 import React from 'react';
 
 
 export default function JobView() {
-  const perfData = useProwPerfData()
+  const perfData = useESPerfData("jobs")
   return (
     <>
     <div className="PlatformView">

diff --git a/frontend/src/components/Platform/PlatformView.js b/frontend/src/components/Platform/PlatformView.js
@@ -2,7 +2,6 @@ import './PlatformView.css';
 import "@patternfly/react-core/dist/styles/base.css";
 import PlatformTabs from './Tabs/Platform';
 import usePerfData from '../../hooks/usePerfData';
-import useProwPerfData from '../../hooks/useProwPerfData';
 import React from 'react';
 
 

diff --git a/frontend/src/hooks/useProwPerfData.js → frontend/src/hooks/useESPerfData.js b/frontend/src/hooks/useProwPerfData.js → frontend/src/hooks/useESPerfData.js
@@ -1,6 +1,6 @@
 import React, { useState, useEffect } from 'react';
 
-export default function useProwPerfData() {
+export default function useESPerfData(endpoint) {
     const [perfData, setPerfData ] = useState([]);
     const query = {
         "query": {
@@ -20,7 +20,6 @@ export default function useProwPerfData() {
             // body: JSON.stringify(query)
         }
         var hostname = window.location.hostname
-        var endpoint = "jobs"
         if (hostname == "localhost") {
             var host = "http://localhost:8000/api/"+endpoint;
         } else {