From 4db3a09ca4dc334d528f15ede995aa23b2999bd8 Mon Sep 17 00:00:00 2001 From: AnandDas227 Date: Tue, 25 Jun 2024 08:21:53 -0700 Subject: [PATCH] push wxa test automation to public github --- README.md | 89 ++++++++++++++++++++++++++ env | 5 ++ main.py | 156 ++++++++++++++++++++++++++++++++++++++++++++++ requirements.txt | 9 +++ sample_input.xlsx | Bin 0 -> 9286 bytes 5 files changed, 259 insertions(+) create mode 100644 README.md create mode 100644 env create mode 100644 main.py create mode 100644 requirements.txt create mode 100644 sample_input.xlsx diff --git a/README.md b/README.md new file mode 100644 index 0000000..1f07e17 --- /dev/null +++ b/README.md @@ -0,0 +1,89 @@ + + +# watsonx Assistant Testing Automation Tool + +The watsonx Assistant Testing Automation Tool is used to batch test question groups against a watsonx Assistant instance. The script ingests an excel file with questions, uses the watsonx Assistant API to query the assistant, and outputs an excel file with the results. The goal of the tool is to reduce execution time for running tests and identifying potential recurring errors. + + + +## Getting Started + +### Prerequisites + +The following prerequisites are required to run the tester: + +1. Python3 +2. IBM Cloud api key (this must be for the same cloud account that hosts the assistant instance) +3. watsonx Assistant service instance url +4. watsonx Assistant environment id + +### Installation + +1. Clone the repo + + ```bash + git clone git@github.ibm.com:EE-WW-BuildLab/wxa-test-automation-tool.git + ``` + +2. Change directory into wxa-test-automation-tool + + ```bash + cd wxa-test-automation-tool + ``` + +3. Create a python virtual environment + + ```bash + python3 -m venv virtual-env + source virtual-env/bin/activate + pip3 install -r requirements.txt + ``` + +4. Copy env file to .env + + ```bash + cp env .env + ``` + +5. Configure parameters in .env and set the values of your environment + 1. For your "input_data_file", see [Configuring Your Input Excel File](#configuring-your-input-excel-file) + 2. You can name your "output_data_file" anything + 3. Ensure both the input and output file names are of type .xlsx + +6. Run the following to start the script + + ```bash + python3 main.py + ``` + +7. Run the following command to exit the python virtual environment: + + ```bash + deactivate + ``` + +## Configuring your Input Excel File + +The repository contains a sample input file that you can copy, edit, and use to test. + +The input excel file must have the following three columns (**note:** they are spelling and case sensitive): + +1. "Question Groups" + 1. This column acts as a label for questions that should be asked to the assistant in one session. + 2. Example values: + 1. "Group 1" + 2. "Group 2" + 3. "Group 3" +2. "Question" + 1. This column contains the question to be asked to the assistant. + +## Understanding the Results + +You can observe real time results in the terminal. Each time a questions is asked, you can view the input, response time, and output. + +When all tests are completed, an output excel file with your results is created using the name specified in your env file. The output file contains the question groups, questions, assistant results, error flags, and response times. + +### Error Flags + +1. Processing error: occurs when the assistant returns a general processing error +2. Timeout error: occurs when the assistant response takes greater than 30 seconds diff --git a/env b/env new file mode 100644 index 0000000..a87864c --- /dev/null +++ b/env @@ -0,0 +1,5 @@ +input_data_file="sample_input.xlsx" +output_data_file="sample_output.xlsx" +api_key= "IBM_Cloud_API_KEY" +assistant_url = "service_instance_url" +assistant_environment_id = "environment_id" \ No newline at end of file diff --git a/main.py b/main.py new file mode 100644 index 0000000..fca44c9 --- /dev/null +++ b/main.py @@ -0,0 +1,156 @@ +import os +import time +from dotenv import load_dotenv +import numpy as np +import pandas as pd +import requests +from requests.auth import HTTPBasicAuth + +load_dotenv() + +testing_data = pd.read_excel(os.getenv("input_data_file")) + +# replaces NaN with None for future string comparison logic +testing_data = testing_data.replace({pd.NaT: None, np.nan: None}) + +# filter our dataset to only include rows with groups +testing_data = testing_data[testing_data['Question Groups'] != 'None'] + +testing_data['Assistant Output'] = 'None' +testing_data['Error Flags'] = 'None' +testing_data['Response Times'] = 'None' + +question_groups_list = [] +for value in testing_data['Question Groups'].unique(): + grouped_df = testing_data[testing_data['Question Groups'] == value].copy() + grouped_df = grouped_df[['Question Groups', 'Question', 'Assistant Output', 'Error Flags', 'Response Times']] + question_groups_list.append(grouped_df) + + +apikey = os.getenv("api_key") +assistant_url = os.getenv("assistant_url") +assistant_environment_id = os.getenv("assistant_environment_id") + +# authenticating to wxa instance +auth = HTTPBasicAuth("apikey", apikey) + +wxa_params = { + "version": "2023-07-15" +} + +# watsonx assistant api functions for a batch test + +# creates an assistant session, 1 per question group +def create_assistant_session(): + url = f'{assistant_url}/v2/assistants/{assistant_environment_id}/sessions' + + try: + response = requests.post(url, params=wxa_params, auth=auth).json() + session_id = response['session_id'] + except Exception as e: + print(f"An error occurred: {e}") + + return session_id + +# queries the assistant +def query_assistant(session_id, query): + print('-' * 41) + print(f'1) Input:\n\n{query}\n') + start = time.time() + + # wxa api message request + url = f'{assistant_url}/v2/assistants/{assistant_environment_id}/sessions/{session_id}/message' + headers = { + "Content-Type": "application/json" + } + data = { + "input": { + "text": query + } + } + + try: + response = requests.post(url, params=wxa_params, headers=headers, json=data, auth=auth).json() + except Exception as e: + print(f"An error occurred: {e}") + + end = time.time() + query_results = "" + print("2) Output:\n") + for index, item in enumerate(response["output"]["generic"], 1): + if item["response_type"] == "text": + print(item['text'] + "\n") + query_results += item['text'] + "\n\n" + query_results += "\n" + + api_response_time = round(end - start,2) + print(f'3) API response time:\n\n{str(api_response_time)}\n') + + return query_results, api_response_time + +# deletes an assistant session +def delete_assistant_session(session_id): + url = f'{assistant_url}/v2/assistants/{assistant_environment_id}/sessions/{session_id}' + + try: + response = requests.delete(url, params=wxa_params, auth=auth) + except Exception as e: + print(f"An error occurred: {e}") + +# query all groups +def batch_assistant_query(question_group_df): + time.sleep(3) + session_id = create_assistant_session() + + for index, row in question_group_df.iterrows(): + time.sleep(3) + flags = "" + query_results, api_response_time = query_assistant(session_id, row['Question']) + row['Assistant Output'] = query_results + + # error handling + if ("I'm sorry, I've run into an error processing your request. Please try the same question again. Thank you!" in query_results): + if (api_response_time > 30): + flags += "Timeout Error\n" + else: + flags += "Processing Error\n" + + + row['Response Times'] = api_response_time + row['Flags'] = flags + + + + delete_assistant_session(session_id) + + + +count = 1 +for df in question_groups_list: + print(f'-------------Testing group {count}-------------\n') + batch_assistant_query(df) + count += 1 + +# create the final dataframe to be exported to an excel sheet +concatenated_rows = [] + +for df in question_groups_list: + for index, row in df.iterrows(): + concatenated_rows.append(row) + +question_groups_df_combined = pd.DataFrame(concatenated_rows) + + + +# write the dataframe to an excel file +writer = pd.ExcelWriter(os.getenv("output_data_file"), engine='xlsxwriter') +question_groups_df_combined.to_excel(writer, index=False, sheet_name='Sheet1') +workbook = writer.book +worksheet = writer.sheets['Sheet1'] +cell_format = workbook.add_format({'text_wrap': True, 'valign': 'top', 'align': 'left'}) +for i, column in enumerate(question_groups_df_combined.columns): + worksheet.set_column(i, i, 40, cell_format) +worksheet.set_column(3, 3, 70, cell_format) +writer.close() + + diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..20aea45 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,9 @@ +ibm-cloud-sdk-core==3.16.0 +ibm-watson==7.0.1 +numpy==1.26.1 +openpyxl==3.1.2 +pandas==1.5.3 +python-dotenv==1.0.0 +XlsxWriter==3.2. +certifi==2023.7.22 +requests==2.31.0 \ No newline at end of file diff --git a/sample_input.xlsx b/sample_input.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..c8f0e9f0437450477d3fadc72e4b0b764ec386f4 GIT binary patch literal 9286 zcmeHN1y@|j)@>|-;ElTzEVw&_5ZoaN1gCLmLgOw$1Hmn5a19PY8h3|a!CiuMaDslF znfKnzWaj$??^UmLYjxk+r*BoAUAyX>tEPyAj1PDKKm`B*Gys#sbV~yS03aF}0KfyF zBI-Q_+dF~moeZ_y9Y7F$b~ig)>TG00rYry=y#4>mfAI*E#t*7=a^gI@kh^)h@kC*v zN)natAfOk>qAuRn71vX2s+VD9b)OM-jU)Y$z*?XZIJ)4;eKc%YWp7s(5ZcqEf$`42 zuT9&4On{@icb}n^h$Plg_Z2J)pG=aA#LzI(JQa}qroK~yS7ud8p_J5;fKc3TJ$ztFN)o0IMo?rTbL)}g~`Y3K;Dg>bO{y#9}l z|HT~qQ`bvll~p@AF@j-oHz9o&Q;X3!(u%H9a!oW^K7R5G*tL;4^kj>zkI8YgNCT0c z`$Bzg2IdzOAGSf;*E7 zG>WCtsqrr6dE&;U8lD_Z#0q>{^m*5KZcar#v$0wb@N2C!> z5>fyvf}1VJ?>un@JKC6l!8SkD>))I~fJ+$M%71q$)lgRK;>3ZX--U3xrn(T|es$uY z-Phj3M(wX*o@b=t@i|>4V`h)sLW6~-oci6Uh&pO{y$6Lm=6b1GlvSjc8}N|QEh)k*q#fVp-4m~HSB zX1>wiJGBg->A$3XFAu2yZsN$m2i9h{{ZH?|JR>bvogw!))*w zUg`#7>H`}`_3!2csmoiMtelZLRg z<#g0hPUmCG=Ml#ph^W$=?GZe?e^BMt`HFVxUiT2QFZ06aFtrbyvBKh+UXH4wU+PbLte;Ju*}gYo=HKrO-k+B5 zWvu}26F4vyV1J>^v7g z%fYng?(nnO*vl-APy*Qx&z;7X)U`>|OHJ^+ZD`zN$aY2o>MXmpzdji=SA2qUq+6qj z_yJqSn3;yRnvB69AX-Xx9?a*9x0>z3mMa}n76;QKY6$} zmEoR(psF%E-SKHKl(#|?sB6FSm=J4W=utt6U1aRonU|}u{LOa}WV%}>A{?`Z8NcB7 z%0!QROd9j%+oc^QkJjYYF=6*?V)wilbueKg?|CtYI;$w_aYfVXzEo-3t5Aaw3+yfv zTaQjU4lR;Yl@-d1J0mY`M7>lTC>q7bSKV8wnJDW<`#*G&mB|O4U(Yg`Ytdxe-(pK**`{x@0AUyD+%j@3GBxv6E1)gjJQmW8Fxr3eg6upK$soeI!gN!a%o{22dU zInJiYGWB}mhD2PQr%uTd-DD00*=spYxtY2hOU0aTgkXSM?;?6sM*3Gd>i%ZyJ}yY}sAOu7$mKnk*B)HN%$g1bRI~TTCp3RjEdt z(!8MQ%Ne_#al@IE1>3?=;5a9*mU}mTvU|SKJM#CE_sMFLM6aWuO5@aVB~<^&*g|EK zw--NmL2$-m{?1s4le;Yl@^f)-*6z2Thp%TuI}$j}th%-hbu`SO>!_&F4^Y++p%Xw5 z-YTtH4PE_Ct7~DmPIQ^FpMkBTpx24{k^_GFOhAs<6oDpp&s|CgZbHgH%0k!WHH&jT zv377@ngfGX1a?iwcK^1jItVLm$&Q&1mD-G8y^L#f!MVg%5v6EEOP(POUjg03s6@bC z+gjmxzyP>_DGFYD)Io2LyG46Vh9h-)`^kx$LBVA9!y9ufOhQ@OR6}s?gxRpa^+A+; zu(fYClGdKMoT)ePd4*Dm0>KuBgT6LCTaN1P$U=%@6&{ef6}ul>pTjNQvurtx*M)D9 zhm}etTUuW8qtZcc*b!>z9B>Gh=Mh~bh8QGLPG9VkEw*ZeR*@=`vZg?XG$FO=WfLPa zHoczBTP$5KV{>Mt`iPcTqQ+u)Q}%e|6(>%cM?<2L;x1kLQjfvYii^i5q&{)dC_N9HuXhdBLTj}FVbp+un-RA0^a}%H%R7Y8DzZYY)uR{Bk5HS~qZh)olJ`V= zI8jUeZh^qv#3u$#m6uOz2t5HJhy(Xn@4l^t-cQ(3U5crGb0dqX+ITpBS(i7zh+;v; zY-yuFI{_XKWZVp&jx13wVK*?Rya}0A*8dob)Ib^L%qCKW&DT@!GUDiVtiL znB*gXr(AWyv+g7!Jw;iQJV)~3Mh&%yy-sf@aptt4)rbqC^|-sYsV9gu6#j!i-P7B! zNQ5b9kr92H#GDo3y5Wm=9MzlJq=K4YF6SOPQsHWcN~eyaO_K&C3xgH?J4&TP?4g6L zr%16^>+apgjeG20-`_0u6Rx-C=PfrYXNhfyqcbi&+woKpKK8N@rO~tyCJ9m@y=r&l zHh3LmCAdP_0%`I51gkfEq$r+7#>iw*HBcRLF>c5d&wdF>7gA@W1+DEf%8L;)dc+w zIr;ctgAoTwjV4xngh$(xivEaJjpuN2UQILzclH`x*wDg9;%G%3iD4jR?Xg%sLIXg z;q7`m!q%e>*N_NelBuIrHxLdzVl;N8YFR8eBQ_~AVv&9H9B@rh>P=pyqkExwnCf79 zx4w$xQWlM0A)XZ1W!s6!3e(wLoM)oq`m7v3iNwP=Qe$l*PrQF4I)>Vih{QOCVy=(9 zm2xtO)}TRY@{;B9L%jIxJRcL2jy~WCoa&aB!U{K?(TZ;Z_iwJTJP`Xu?_F~Ba=WkCBAoeQ}xTY@g6U& zgP0BOew4g0^gTP~@V&gFG8C@4K+Z^9#UsTFQ|V*y7&3K}G<}pRkrJLNuSBc#1dZI^ zKIB_WINM4;DG$*nRa-o9f@sj|w(AlL!kKEhQhev38_w`2D_q;9qd{K)*k`X9&{cK% zC&U_FCcKH_YH=7AYYW?48D|MPRnTX7h+Aoy!JWp`Opm56ci%IL_C|EkB3{=;ET5;H z!I0^c-kQbM2Pr zu&*K1&c#Z0gIdz}?AZ2+IcPzS<@;$Y=X+TQZJy#*jNkSXp<9~Zt}Ul)F1=oU4JMDn z3HP@`9CRT!UO20Q&W`!=T0K52qck#H!kYsq5?kkw#xGVLrtLdlXezL|wu7wQqI`=hp`Tx%GiQtGO|W?cFGfVEU9oizB6Rt1!Q=AGaB120-Xe$ z2BKCC(7ZvJMzZZKZcz<=%Pl{{*Tj4rs=#VC+7}C3g*3M$x=jm)Rn8XtKyJ1VI@D@2 zaZ+>ZGT#QRpamUcuXS6+KL_Wx#*-HfYDHi zL4Ph#H;l|<6F_Ukn4C@bq7emm@HQxP=|$x10CRl>k=^P8rifCH$NBwx;hA`B$gMZ- z)|uEkXP*gL6`|D}<>Uq>j_0dBLAAp;<*}NG^ZoW~jZf_C70F=tEj3^36T#>o<^#6s zvfe-gyrTp13yrgynRUhTkMLjz*KACt7S_XkF}!6|Xytq-To@lR;;oeIul=l$ldjy2 z5Yg-LXZxtzr@)2bBeZGKzD%6@LwQ5m91IE#Z|h=6_;0lwmFum<%hpWAN9WS>TQ@8i z<6jc2`yi33eif`K)(W<-JC+&-KV7WQ|DdxM?3V~d3!5y=2UzQsw|NQ#)SrEtDYS^lqI<}9_4=9_a@Nx{Cj^`Navjk>fD0(=)>n1q zoqXQ0Rq|A%=FD63k zoLtbE#T5xh3?nb+18eCG{hLO!e=vNgN^pyy=^`zY^mJ8WAg4l3DcQy!>8wjB4Q;g% z(ag9ld-{o%$}~|{a9LyEt7u>VD22MS+y;^no0W2{gs(lG+xb_%1u#KfaE{3GKT|ir9%94k&^GRS^#_2UD>Zx;{ zrKk!`s*tF~y41H{fZ(RHz=gBIDUq)H3SNIO*2tj6yw?>*Y%h)q?h7`N19P;6lnHJz zf}VO!al`E-CG#-GlLmH$YOT9uHk@=rXXu!=&{jyVJ6{`6GW7$I&aBSZtAnor9@T@- zs!^HsFDS*bv3>vfn`bL)jKeCTZ|+tIm%fD5f)3J z@mT3(r*ZX!{gg+7)GT?h*lbPe`qDOKBj-w%1o`{k$chPaz^B6t(gcH6{zBgQ93Md} z*|U$>D_++P0rkY`5Qtt|-l{X~`9BK%Gw0Z~82I580^bJ{|1R`D53dkQ5XcF_@muwq z+NZ@SMx}D%^uiLZD57CTlr*VX(Nn>^-oO(w z^tmT7Z*~)9-KA)M`7t$3MoLc9@-zFh>cRdf7@4oE=-x|B?jo|jeiSzM*FHBDf>zb_ ze10d=JpS0$fw!VMaFSNs~sKVBW?J*Y23l9s%cssPvIi}4Z zSjs2CWVry~qE6|&v3sH`GJ)Uvups#>m2RfZ6s-wsu0s-~f@pX3wtf!gyh71pLr&SJ*M4Tg8(w;MX(P<*FDC46U5&HR_#E){kqp(3t~J6ZSXq67 z7+`Oe%PS6DQ}(F`(r)CbELC&{}2#pe)U`&uf(>#tkFtL4=Q<_=ZhPF@-%YcpFZUdtI!WDYb8%6o(JJ zF-q6Td8oN(!N|&CQ$Zd=3_)L0NwNu}y*!QI!J-?X7QybuAhjYMiWOAK7Nn@mxoW+n z0ea3nU%Td&QHfn?rKy?m`Yjsvn~RhtD%*E(;slige*l~r=_O1F6um&XzDa0rHhhn)zhfvF*I2f#>{YT)W7UG@vi$cL%{?)@w-^} z&V@Wj;up^?pl1d0#m=Rde4D|ZbU-jege;=0p2Aqg+i|%zOBj%s&6pR+Ns7FS8j7k% z3>&{z5LD~&Qmu?BK%`%l#I2R$eR0ok)!|Qj7*LpxAP2<4kHM3th}$O`qQ9smJi z-y%IbMGP~vgtrOym*L`tqxvOHNIt+DSWx3)DrZQF-CQ@oS#^K^+lW>%%PFazDYGez z4zcR=$t$x~YIBeAywG48P#X2CR6Is+%g$GLB>#hf;YS0ffvmNOf;DcaMQLtyVx}SD z`p*2;_mg3L6X9IfmvemmGY zMN5G@I5C1?ahKRJFuEY+FZ_9{e!L@c`wwNxW@N3?*j>ghFOV54(RRj{zj%(;y}ErX zb-+~_rPXO-;4f`$vR72H3b{DCd=%cTz85U_vJ-(s$K~88`56Jn8%${5U8bWG0)gnTfu{eeJV9plQxaY#y+ za`L>o_rno(-NQ5-?p`1{-VoOU!c#HeI0^G*=wxR8Awqy@p;Uk)VXu0GZ!=azjjzE@ zLYSt`VXk)pWPw@lyA=82bG^u(I=dYH1JlbJE}wZ(=&5$6f^*ZFU$~OP04C>5@|wYq zTNGAv9?=vRveFc1vetcll(kh}K7)4|vqZ!d)QD7JSSaiR{NAq2h;^}0$v6F2MITxkvp=I+ zH$M5*WiFD6!$3T{@DMAV)kDcFu)NDrJsc5=R(p1>APS>omv|RI?1jnhR@#Ltzp(pG z)jCePtBbIxk1&H}oVzz6-b{DjkFenYQh-*3G-5S3IL14@;>`)B@JM_Ca%Vki+p#I- zSnpR;dS8-pKU0HczLJx)$YpL+*3z|8iZD_>sXCo2`Ehj(E#i#4@JS0LhA_i6cY;v8 z-ynA&lmf-* z(KC2fs8eIPTw(-wsc4`Y@A`7h2$j0I(rmUXq2q(<(SeXvBToN~9-+07s;5^=w$w)w zp-GUn7~{2chdQKXUdV`mxRB~qrzb@t>RmEf!A|N~`6;>UDkbsqjJtu>JRxxU$3vAp zae_(HB{Rps{zjD6-YfkRS5v{jq}}PWV;6OUq#uSBd<+twbnpL@7)C&R0vC#Zeq-a` zv-a=$AKu?kQ~axezvh;Ip9`_;~`Ntiz@!QesO zUs5u^f`5$%{{jAp@w=A(Cp!Eq^w&VyAJ9&CE(rd=zlP0zHSp^O{0{@AME`m4zir2V zweqX}|6v6}_InS0-3|O|;jb$C2Oa