From 236e1089240ca18a8b343628257274f5dfce5a11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Valent=C3=ADn=20Barral=20Vales?= Date: Wed, 21 Oct 2020 09:56:45 +0200 Subject: [PATCH] Added source code --- ...xternal_dataset_to_csv_train_test_random.m | 33 ++ Matlab/ExtractFeatures.m | 19 + ...rnal_dataset_to_csv_train_test_random_v5.m | 165 +++++++ Matlab/extractFeaturesFromCir.m | 242 ++++++++++ Matlab/parseNLOSClassificationData.m | 44 ++ README.md | 31 +- main.py | 428 ++++++++++++++++++ plot_results.py | 320 +++++++++++++ plot_results_multi_pdp.py | 321 +++++++++++++ 9 files changed, 1602 insertions(+), 1 deletion(-) create mode 100644 Matlab/Export_external_dataset_to_csv_train_test_random.m create mode 100644 Matlab/ExtractFeatures.m create mode 100644 Matlab/do_export_external_dataset_to_csv_train_test_random_v5.m create mode 100644 Matlab/extractFeaturesFromCir.m create mode 100644 Matlab/parseNLOSClassificationData.m create mode 100644 main.py create mode 100644 plot_results.py create mode 100644 plot_results_multi_pdp.py diff --git a/Matlab/Export_external_dataset_to_csv_train_test_random.m b/Matlab/Export_external_dataset_to_csv_train_test_random.m new file mode 100644 index 0000000..2045858 --- /dev/null +++ b/Matlab/Export_external_dataset_to_csv_train_test_random.m @@ -0,0 +1,33 @@ + +chunks = 1; +version = 'v5'; +numReps = 5; +trainPercent = 0.8; + +labelRepIncrement = 0; %Top avoid overwrite previous files. Default= 0 + +pdpSizes = [3,7,15,30]; +for ii=1:1:numReps + for jj=1:1:length(pdpSizes) + pdpSize = pdpSizes(jj); + pathFeatures = ['./Measurements/External/RangingWithCIRData3_' version '_features_' version '_pdp_' num2str(pdpSize) '.mat']; + + useNormalized=0 + if (jj==1) + % Split set in training and test + % 50% of los/nlos in each set + [shuffleIndexes, trainRows, testRows ]=do_export_external_dataset_to_csv_train_test_random_v5(pathFeatures, chunks, ['Rand_' num2str(ii+labelRepIncrement) '_pdp_' num2str(pdpSize) '_'], useNormalized,trainPercent); + else + do_export_external_dataset_to_csv_train_test_random_v5(pathFeatures, chunks, ['Rand_' num2str(ii+labelRepIncrement) '_pdp_' num2str(pdpSize) '_'], useNormalized, trainPercent,shuffleIndexes, trainRows, testRows); + end + + useNormalized=1 + do_export_external_dataset_to_csv_train_test_random_v5(pathFeatures, chunks, ['Rand_' num2str(ii+labelRepIncrement) '_pdp_' num2str(pdpSize) '_'], useNormalized,trainPercent, shuffleIndexes,trainRows, testRows); + end +end + + + + + + diff --git a/Matlab/ExtractFeatures.m b/Matlab/ExtractFeatures.m new file mode 100644 index 0000000..2e983c3 --- /dev/null +++ b/Matlab/ExtractFeatures.m @@ -0,0 +1,19 @@ + +version = 'v5'; +fileName = './Measurements/External/RangingWithCIRData3_'; + +pdp_downsampled_out = 3; +pdp_downsampling_factor = 40; +extractFeaturesFromCir(version, fileName, pdp_downsampled_out, pdp_downsampling_factor); + +pdp_downsampled_out = 7; +pdp_downsampling_factor = 20; +extractFeaturesFromCir(version, fileName, pdp_downsampled_out, pdp_downsampling_factor); + +pdp_downsampled_out = 15; +pdp_downsampling_factor = 10; +extractFeaturesFromCir(version, fileName, pdp_downsampled_out, pdp_downsampling_factor); + +pdp_downsampled_out = 30; +pdp_downsampling_factor = 5; +extractFeaturesFromCir(version, fileName, pdp_downsampled_out, pdp_downsampling_factor); \ No newline at end of file diff --git a/Matlab/do_export_external_dataset_to_csv_train_test_random_v5.m b/Matlab/do_export_external_dataset_to_csv_train_test_random_v5.m new file mode 100644 index 0000000..7aea7be --- /dev/null +++ b/Matlab/do_export_external_dataset_to_csv_train_test_random_v5.m @@ -0,0 +1,165 @@ +function [shuffleRows, rowsTrain, rowsTest]=do_export_external_dataset_to_csv_train_test_random_v5(pathFeatures, chunks, prefix, useNormalized, trainPercent, shuffleRowsExplicit, rowsTrainExplicit, rowsTestExplicit) + + theFeatures = load(pathFeatures); + + nlos_all = theFeatures.features.nlos; + rss_all = theFeatures.features.rss; + range_all = theFeatures.features.range; + energy_all = theFeatures.features.energy; + mean_delay_all = theFeatures.features.mean_delay; + rms_delay_all = theFeatures.features.rms_delay; + label_normalized = ''; + if (useNormalized==1) + pdp_resampled_all = cell2mat(theFeatures.features.pdp_downsampled_normalized_no_noise(1:end)); + cir_first_all = cell2mat(theFeatures.features.cir_152_normalized_no_noise(1:end)); + cir_all = cell2mat(theFeatures.features.cir_normalized_no_noise(1:end)); + label_normalized ='_normalized'; + else + pdp_resampled_all = cell2mat(theFeatures.features.pdp_downsampled(1:end)); + cir_first_all = cell2mat(theFeatures.features.cir_152(1:end)); + cir_all = cell2mat(theFeatures.features.cir(1:end)); + end + + + rss_all = normalize(rss_all); + range_all = normalize(range_all); + energy_all = normalize(energy_all); + mean_delay_all = normalize(mean_delay_all); + rms_delay_all = normalize(rms_delay_all); + + + if ~exist('shuffleRowsExplicit','var') + %shuffleRows = randperm(size(nlos_all,1)); + shuffleRows = Shuffle(1:size(nlos_all,1)); + else + shuffleRows= shuffleRowsExplicit; + end + + + nlos_all = nlos_all(shuffleRows,:); + rss_all = rss_all(shuffleRows,:); + range_all = range_all(shuffleRows,:); + energy_all = energy_all(shuffleRows,:); + mean_delay_all =mean_delay_all(shuffleRows,:); + rms_delay_all =rms_delay_all(shuffleRows,:); + pdp_resampled_all = pdp_resampled_all(shuffleRows,:); + cir_first_all = cir_first_all(shuffleRows,:); + cir_all = cir_all(shuffleRows,:); + + numMeasurements = size(nlos_all,1); + + + %Split Train and Test + + if ~exist('rowsTrainExplicit','var') + numMeasurementsTrain = floor(numMeasurements * trainPercent); + numMeasurementsTrainLOS = floor(0.5*numMeasurementsTrain); + numMeasurementsTrainNLOS = numMeasurementsTrain - numMeasurementsTrainLOS; + + secuence = 1:1:size(nlos_all,1); + + losIndexes = secuence(nlos_all==0); + nlosIndexes = secuence(nlos_all==1); + + losTrainIndexes = losIndexes(1:numMeasurementsTrainLOS); + nlosTrainIndexes = nlosIndexes(1:numMeasurementsTrainNLOS); + rowsTrain = mergesorted(losTrainIndexes, nlosTrainIndexes); + + losTestIndexes = losIndexes(numMeasurementsTrainLOS+1:end); + nlosTestIndexes = nlosIndexes(numMeasurementsTrainNLOS+1:end); + rowsTest = mergesorted(losTestIndexes, nlosTestIndexes); + else + rowsTrain= rowsTrainExplicit; + rowsTest = rowsTestExplicit; + end + + + train_nlos_all = nlos_all(rowsTrain,:); + train_rss_all = rss_all(rowsTrain,:); + train_range_all = range_all(rowsTrain,:); + train_energy_all = energy_all(rowsTrain,:); + train_mean_delay_all =mean_delay_all(rowsTrain,:); + train_rms_delay_all =rms_delay_all(rowsTrain,:); + train_pdp_resampled_all = pdp_resampled_all(rowsTrain,:); + train_cir_first_all = cir_first_all(rowsTrain,:); + train_cir_all = cir_all(rowsTrain,:); + + test_nlos_all = nlos_all(rowsTest,:); + test_rss_all = rss_all(rowsTest,:); + test_range_all = range_all(rowsTest,:); + test_energy_all = energy_all(rowsTest,:); + test_mean_delay_all =mean_delay_all(rowsTest,:); + test_rms_delay_all =rms_delay_all(rowsTest,:); + test_pdp_resampled_all = pdp_resampled_all(rowsTest,:); + test_cir_first_all = cir_first_all(rowsTest,:); + test_cir_all = cir_all(rowsTest,:); + + + %Saving Training + + numMeasurementsTrain = size(train_nlos_all,1); + numMeasurementsTrainPerChunk = floor(numMeasurementsTrain/chunks); + + for ii=1:1:chunks + + start = (ii-1)*numMeasurementsTrainPerChunk + 1; + if (ii==chunks) + %Last chunk + limit=size(train_nlos_all,1); + else + limit= (ii-1)*numMeasurementsTrainPerChunk +numMeasurementsTrainPerChunk; + end + nlos = train_nlos_all(start:limit); + rss = train_rss_all(start:limit); + range = train_range_all(start:limit); + energy = train_energy_all(start:limit); + mean_delay = train_mean_delay_all(start:limit); + rms_delay = train_rms_delay_all(start:limit); + pdp_resampled = train_pdp_resampled_all(start:limit,:); + cir_first = train_cir_first_all(start:limit,:); + cir = train_cir_all(start:limit,:); + + T = table(nlos, rss, range, energy, mean_delay, rms_delay, pdp_resampled, cir_first, cir); + writetable(T,[prefix 'External_cir_and_pdp_set_3_TRAIN_' label_normalized '_' num2str(ii) '.csv'],'Delimiter',',','QuoteStrings',true); + end + + %Saving Test + + numMeasurementsTest = size(test_nlos_all,1); + numMeasurementsTestPerChunk = floor(numMeasurementsTest/chunks); + + + for ii=1:1:chunks + + start = (ii-1)*numMeasurementsTestPerChunk + 1; + if (ii==chunks) + %Last chunk + limit=size(test_nlos_all,1); + else + limit= (ii-1)*numMeasurementsTestPerChunk +numMeasurementsTestPerChunk; + end + + nlos = test_nlos_all(start:limit); + rss = test_rss_all(start:limit); + range = test_range_all(start:limit); + energy = test_energy_all(start:limit); + mean_delay = test_mean_delay_all(start:limit); + rms_delay = test_rms_delay_all(start:limit); + pdp_resampled = test_pdp_resampled_all(start:limit,:); + cir_first = test_cir_first_all(start:limit,:); + cir = test_cir_all(start:limit,:); + + T = table(nlos, rss, range, energy, mean_delay, rms_delay, pdp_resampled, cir_first, cir); + writetable(T,[prefix 'External_cir_and_pdp_set_3_TEST_' label_normalized '_' num2str(ii) '.csv'],'Delimiter',',','QuoteStrings',true); + end + + function result = normalize(vector) + minValue = min(vector); + maxValue = max(vector); + result = (vector - ones(length(vector),1).*minValue)./(ones(length(vector),1).*maxValue - ones(length(vector),1).*minValue); + end + +end + + + diff --git a/Matlab/extractFeaturesFromCir.m b/Matlab/extractFeaturesFromCir.m new file mode 100644 index 0000000..3ffc083 --- /dev/null +++ b/Matlab/extractFeaturesFromCir.m @@ -0,0 +1,242 @@ +function extractFeaturesFromCir(version,fileName,pdp_downsampled_out,pdp_downsampling_factor) +%extractFeaturesFromCir Gets PDP from CIR measurements + +% References: +% [1] Valentín Barral Vales, "ULTRA WIDEBAND LOCATION IN SCENARIOS WITHOUT +% CLEAR LINE OF SIGHT: A PRACTICAL APPROACH", Doctoral Thesis, 2020. +% +% [2] Stefano Maranò, Wesley M. Gifford, Henk Wymeersch, Moe Z. Win, +% "NLOS Identification and Mitigation for Localization Based on UWB +% Experimental Data", IEEE JOURNAL ON SELECTED AREAS IN COMMUNICATIONS, +% VOL. 28, NO. 7, SEPTEMBER 2010. + + +%% Configuration + + +% Measurement data +measfiles = {[fileName version]}; + +% Settings for the rise time feature (see reference [2]) +alpha = 6; % alpha parameter to calculate the rise time +beta = 0.6; % beta parameters to calculare the rise time + + +%% Code + +for ii = 1:numel(measfiles) + file = measfiles{ii}; + + % extract struct array with the measurement data + measdata = load(file); + measdata = struct2cell(measdata); + measdata = measdata{1}; + + nmeas = numel(measdata.distance); % number of measurements + + % variables to save the features + cir_cell = cell(nmeas,1); % CIRs + cir_original_cell = cell(nmeas,1); + cir_original_normalized_cell = cell(nmeas,1); + cir_original_normalized_no_noise_cell = cell(nmeas,1); + + pdp_downsampled_cell = cell(nmeas,1); % Downsampled PDP + pdp_downsampled_normalized_cell = cell(nmeas,1); % Downsampled PDP + pdp_downsampled_normalized_no_noise_cell = cell(nmeas,1); + + cir_152_cell = cell(nmeas,1); % 152 first samples of the CIR + cir_152_normalized_cell = cell(nmeas,1); % 152 first samples of the CIR + cir_152_normalized_no_noise_cell = cell(nmeas,1); + + energy_vec = zeros(nmeas,1); % Energy of the CIR (not including the noise) + energy_152_vec = zeros(nmeas,1); % Energy of the first 152 CIR samples (including the noise) + max_amplitude_vec = zeros(nmeas,1); % Maximum amplitude + t_rise_vec = zeros(nmeas,1); % Rise time + mean_delay_vec = zeros(nmeas,1); % Mean excess delay + rms_delay_vec = zeros(nmeas,1); % Root mean square delay + kurtosis_vec = zeros(nmeas,1); % Kurtosis + + % Mask of valid measurements: we remove too noisy measurements + valid_meas_mask = false(nmeas, 1); + + meastimer = tic(); + fprintf('Processing %s... ', file); + + % print simple progress indicator + progressmsg = sprintf('[%3d/%3d](%5.2f%%)', 0, nmeas, 0); + fprintf('%s', progressmsg); + + start_tomas = zeros(nmeas,1); + start_measurements = zeros(nmeas,1); + % process measurements + for jj = 1:nmeas + % get CIR + % we discard the 4 first CIR samples as said in the Decawave + % documentation. It seems that for some unknown reason the last 2 + % samples are [0 NaN] in most of the cases so we also discard them + cir = measdata.cir(jj,5:end-2); + cir_original_cell{jj} = cir; + + % estimate noise power + noise_tmax = 500; + assert(noise_tmax > 1); + noise_pw_est = mean(abs(cir(1:noise_tmax)).^2); + + % the first 600~700 samples of the CIR are only noise + % here we discard the first 500 samples of the CIR + cir_ndiscard = 500; + cir = cir(cir_ndiscard+1:end); + cir_cell{jj} = cir; + + % max absolute value and corresponding index + [max_amplitude, cir_max_i] = max(abs(cir)); + + % CIR energy + energy = sum(abs(cir).^2 - noise_pw_est); + energy_vec(jj) = energy; + + if energy <= 0 + % If this happens the measurent is too noisy + continue; + end + + % maximum amplitude + max_amplitude_vec(jj) = max_amplitude; + + % rise time + t_high = find(abs(cir) >= beta*max_amplitude, 1, 'first'); + t_low = find(abs(cir) >= alpha*sqrt(noise_pw_est), 1, 'last'); + + t_high_cell{jj} = t_high; + t_low_cell{jj} = t_low; + + if isempty(t_low) || isempty(t_high) + t_rise_vec(jj) = nan; + continue; + else + t_rise_vec(jj) = t_low - t_high; + end + + % create variables t_start and t_end to select CIR samples + t_start = max(1, t_high - 3); + t_end = t_start + t_low - t_high + 1; + t_start_measurement = measdata.fpindex(jj) - cir_ndiscard; + + start_tomas(jj) = t_start; + start_measurements(jj) = t_start_measurement; + if t_end > numel(cir) + continue; + end + + % Downsampled PDP + % the downsampled PDP is calculated starting at t_high + pdp_start_t_cell{jj} = t_start; + + cir1 = cir(t_start:end); + cir1 = reshape(cir1(1:pdp_downsampling_factor*pdp_downsampled_out), ... + [], pdp_downsampled_out); + + pdp_downsampled = mean(abs(cir1).^2); + pdp_downsampled_cell{jj} = pdp_downsampled; + + % 152 CIR samples from the first path + cir_152 = cir(t_start + (0:151)); + cir_152_cell{jj} = cir_152; + energy_152 = sum(abs(cir_152).^2); + energy_152_vec(jj) = energy_152; + + % normalized CIR + cir_original_normalized_cell{jj} = ... + cir_original_cell{jj}/sqrt(energy_vec(jj)); + + cir_t_rise_mask = false(1, numel(cir_original_cell{jj})); + cir_t_rise_mask(cir_ndiscard + (t_start:t_end)) = true; + + cir_original_normalized_no_noise_cell{jj} = cir_original_normalized_cell{jj}; + cir_original_normalized_no_noise_cell{jj}(~cir_t_rise_mask) = 0; + + + % normalized CIR first 152 samples + cir_152_normalized_cell{jj} = ... + cir_152_cell{jj}/sqrt(energy_vec(jj)); + + cir_152_t_rise_mask = false(1, 152); + cir_152_t_rise_mask(1:(t_end - t_start + 1)) = true; + + cir_152_normalized_no_noise_cell{jj} = cir_152_normalized_cell{jj}; + cir_152_normalized_no_noise_cell{jj}(~cir_152_t_rise_mask) = 0; + + + % normalized downsampled PDP + cir1 = cir(t_start:end)/sqrt(energy_vec(jj)); + cir1 = reshape(cir1(1:pdp_downsampling_factor*pdp_downsampled_out), ... + [], pdp_downsampled_out); + + pdp_downsampled = mean(abs(cir1).^2); + pdp_downsampled_normalized_cell{jj} = pdp_downsampled; + + % normalized downsampled PDP without noise + cir1 = cir_original_normalized_no_noise_cell{jj}((cir_ndiscard + t_start):end); + cir1 = reshape(cir1(1:pdp_downsampling_factor*pdp_downsampled_out), ... + [], pdp_downsampled_out); + + pdp_downsampled = mean(abs(cir1).^2); + pdp_downsampled_normalized_no_noise_cell{jj} = pdp_downsampled; + + % mean excess delay + t_152 = t_high + (0:151); + mean_delay = sum(t_152.*(abs(cir_152).^2)./energy_152); + mean_delay_vec(jj) = mean_delay; + + % RMS delay excess + rms_delay_vec(jj) =... + sqrt(sum((t_152 - mean_delay).^2.*(abs(cir_152).^2)/energy_152)); + + % kurtosis + kurtosis_vec(jj) =... + sum((t_152 - mean_delay).^4.*(abs(cir_152).^2)/energy_152)./... + sum((t_152 - mean_delay).^2.*(abs(cir_152).^2)/energy_152).^2; + + % mark measurement as valid + valid_meas_mask(jj) = true; + + progressmsglen = numel(progressmsg); + progressmsg = sprintf('[%3d/%3d](%5.2f%%)', jj, nmeas, 100*jj/nmeas); + fprintf([repmat('\b', 1, progressmsglen), '%s'], progressmsg); + end + fprintf(' OK (%g s)\n', toc(meastimer)); + + % create struct with the results + features.cir = cir_original_cell(valid_meas_mask); + features.cir_normalized = cir_original_normalized_cell(valid_meas_mask); + features.cir_normalized_no_noise = cir_original_normalized_no_noise_cell(valid_meas_mask); + + features.pdp_downsampled = pdp_downsampled_cell(valid_meas_mask); + features.pdp_downsampled_normalized = pdp_downsampled_normalized_cell(valid_meas_mask); + features.pdp_downsampled_normalized_no_noise = pdp_downsampled_normalized_no_noise_cell(valid_meas_mask); + + features.cir_152 = cir_152_cell(valid_meas_mask); + features.cir_152_normalized = cir_152_normalized_cell(valid_meas_mask); + features.cir_152_normalized_no_noise = cir_152_normalized_no_noise_cell(valid_meas_mask); + + features.energy = energy_vec(valid_meas_mask); + features.energy_152 = energy_152_vec(valid_meas_mask); + features.max_amplitude = max_amplitude_vec(valid_meas_mask); + features.t_rise = t_rise_vec(valid_meas_mask); + features.mean_delay = mean_delay_vec(valid_meas_mask); + features.rms_delay = rms_delay_vec(valid_meas_mask); + features.kurtosis = kurtosis_vec(valid_meas_mask); + + features.range = measdata.range(valid_meas_mask); + features.distance = measdata.distance(valid_meas_mask); + features.rss = measdata.rss(valid_meas_mask); + features.nlos = measdata.nlos(valid_meas_mask); + features.channel = measdata.channel(valid_meas_mask); + + % save struct to file + file_path=strsplit(file,filesep); + file_name=file_path{end}; + save([file_name '_features_' version '_pdp_' num2str(pdp_downsampled_out)], 'features'); + +end + diff --git a/Matlab/parseNLOSClassificationData.m b/Matlab/parseNLOSClassificationData.m new file mode 100644 index 0000000..b059857 --- /dev/null +++ b/Matlab/parseNLOSClassificationData.m @@ -0,0 +1,44 @@ +outputVersion = 'v5'; + +distance = []; +range = []; +cir = []; +rss = []; +nlos = []; +channel =[]; +fpindex = []; + +for ii=1:1:7 + file = readtable(['./Measurements/UWB-LOS-NLOS-Data-Set/dataset/uwb_dataset_part' num2str(ii) '.csv']); + file = file{:,:}; + A=121.74; + C=file(:,8); + N=file(:,10); + rssVal = 10.*log10(C*2^17./N.^2) - A; + + rssVal(~isfinite(rssVal)) = -120; + rss = [rss;rssVal]; + nlos = [nlos;file(:,1)]; + + channel = [channel;file(:,11)]; + + distance = [distance; file(:,2)]; + range = [range; file(:,2)]; + cir = [cir; file(:,16:end)./file(:,10)]; + fpindex = [fpindex; file(:,3)]; + + +end + +RangingWithCIRData3 =struct(); + +RangingWithCIRData3.range = range; +RangingWithCIRData3.distance = distance; +RangingWithCIRData3.cir = cir; +RangingWithCIRData3.rss = rss; +RangingWithCIRData3.nlos = nlos; +RangingWithCIRData3.channel = channel; +RangingWithCIRData3.fpindex = fpindex; + +save(['./Measurements/External/RangingWithCIRData3_' outputVersion '.mat'],'RangingWithCIRData3'); + diff --git a/README.md b/README.md index 511adec..dc33448 100644 --- a/README.md +++ b/README.md @@ -1 +1,30 @@ -# uwb_los_classification_pdp_dl \ No newline at end of file +# Using the Power Delay Profile to accelerate the training of neural network-based classifiers for the identification of LOS and NLOS UWB propagation conditions + + +This repository contains the code needed to replicate the experiments described in the article : "Using the Power Delay Profile to accelerate the training of neural network-based classifiers for the identification of LOS and NLOS UWB propagation conditions". The code is divided into two parts, one in Matlab to pre-process the samples and extract the PDP, and another part in Python+Tensorflow to train and test the LOS-NLOS classifier. + +## Pre-processing in Matlab + +To generate measurement sets be used later in Tensorflow, it is necessary to pre-process the original data. This requires the following steps: + +- Clone the repository [https://github.com/ewine-project/UWB-LOS-NLOS-Data-Set](https://github.com/ewine-project/UWB-LOS-NLOS-Data-Set) inside the "./Matlab/Measurements/ directory. + +- Run the file "./Matlab/parseNLOSClassificationData.m". This will generate a file called "./Measurements/External/RangingWithCIRData3_v5.mat". + +- Run the file "./Matlab/extractFeaturesFromCir.m". This will generate four new files with the PDP samples and the rest of the features. + +- Run the file "./Matlab/Export_external_dataset_to_csv_train_test_random.m". This will generate random sets of training and testing. The number of generated sets is configured within the script in the variable "numReps". + +- Finally, the generated ".csv" files must be copied into the "./ExternalDatasetWithPDP_v5" folder + +## Classification + +To run the simulation, the "main.py" file must be executed (it is necessary to have previously installed the [Tensorflow](https://www.tensorflow.org/install) libraries). + +The results are stored in "./Results_v5". Different figures can be obtained using the scripts "./plot_results_multi_pdp.py" and "plot_results.py". + +### References + +This work uses the UWB measurements datataset related with the paper: + +Klemen Bregar, Andrej Hrovat, Mihael Mohorčič, "NLOS Channel Detection with Multilayer Perceptron in Low-Rate Personal Area Networks for Indoor Localization Accuracy Improvement". Proceedings of the 8th Jožef Stefan International Postgraduate School Students’ Conference, Ljubljana, Slovenia, May 31-June 1, 2016. diff --git a/main.py b/main.py new file mode 100644 index 0000000..5160a15 --- /dev/null +++ b/main.py @@ -0,0 +1,428 @@ +# MIT License + +# Copyright (c) 2020 Group of Electronic Technology and Communications. University of A Coruna. + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import os +import pandas as pd +from numpy import vstack +import numpy as np +import tensorflow_addons as tfa +import tensorflow as tf +from tensorflow.keras.models import Sequential +from tensorflow.keras.layers import Dense, DenseFeatures, Conv1D, Conv2D, Flatten, Dropout, MaxPooling2D, BatchNormalization, Reshape, MaxPooling1D, LSTM +import glob +import functools +import time +from tensorflow.keras import backend as K +import pydotplus +import collections + +def recall_m(y_true, y_pred): + true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1))) + possible_positives = K.sum(K.round(K.clip(y_true, 0, 1))) + recall = true_positives / (possible_positives + K.epsilon()) + return recall + +def precision_m(y_true, y_pred): + true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1))) + predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1))) + precision = true_positives / (predicted_positives + K.epsilon()) + return precision + +def f1_m(y_true, y_pred): + precision = precision_m(y_true, y_pred) + recall = recall_m(y_true, y_pred) + return 2*((precision*recall)/(precision+recall+K.epsilon())) + +def get_dataset(file_path, **kwargs): + #batch_size=batch_size + dataset = tf.data.experimental.make_csv_dataset(file_path,label_name=LABEL_COLUMN, batch_size=batch_size,na_value="?",num_epochs=1,ignore_errors=True, shuffle=False, header=True, **kwargs) + return dataset + +def show_batch(dataset): + for batch, label in dataset.take(1): + for key, value in batch.items(): + print("{:20s}: {}".format(key,value.numpy())) + +def pack(features, label): + return tf.stack([float(i) for i in list(features.values())], axis=-1), label + +def split_dataset(dataset: tf.data.Dataset, validation_data_fraction: float): + """ + Splits a dataset of type tf.data.Dataset into a training and validation dataset using given ratio. Fractions are + rounded up to two decimal places. + @param dataset: the input dataset to split. + @param validation_data_fraction: the fraction of the validation data as a float between 0 and 1. + @return: a tuple of two tf.data.Datasets as (training, validation) + """ + + validation_data_percent = round(validation_data_fraction * 100) + if not (0 <= validation_data_percent <= 100): + raise ValueError("validation data fraction must be ∈ [0,1]") + + dataset = dataset.enumerate() + train_dataset = dataset.filter(lambda f, data: f % 100 > validation_data_percent) + validation_dataset = dataset.filter(lambda f, data: f % 100 <= validation_data_percent) + + # remove enumeration + train_dataset = train_dataset.map(lambda f, data: data) + validation_dataset = validation_dataset.map(lambda f, data: data) + + return train_dataset, validation_dataset + + +print("Num GPUs Available: ", len(tf.config.experimental.list_physical_devices('GPU'))) + +# DATASET_SIZE=41996 +batch_size = 64 +epochs = 20 +numReps =10 +cir_energy_mode = 0 # 0:raw, 1:normalized +version = 'v5' + +# Variant: +# all: test all modes with all pdp sizes +# all_no_extra: test only modes without extra features, but with all pdp sizes +# all_only_extra: test only modes with extra features, but with all pdp sizes +# all_single_pdp: test all modes but only one pdp size. The pdp size is the first of the array pdpLFactors +# pdps: test only the modes that use pdp, with all the pdp sizes +# all_only_extra_single_pdp: test only modes with extra features and only one PDP size. +variant = 'all' + +pdpLFactors = [5,10,20,40] +modesToTest = [0,1,2,3,4,5] #0:CIR, 1:CIR152, 2:PDP, 3:CIR + EXTRA, 4:CIR152 + EXTRA, 5:PDP + EXTRA +if variant== 'all': + modesToTest = [0,1,2,3,4,5] +elif variant == 'all_no_extra': + modesToTest = [0,1,2] +elif variant == 'all_only_extra': + modesToTest = [3,4,5] +elif variant == 'all_single_pdp': + modesToTest = [0,1,2,3,4,5] + pdpLFactors = [pdpLFactors[0]] +elif variant == 'all_only_extra_single_pdp': + modesToTest = [3,4,5] + pdpLFactors = [pdpLFactors[0]] +elif variant == 'pdps': + modesToTest = [2,5] + + + +modeStr = ['cir', 'cir[152]', 'pdp', 'others+cir', 'others+cir[152]', 'others+pdp' ] +cir_size = 1010 + +cir_first_size = 152 +LABEL_COLUMN = 'nlos' +LABELS = [0, 1] + +cir_energy_mode_label = '' +if cir_energy_mode==1: + cir_energy_mode_label = '_normalized' + + +for mode in modesToTest: #0:CIR, 1:CIR152, 2:PDP, 3:CIR + EXTRA, 4:CIR152 + EXTRA, 5:PDP + EXTRA + usesPdp = False + pdpLFactorsToTest = [5] + if ((2 == mode) or (5 == mode)): + #WE test different PDP values + usesPdp = True + pdpLFactorsToTest = pdpLFactors + + for pdp_factor in pdpLFactorsToTest: + pdp_size = int(cir_first_size/pdp_factor) + + print('++++++++++++++ START +++++++++++++++++') + print('MODE: ' + modeStr[mode]) + if usesPdp: + print('PDP L Factor: ' + str(pdp_factor)) + print('PDP Num. Samples: ' + str(pdp_size)) + print('Num Reps: ' + str(numReps)) + print('......................................') + print('......................................') + + resultsAccuracy = [] + resultsExecutioinTime = [] + resultsF1 = [] + resultsPrecision = [] + resultsRecall = [] + + + if mode==0: + #ONLY CIR + SELECT_COLUMNS_CIR = ['nlos'] + + for x in range(cir_size): + SELECT_COLUMNS_CIR.append('cir_' + str(x+1)) + SELECT_COLUMNS = [] + SELECT_COLUMNS_CIR + inputSizeNoCir =0 + inputSizeCir = cir_size + elif mode==1: + #first 152 cir + SELECT_COLUMNS_CIR = ['nlos'] + for x in range(cir_first_size): + SELECT_COLUMNS_CIR.append('cir_first_' + str(x+1)) + SELECT_COLUMNS = [] + SELECT_COLUMNS_CIR + inputSizeNoCir =0 + inputSizeCir = cir_first_size + elif mode==2: + #ONLY pdp + SELECT_COLUMNS_CIR = ['nlos'] + for x in range(pdp_size): + SELECT_COLUMNS_CIR.append('pdp_resampled_' + str(x+1)) + SELECT_COLUMNS = [] + SELECT_COLUMNS_CIR + inputSizeNoCir =0 + inputSizeCir = pdp_size + elif mode==3: + #Others and cir + SELECT_COLUMNS_CIR = [] + #SELECT_COLUMNS_NO_CIR = ['rss', 'range','energy','mean_delay','rms_delay'] + SELECT_COLUMNS_NO_CIR = ['range','energy'] + others_size = len(SELECT_COLUMNS_NO_CIR) + SELECT_COLUMNS_CIR = [] + for x in range(cir_size): + SELECT_COLUMNS_CIR.append('cir_' + str(x+1)) + SELECT_COLUMNS = ['nlos'] + SELECT_COLUMNS_NO_CIR + SELECT_COLUMNS_CIR + SELECT_COLUMNS_NO_CIR = ['nlos'] + SELECT_COLUMNS_NO_CIR + SELECT_COLUMNS_CIR = ['nlos'] + SELECT_COLUMNS_CIR + inputSizeNoCir =others_size + inputSizeCir = cir_size + elif mode==4: + #first 152 cir + extra + SELECT_COLUMNS_CIR = [] + #SELECT_COLUMNS_NO_CIR = ['rss', 'range','energy','mean_delay','rms_delay'] + SELECT_COLUMNS_NO_CIR = ['range','energy'] + others_size = len(SELECT_COLUMNS_NO_CIR) + SELECT_COLUMNS_CIR = [] + for x in range(cir_first_size): + SELECT_COLUMNS_CIR.append('cir_first_' + str(x+1)) + SELECT_COLUMNS = ['nlos'] + SELECT_COLUMNS_NO_CIR + SELECT_COLUMNS_CIR + SELECT_COLUMNS_NO_CIR = ['nlos'] + SELECT_COLUMNS_NO_CIR + SELECT_COLUMNS_CIR = ['nlos'] + SELECT_COLUMNS_CIR + inputSizeNoCir =others_size + inputSizeCir = cir_first_size + elif mode==5: + #PDP + EXTRA + SELECT_COLUMNS_CIR = [] + #SELECT_COLUMNS_NO_CIR = ['rss', 'range','energy','mean_delay','rms_delay'] + SELECT_COLUMNS_NO_CIR = ['range','energy'] + others_size = len(SELECT_COLUMNS_NO_CIR) + for x in range(pdp_size): + SELECT_COLUMNS_CIR.append('pdp_resampled_' + str(x+1)) + SELECT_COLUMNS = ['nlos'] + SELECT_COLUMNS_NO_CIR + SELECT_COLUMNS_CIR + SELECT_COLUMNS_NO_CIR = ['nlos'] + SELECT_COLUMNS_NO_CIR + SELECT_COLUMNS_CIR = ['nlos'] + SELECT_COLUMNS_CIR + inputSizeNoCir =others_size + inputSizeCir = pdp_size + + + + + train_only_with_cir = mode in [0,1,2] + + # numBatchs = int(DATASET_SIZE/batch_size) + # trainValSize = int(0.8 * numBatchs) + # train_size = int(0.9 * trainValSize) + + feature_columns_cir = [] + for header in SELECT_COLUMNS_CIR: + feature_columns_cir.append(tf.feature_column.numeric_column(header)) + + if (not train_only_with_cir): + feature_columns_no_cir = [] + for header in SELECT_COLUMNS_NO_CIR: + feature_columns_no_cir.append(tf.feature_column.numeric_column(header)) + + for ii in range(numReps): + + print('### Rep: ' + str(ii+1) + ' of ' + str(numReps)) + + + train_file_path = os.path.join(os.path.dirname("./"), 'ExternalDatasetWithPDP_'+ version +'/'+'Rand_' + str(ii+1) + '_pdp_' + str(pdp_size) +'_External_cir_and_pdp_set_3_TRAIN_'+ cir_energy_mode_label+'_1.csv') + #full_dataset_only_cir = get_dataset(train_file_path, select_columns=SELECT_COLUMNS) + train_dataset = get_dataset(train_file_path, select_columns=SELECT_COLUMNS) + #show_batch(train_dataset) + + test_file_path = os.path.join(os.path.dirname("./"), 'ExternalDatasetWithPDP_'+ version +'/'+'Rand_' + str(ii+1) + '_pdp_' + str(pdp_size) +'_External_cir_and_pdp_set_3_TEST_'+ cir_energy_mode_label+'_1.csv') + test_dataset = get_dataset(test_file_path, select_columns=SELECT_COLUMNS) + #show_batch(test_dataset) + + # train_dataset, test_dataset= split_dataset(full_dataset_only_cir,0.2) + #train_dataset, val_dataset = split_dataset(train_dataset,0.1) + + def select_features(cols): + def select_cols(features,label): + # print(features.keys()) + # print('...') + # print(label) + key = features.keys() + r = collections.OrderedDict() + for key in features.keys(): + if key in cols: + r[key]=features[key] + + return r,label + return select_cols + + # TRAIN DATASET + train_dataset_cir = train_dataset.map(select_features(SELECT_COLUMNS_CIR)) + if (not train_only_with_cir): + train_dataset_no_cir = train_dataset.map(select_features(SELECT_COLUMNS_NO_CIR)) + + packed_train_dataset_cir = train_dataset_cir.map(pack) + if (not train_only_with_cir): + packed_train_dataset_no_cir = train_dataset_no_cir.map(pack) + train_dataset_all_zip = tf.data.Dataset.zip((packed_train_dataset_cir, packed_train_dataset_no_cir)) + train_dataset_all_X = train_dataset_all_zip.map(lambda x1, x2: {'input_1': x1[0], 'input_2': x2[0]}) + train_dataset_all_Y = train_dataset_all_zip.map(lambda x1, x2: x2[1]) + train_dataset_all = tf.data.Dataset.zip((train_dataset_all_X, train_dataset_all_Y)) + else: + train_dataset_all =packed_train_dataset_cir + + # VALIDATION DATASET + + # val_dataset_cir = val_dataset.map(select_features(SELECT_COLUMNS_CIR)) + # if (not train_only_with_cir): + # val_dataset_no_cir = val_dataset.map(select_features(SELECT_COLUMNS_NO_CIR)) + + # packed_val_dataset_cir = val_dataset_cir.map(pack) + # if (not train_only_with_cir): + # packed_val_dataset_no_cir = val_dataset_no_cir.map(pack) + # val_dataset_zip = tf.data.Dataset.zip((packed_val_dataset_cir, packed_val_dataset_no_cir)) + # val_dataset_all_X = val_dataset_zip.map(lambda x1, x2: {'input_1': x1[0], 'input_2': x2[0]}) + # val_dataset_all_Y = val_dataset_zip.map(lambda x1, x2: x1[1]) + # val_dataset_all = tf.data.Dataset.zip((val_dataset_all_X, val_dataset_all_Y)) + # else: + # val_dataset_all = packed_val_dataset_cir + + # TEST DATASET + test_dataset_cir = test_dataset.map(select_features(SELECT_COLUMNS_CIR)) + if (not train_only_with_cir): + test_dataset_no_cir = test_dataset.map(select_features(SELECT_COLUMNS_NO_CIR)) + + packed_test_dataset_cir = test_dataset_cir.map(pack) + if (not train_only_with_cir): + packed_test_dataset_no_cir = test_dataset_no_cir.map(pack) + test_dataset_zip = tf.data.Dataset.zip((packed_test_dataset_cir, packed_test_dataset_no_cir)) + test_dataset_all_X = test_dataset_zip.map(lambda x1, x2: {'input_1': x1[0], 'input_2': x2[0]}) + test_dataset_all_Y = test_dataset_zip.map(lambda x1, x2: x1[1]) + test_dataset_all = tf.data.Dataset.zip((test_dataset_all_X, test_dataset_all_Y)) + else: + test_dataset_all = packed_test_dataset_cir + + # FIRST INPUT (CIR RELATED FEATURES) + model1 = Sequential() + if (not train_only_with_cir): + model1.add(tf.keras.layers.Input(shape=(inputSizeCir,),name='input_1')) + else: + model1.add(tf.keras.layers.Input(shape=(inputSizeCir,))) + + model1.add(Reshape((1,inputSizeCir))) + model1.add(Conv1D(10, 4, padding='same', activation='relu', input_shape=(inputSizeCir,))) + model1.add(BatchNormalization()) + model1.add(Conv1D(20, 5, padding='same', activation='relu')) + model1.add(BatchNormalization()) + model1.add(MaxPooling1D(2,strides=2, padding='same')) + model1.add(BatchNormalization()) + model1.add(Conv1D(20, 4, padding='same', activation='relu')) + model1.add(BatchNormalization()) + model1.add(Conv1D(40, 4, padding='same', activation='relu')) + model1.add(BatchNormalization()) + model1.add(MaxPooling1D(2,strides=2, padding='same')) + model1.add(BatchNormalization()) + model1.add(Dense(128, activation='relu')) + model1.add(BatchNormalization()) + ##model1.add(Dropout(0.25)) + + + if (not train_only_with_cir): + model2 = Sequential() + model2.add(tf.keras.layers.Input(shape=(inputSizeNoCir,),name='input_2')) + model2.add(Reshape((1,inputSizeNoCir))) + model2.add(Dense(64, activation='relu')) + model2.add(BatchNormalization()) + ##model2.add(Dropout(0.25)) + + # CONCATENATE INPUTS + if (not train_only_with_cir): + output = tf.keras.layers.Concatenate()([model1.output, model2.output]) + #output = Dense(16, activation='relu')(output) + output = Dense(1, activation='sigmoid')(output) + else: + #output = Dense(16, activation='relu')(model1.output) + output = Dense(1, activation='sigmoid')(model1.output) + + #output = Dense(1, activation='sigmoid')(output) + + # FINAL MODEL + if (not train_only_with_cir): + model = tf.keras.Model(inputs=[model1.input, model2.input], outputs=[output]) + else: + model = tf.keras.Model(inputs=[model1.input], outputs=[output]) + + tf.keras.utils.plot_model(model, "model_two_branches.png", show_shapes=True, dpi=300, show_layer_names=False) + #tf.keras.utils.plot_model(model, "model.png") + #print(model.summary()) + #print('### Network parameters count: ' + str(model.count_params())) + + ada_grad = tf.keras.optimizers.Adagrad(lr=0.1, epsilon=1e-08, decay=0.0) + model.compile(optimizer=ada_grad, + loss=tf.keras.losses.BinaryCrossentropy(), + metrics=['accuracy',f1_m,precision_m, recall_m]) + + start_time = time.time() + history = model.fit( + train_dataset_all, + epochs=epochs, + # validation_data= val_dataset_all, + shuffle=False, + verbose=1 + ) + executionTime = time.time() - start_time + print(executionTime) + resultsExecutioinTime.append(executionTime) + + # Test model + [loss, accuracy, f1_score, precision, recall] = model.evaluate(test_dataset_all) + resultsAccuracy.append(accuracy) + resultsF1.append(f1_score) + resultsPrecision.append(precision) + resultsRecall.append(recall) + + print('++++++++++++++ END +++++++++++++++++') + print('MODE: ' + modeStr[mode]) + print('Num Reps: ' + str(numReps)) + print('Accuracy: ' + str(resultsAccuracy)) + print('Time: ' + str(resultsExecutioinTime)) + print('......................................') + + if usesPdp: + pdpLabel = '_pdp_'+ str(pdp_size) + else: + pdpLabel='' + + np.save('Results_'+version+'/execution_'+str(mode) + pdpLabel + cir_energy_mode_label, resultsExecutioinTime) + np.save('Results_'+version+'/accuracy_'+str(mode) + pdpLabel+cir_energy_mode_label, resultsAccuracy) + np.save('Results_'+version+'/f1_'+str(mode) +pdpLabel+cir_energy_mode_label, resultsF1) + np.save('Results_'+version+'/precision_'+str(mode) +pdpLabel+cir_energy_mode_label, resultsPrecision) + np.save('Results_'+version+'/recall_'+str(mode) + pdpLabel+cir_energy_mode_label, resultsRecall) + + # model.save('TrainedModels/losnlos_'+str(mode)) + #show_batch(raw_train_data) \ No newline at end of file diff --git a/plot_results.py b/plot_results.py new file mode 100644 index 0000000..11f3463 --- /dev/null +++ b/plot_results.py @@ -0,0 +1,320 @@ +# MIT License + +# Copyright (c) 2020 Group of Electronic Technology and Communications. University of A Coruna. + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + + +import matplotlib.pyplot as plt +import numpy as np +from matplotlib import rc +import matplotlib.patches as mpatches +import matplotlib.lines as mlines + + +version = 'v5' + +resultsAccuracy = [] +resultsExecutioinTime = [] +resultsF1 = [] +resultsPrecision = [] +resultsRecall = [] + +timeMakePDP= 14.0108*0.7 + +sequence = [0,3,2,1,4,5,6,7] + +plot_only_cir = True +plot_cir_and_extra = True + +pdp_size = 30 + +cir_energy_mode = 1 # 0:raw, 1:normalized +cir_energy_mode_label = '' +if cir_energy_mode==1: + cir_energy_mode_label = '_normalized' + +# #NORMALIZED +# for mode in sequence: +# # resultsAccuracy.append(np.load('Results/accuracy_'+str(mode)+'_normalized_energy'+'.npy')) +# if (mode==2 or mode == 1): +# resultsExecutioinTime.append(np.array(np.load('Results/execution_'+str(mode)+'_normalized_energy'+'.npy'))+timeMakePDP) +# else: +# resultsExecutioinTime.append(np.load('Results/execution_'+str(mode)+'_normalized_energy'+'.npy')) + +# resultsF1.append(np.load('Results/f1_'+str(mode)+'_normalized_energy'+'.npy')) +# # resultsPrecision.append(np.load('Results/precision_'+str(mode)+'_normalized_energy'+'.npy')) +# # resultsRecall.append(np.load('Results/recall_'+str(mode)+'_normalized_energy'+'.npy')) + + +sequence = [0,1,2,3,4,5] +#NOT NORMALIZED +for mode in sequence: + # resultsAccuracy.append(np.load('Results/accuracy_'+str(mode)+'.npy')) + if (mode==2 or mode == 5): + resultsExecutioinTime.append(np.array(np.load('Results_'+version+'/execution_'+str(mode)+ '_pdp_'+ str(pdp_size) +cir_energy_mode_label+'.npy'))+timeMakePDP) + resultsF1.append(np.load('Results_'+version+'/f1_'+str(mode)+ '_pdp_'+ str(pdp_size) +cir_energy_mode_label+'.npy')) + else: + resultsExecutioinTime.append(np.load('Results_'+version+'/execution_'+str(mode)+ cir_energy_mode_label+'.npy')) + resultsF1.append(np.load('Results_'+version+'/f1_'+str(mode)+cir_energy_mode_label+'.npy')) + + + + # resultsPrecision.append(np.load('Results/precision_'+str(mode)+'.npy')) + # resultsRecall.append(np.load('Results/recall_'+str(mode)+'.npy')) + + +mode = ['cir', 'others+cir', 'pdp', 'others+pdp', 'cir[152]', 'others+cir[152]', 'others'] +colors = ['blue', 'blue', 'purple', 'purple', 'pink', 'pink', 'steelblue'] +ind = np.arange(7) + 1 + +plt.rc('axes', axisbelow=True) +plt.rc('text', usetex=True) +plt.rc('font', family='serif', size=20) + +# fig_accuracy = plt.figure() +# plt.axes().minorticks_on() +# plt.grid(b=True, which='both', color='k', linestyle='-', alpha=0.4) +# plt.grid(b=True, which='minor', color='k', linestyle='--', alpha=0.2) +# # plt.bar(mode,resultsAccuracy) +# bp=plt.boxplot(resultsAccuracy,showfliers=False, patch_artist=True, whis=[0,100],showmeans=True) +# for patch, color in zip(bp['boxes'], colors): +# patch.set_facecolor(color) + +mean_patch = mlines.Line2D([], [], color='green', marker='^', + markersize=10, label='Mean', linewidth=0) +median_patch = mlines.Line2D([], [], color='red', label='Median') +# plt.legend(handles=[mean_patch, median_patch]) + +# plt.title('Accuracy') +# plt.ylabel('Accuracy') +# plt.xticks(ind,mode) +# plt.ylim(0.70, 1.0) +# plt.tight_layout() + +# plt.savefig("plot_accuracy.pdf") +# plt.show() + + +# fig_time = plt.figure() +# plt.axes().minorticks_on() +# plt.grid(b=True, which='both', color='k', linestyle='-', alpha=0.4) +# plt.grid(b=True, which='minor', color='k', linestyle='--', alpha=0.2) +# bp=plt.boxplot(resultsExecutioinTime,showfliers=False, patch_artist=True, whis=[0,100],showmeans=True) +# for patch, color in zip(bp['boxes'], colors): +# patch.set_facecolor(color) +# plt.legend(handles=[mean_patch, median_patch]) +# #plt.title('Execution time') +# for aMean in bp['means']: +# plt.text(aMean._x+0.35, aMean._y -0.005, '%.3f' % aMean._y, +# horizontalalignment='center',size=18) +# plt.ylabel('Time (s)') +# plt.xticks(ind,mode) +# plt.tight_layout() +# plt.savefig("plot_time.pdf", bbox_inches = 'tight', +# pad_inches = 0.02) +# plt.show() + + +# fig_f1 = plt.figure() +# plt.axes().minorticks_on() +# plt.grid(b=True, which='both', color='k', linestyle='-', alpha=0.4) +# plt.grid(b=True, which='minor', color='k', linestyle='--', alpha=0.2) +# bp=plt.boxplot(resultsF1,showfliers=False, patch_artist=True, whis=[0,100],showmeans=True) +# for patch, color in zip(bp['boxes'], colors): +# patch.set_facecolor(color) +# #plt.title('F1-Score') +# for aMean in bp['means']: +# plt.text(aMean._x+0.35, aMean._y -0.005, '%.1f' % aMean._y, horizontalalignment='center',size=18) + +# plt.legend(handles=[mean_patch, median_patch]) +# plt.ylabel('F1-Score') +# plt.xticks(ind,mode) +# plt.ylim(0.70, 1.0) +# plt.tight_layout() +# plt.savefig("plot_f1score.pdf", bbox_inches = 'tight', +# pad_inches = 0.02) +# plt.show() + + +# fig_precision = plt.figure() +# plt.axes().minorticks_on() +# plt.grid(b=True, which='both', color='k', linestyle='-', alpha=0.4) +# plt.grid(b=True, which='minor', color='k', linestyle='--', alpha=0.2) +# bp=plt.boxplot(resultsPrecision,showfliers=False, patch_artist=True, whis=[0,100],showmeans=True) +# for patch, color in zip(bp['boxes'], colors): +# patch.set_facecolor(color) +# plt.title('Precision') +# plt.legend(handles=[mean_patch, median_patch]) +# plt.ylabel('Precision') +# plt.ylim(0.70, 1.0) +# plt.xticks(ind,mode) +# plt.tight_layout() +# plt.savefig("plot_precision.pdf") +# plt.show() + + +# fig_recall = plt.figure() +# plt.axes().minorticks_on() +# plt.grid(b=True, which='both', color='k', linestyle='-', alpha=0.4) +# plt.grid(b=True, which='minor', color='k', linestyle='--', alpha=0.2) +# bp=plt.boxplot(resultsRecall,showfliers=False, patch_artist=True, whis=[0,100],showmeans=True) +# for patch, color in zip(bp['boxes'], colors): +# patch.set_facecolor(color) +# plt.title('Recall') +# plt.legend(handles=[mean_patch, median_patch]) +# plt.ylabel('Recall') +# plt.ylim(0.70, 1.0) +# plt.xticks(ind,mode) +# plt.tight_layout() +# plt.savefig("plot_recall.pdf") +# plt.show() + + +########################## +# PLOTS ONLY WITH CIR + +if plot_only_cir: + resultsSelection = [0,1,2] + resultsF1Onlycir = np.array(resultsF1)[resultsSelection] + resultsExecutioinTimeOnlycir = np.array(resultsExecutioinTime)[resultsSelection] + modeOnlyCir = ['cir', 'cir[152]', 'pdp(L=5)'] + colorsOnlyCir = ['blue','steelblue', 'pink'] + indOnlyCir = np.arange(len(resultsSelection)) + 1 + + + fig_f1_only_cir = plt.figure() + plt.axes().minorticks_on() + plt.grid(b=True, which='both', color='k', linestyle='-', alpha=0.4) + plt.grid(b=True, which='minor', color='k', linestyle='--', alpha=0.2) + # plt.bar(mode,resultsAccuracy) + bp=plt.boxplot(resultsF1Onlycir.tolist(),showfliers=True, patch_artist=True, whis=[0,100],showmeans=True) + for patch, color in zip(bp['boxes'], colorsOnlyCir): + patch.set_facecolor(color) + + mean_patch = mlines.Line2D([], [], color='green', marker='^', + markersize=10, label='Mean', linewidth=0) + median_patch = mlines.Line2D([], [], color='red', label='Median') + plt.legend(handles=[mean_patch, median_patch]) + + for aMean in bp['means']: + plt.text(aMean._x+0.31, aMean._y -0.038, '%.3f' % aMean._y, + horizontalalignment='center',size=18) + + #plt.title('F1-Score') + plt.ylabel('F1-Score') + plt.xticks(indOnlyCir,modeOnlyCir) + plt.ylim(0.0, 1.0) + plt.tight_layout() + plt.savefig('Results_'+version+'/fig_f1' + cir_energy_mode_label +'.pdf', bbox_inches = 'tight', + pad_inches = 0.02) + plt.show() + + + fig_time_only_cir = plt.figure() + plt.axes().minorticks_on() + plt.grid(b=True, which='both', color='k', linestyle='-', alpha=0.4) + plt.grid(b=True, which='minor', color='k', linestyle='--', alpha=0.2) + # plt.bar(mode,resultsAccuracy) + bp=plt.boxplot(resultsExecutioinTimeOnlycir.tolist(),showfliers=True, patch_artist=True, whis=[0,100],showmeans=True) + for patch, color in zip(bp['boxes'], colorsOnlyCir): + patch.set_facecolor(color) + + mean_patch = mlines.Line2D([], [], color='green', marker='^', + markersize=10, label='Mean', linewidth=0) + median_patch = mlines.Line2D([], [], color='red', label='Median') + plt.legend(handles=[mean_patch, median_patch]) + + for aMean in bp['means']: + plt.text(aMean._x+0.35, aMean._y -0.001, '%.1f' % aMean._y, + horizontalalignment='center',size=18) + + #plt.title('Training time') + plt.ylabel('Time (s)') + plt.xticks(indOnlyCir,modeOnlyCir) + plt.tight_layout() + plt.savefig('Results_'+version+'/fig_time' + cir_energy_mode_label +'.pdf', bbox_inches = 'tight', + pad_inches = 0.02) + plt.show() + +########################## +# PLOTS Others + +if plot_cir_and_extra: + resultsSelection = [3,4,5] + resultsF1Other = np.array(resultsF1)[resultsSelection] + resultsExecutioinTimeOther = np.array(resultsExecutioinTime)[resultsSelection] + modeOther = ['cir\n+extra', 'cir[152]\n+extra', 'pdp(L=5)\n+extra'] + colorsOther = ['blue','steelblue', 'pink'] + indOther = np.arange(len(resultsSelection)) + 1 + + + fig_f1_other = plt.figure() + plt.axes().minorticks_on() + plt.grid(b=True, which='both', color='k', linestyle='-', alpha=0.4) + plt.grid(b=True, which='minor', color='k', linestyle='--', alpha=0.2) + # plt.bar(mode,resultsAccuracy) + bp=plt.boxplot(resultsF1Other.tolist(),showfliers=True, patch_artist=True, whis=[0,100],showmeans=True) + for patch, color in zip(bp['boxes'], colorsOther): + patch.set_facecolor(color) + + mean_patch = mlines.Line2D([], [], color='green', marker='^', + markersize=10, label='Mean', linewidth=0) + median_patch = mlines.Line2D([], [], color='red', label='Median') + plt.legend(handles=[mean_patch, median_patch]) + + for aMean in bp['means']: + plt.text(aMean._x+0.30, aMean._y -0.032, '%.3f' % aMean._y, + horizontalalignment='center',size=18) + + #plt.title('F1-Score') + plt.ylabel('F1-Score') + plt.xticks(indOther,modeOther) + plt.ylim(0.0, 1.0) + plt.tight_layout() + plt.savefig('Results_'+version+'/fig_f1_others' + cir_energy_mode_label +'.pdf', bbox_inches = 'tight', + pad_inches = 0.02) + plt.show() + + + fig_time_only_cir = plt.figure() + plt.axes().minorticks_on() + plt.grid(b=True, which='both', color='k', linestyle='-', alpha=0.4) + plt.grid(b=True, which='minor', color='k', linestyle='--', alpha=0.2) + # plt.bar(mode,resultsAccuracy) + bp=plt.boxplot(resultsExecutioinTimeOther.tolist(),showfliers=True, patch_artist=True, whis=[0,100],showmeans=True) + for patch, color in zip(bp['boxes'], colorsOther): + patch.set_facecolor(color) + + mean_patch = mlines.Line2D([], [], color='green', marker='^', + markersize=10, label='Mean', linewidth=0) + median_patch = mlines.Line2D([], [], color='red', label='Median') + plt.legend(handles=[mean_patch, median_patch]) + + for aMean in bp['means']: + plt.text(aMean._x+0.35, aMean._y -0.005, '%.1f' % aMean._y, + horizontalalignment='center',size=18) + + #plt.title('Training time') + plt.ylabel('Time (s)') + plt.xticks(indOther,modeOther) + plt.tight_layout() + plt.savefig('Results_'+version+'/fig_time_others' + cir_energy_mode_label +'.pdf', bbox_inches = 'tight', + pad_inches = 0.02) + plt.show() \ No newline at end of file diff --git a/plot_results_multi_pdp.py b/plot_results_multi_pdp.py new file mode 100644 index 0000000..09fca36 --- /dev/null +++ b/plot_results_multi_pdp.py @@ -0,0 +1,321 @@ +# MIT License + +# Copyright (c) 2020 Group of Electronic Technology and Communications. University of A Coruna. + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + + +import matplotlib.pyplot as plt +import numpy as np +from matplotlib import rc +import matplotlib.patches as mpatches +import matplotlib.lines as mlines + + + +resultsAccuracy = [] +resultsExecutioinTime = [] +resultsF1 = [] +resultsPrecision = [] +resultsRecall = [] +version = 'v5' + +cir_first_size = 152 +pdpLFactors = [5,10,20,40] +pdpSizes = map(lambda x: int(cir_first_size/x), pdpLFactors) +useExtraFeatures = False +cir_energy_mode = 0 # 0:raw, 1:normalized + + + +cir_energy_mode_label = '' +if cir_energy_mode==1: + cir_energy_mode_label = '_normalized' + +timeMakePDP= 14.0108*0.7 + + +modesToLoad = [2] +modeLabel = 'pdp' +if (useExtraFeatures): + modesToLoad = [5] + modeLabel = 'pdp+extras' + +indexPdp = -1 + +for pdp_size in pdpSizes: + resultsAccuracy.append([]) + resultsExecutioinTime.append([]) + resultsF1.append([]) + resultsPrecision.append([]) + resultsRecall.append([]) + indexPdp +=1 + for mode in modesToLoad: + resultsAccuracy[indexPdp].append(np.load('Results_'+version+'/accuracy_'+str(mode)+ '_pdp_'+ str(pdp_size) + cir_energy_mode_label+'.npy')) + resultsExecutioinTime[indexPdp].append(np.array(np.load('Results_'+version+'/execution_'+str(mode)+ '_pdp_'+ str(pdp_size) + cir_energy_mode_label+'.npy'))+timeMakePDP) + resultsF1[indexPdp].append(np.load('Results_'+version+'/f1_'+str(mode)+ '_pdp_'+ str(pdp_size) + cir_energy_mode_label+'.npy')) + resultsPrecision[indexPdp].append(np.load('Results_'+version+'/precision_'+str(mode)+ '_pdp_'+ str(pdp_size) + cir_energy_mode_label+'.npy')) + resultsRecall[indexPdp].append(np.load('Results_'+version+'/recall_'+str(mode)+ '_pdp_'+ str(pdp_size) + cir_energy_mode_label+'.npy')) + + + +mode = ['pdp', 'pdp_20', 'pdp_30' , 'others+pdp_10', 'others+pdp_20', 'others+pdp_30'] +colors = ['blue', 'purple', 'pink', 'blue', 'purple', 'pink'] +ind = np.arange(6) + 1 + +plt.rc('axes', axisbelow=True) +plt.rc('text', usetex=True) +plt.rc('font', family='serif', size=20) + +# fig_accuracy = plt.figure() +# plt.axes().minorticks_on() +# plt.grid(b=True, which='both', color='k', linestyle='-', alpha=0.4) +# plt.grid(b=True, which='minor', color='k', linestyle='--', alpha=0.2) +# # plt.bar(mode,resultsAccuracy) +# bp=plt.boxplot(resultsAccuracy,showfliers=False, patch_artist=True, whis=[0,100],showmeans=True) +# for patch, color in zip(bp['boxes'], colors): +# patch.set_facecolor(color) + +mean_patch = mlines.Line2D([], [], color='green', marker='^', + markersize=10, label='Mean', linewidth=0) +median_patch = mlines.Line2D([], [], color='red', label='Median') +# plt.legend(handles=[mean_patch, median_patch]) + +# plt.title('Accuracy') +# plt.ylabel('Accuracy') +# plt.xticks(ind,mode) +# plt.ylim(0.70, 1.0) +# plt.tight_layout() + +# plt.savefig("plot_accuracy.pdf") +# plt.show() + + +# fig_time = plt.figure() +# plt.axes().minorticks_on() +# plt.grid(b=True, which='both', color='k', linestyle='-', alpha=0.4) +# plt.grid(b=True, which='minor', color='k', linestyle='--', alpha=0.2) +# bp=plt.boxplot(resultsExecutioinTime,showfliers=False, patch_artist=True, whis=[0,100],showmeans=True) +# for patch, color in zip(bp['boxes'], colors): +# patch.set_facecolor(color) +# plt.legend(handles=[mean_patch, median_patch]) +# #plt.title('Execution time') +# for aMean in bp['means']: +# plt.text(aMean._x+0.35, aMean._y -0.005, '%.3f' % aMean._y, +# horizontalalignment='center',size=18) +# plt.ylabel('Time (s)') +# plt.xticks(ind,mode) +# plt.tight_layout() +# plt.savefig("plot_time.pdf", bbox_inches = 'tight', +# pad_inches = 0.02) +# plt.show() + + +# fig_f1 = plt.figure() +# plt.axes().minorticks_on() +# plt.grid(b=True, which='both', color='k', linestyle='-', alpha=0.4) +# plt.grid(b=True, which='minor', color='k', linestyle='--', alpha=0.2) +# bp=plt.boxplot(resultsF1,showfliers=False, patch_artist=True, whis=[0,100],showmeans=True) +# for patch, color in zip(bp['boxes'], colors): +# patch.set_facecolor(color) +# #plt.title('F1-Score') +# for aMean in bp['means']: +# plt.text(aMean._x+0.35, aMean._y -0.005, '%.1f' % aMean._y, horizontalalignment='center',size=18) + +# plt.legend(handles=[mean_patch, median_patch]) +# plt.ylabel('F1-Score') +# plt.xticks(ind,mode) +# plt.ylim(0.70, 1.0) +# plt.tight_layout() +# plt.savefig("plot_f1score.pdf", bbox_inches = 'tight', +# pad_inches = 0.02) +# plt.show() + + +# fig_precision = plt.figure() +# plt.axes().minorticks_on() +# plt.grid(b=True, which='both', color='k', linestyle='-', alpha=0.4) +# plt.grid(b=True, which='minor', color='k', linestyle='--', alpha=0.2) +# bp=plt.boxplot(resultsPrecision,showfliers=False, patch_artist=True, whis=[0,100],showmeans=True) +# for patch, color in zip(bp['boxes'], colors): +# patch.set_facecolor(color) +# plt.title('Precision') +# plt.legend(handles=[mean_patch, median_patch]) +# plt.ylabel('Precision') +# plt.ylim(0.70, 1.0) +# plt.xticks(ind,mode) +# plt.tight_layout() +# plt.savefig("plot_precision.pdf") +# plt.show() + + +# fig_recall = plt.figure() +# plt.axes().minorticks_on() +# plt.grid(b=True, which='both', color='k', linestyle='-', alpha=0.4) +# plt.grid(b=True, which='minor', color='k', linestyle='--', alpha=0.2) +# bp=plt.boxplot(resultsRecall,showfliers=False, patch_artist=True, whis=[0,100],showmeans=True) +# for patch, color in zip(bp['boxes'], colors): +# patch.set_facecolor(color) +# plt.title('Recall') +# plt.legend(handles=[mean_patch, median_patch]) +# plt.ylabel('Recall') +# plt.ylim(0.70, 1.0) +# plt.xticks(ind,mode) +# plt.tight_layout() +# plt.savefig("plot_recall.pdf") +# plt.show() + + +########################## +# PLOTS ONLY WITH CIR + + +pdpLabels = ['L=5','L=10', 'L=20', 'L=40'] +pdpColors = ['blue', 'purple', 'pink', 'steelblue'] + +resultsAccuracyOnlycir = np.vstack(resultsAccuracy) +#resultsAccuracyOnlycir= resultsAccuracyOnlycir[:,:,0] +resultsF1Onlycir = np.vstack(resultsF1) +#resultsF1Onlycir[2,0]= 0.76 +#resultsF1Onlycir[2,7]= 0.76 +#resultsF1Onlycir = resultsF1Onlycir[:,:,0] +resultsExecutioinTimeOnlycir = np.vstack(resultsExecutioinTime) +#resultsExecutioinTimeOnlycir = resultsExecutioinTimeOnlycir[:,:,0] + +indOnlyCir = np.arange(4) + 1 + + +fig_f1_only_cir = plt.figure() +plt.axes().minorticks_on() +plt.grid(b=True, which='both', color='k', linestyle='-', alpha=0.4) +plt.grid(b=True, which='minor', color='k', linestyle='--', alpha=0.2) +# plt.bar(mode,resultsAccuracy) +bp=plt.boxplot(resultsF1Onlycir.tolist(),showfliers=True, patch_artist=True, whis=[0,100],showmeans=True) +for patch, color in zip(bp['boxes'], pdpColors): + patch.set_facecolor(color) + +mean_patch = mlines.Line2D([], [], color='green', marker='^', + markersize=10, label='Mean', linewidth=0) +median_patch = mlines.Line2D([], [], color='red', label='Median') +plt.legend(handles=[mean_patch, median_patch]) + +for aMean in bp['means']: + plt.text(aMean._x+0.35, aMean._y -0.005, '%.3f' % aMean._y, + horizontalalignment='center',size=18) + +#plt.title('F1-Score') +plt.ylabel('F1-Score') +plt.xticks(indOnlyCir,pdpLabels) +plt.ylim(0, 1.0) +plt.tight_layout() +plt.savefig('fig_f1_multi_'+ modeLabel + cir_energy_mode_label+ '.pdf', bbox_inches = 'tight', + pad_inches = 0.02) +plt.show() + + +fig_time_only_cir = plt.figure() +plt.axes().minorticks_on() +plt.grid(b=True, which='both', color='k', linestyle='-', alpha=0.4) +plt.grid(b=True, which='minor', color='k', linestyle='--', alpha=0.2) +# plt.bar(mode,resultsAccuracy) +bp=plt.boxplot(resultsExecutioinTimeOnlycir.tolist(),showfliers=True, patch_artist=True, whis=[0,100],showmeans=True) +for patch, color in zip(bp['boxes'], pdpColors): + patch.set_facecolor(color) + +mean_patch = mlines.Line2D([], [], color='green', marker='^', + markersize=10, label='Mean', linewidth=0) +median_patch = mlines.Line2D([], [], color='red', label='Median') +plt.legend(handles=[mean_patch, median_patch]) + +for aMean in bp['means']: + plt.text(aMean._x+0.35, aMean._y -0.005, '%.1f' % aMean._y, + horizontalalignment='center',size=18) + +#plt.title('Training time') +plt.ylabel('Time (s)') +plt.xticks(indOnlyCir,pdpLabels) +plt.tight_layout() +plt.savefig('fig_time_multi_'+ modeLabel + cir_energy_mode_label+ '.pdf', bbox_inches = 'tight', + pad_inches = 0.02) +plt.show() + +# ########################## +# # PLOTS Others + +# resultsAccuracyOther = np.array(resultsAccuracy)[[1,3,5],:] +# resultsF1Other = np.array(resultsF1)[[1,3,5],:] +# resultsExecutioinTimeOther = np.array(resultsExecutioinTime)[[1,3,5],:] +# modeOther = ['cir+extra', 'pdp+extra', 'cir[152]+extra'] +# colorsOther = ['blue', 'purple', 'pink'] +# indOther = np.arange(3) + 1 + + +# fig_f1_other = plt.figure() +# plt.axes().minorticks_on() +# plt.grid(b=True, which='both', color='k', linestyle='-', alpha=0.4) +# plt.grid(b=True, which='minor', color='k', linestyle='--', alpha=0.2) +# # plt.bar(mode,resultsAccuracy) +# bp=plt.boxplot(resultsF1Other.tolist(),showfliers=False, patch_artist=True, whis=[0,100],showmeans=True) +# for patch, color in zip(bp['boxes'], colorsOther): +# patch.set_facecolor(color) + +# mean_patch = mlines.Line2D([], [], color='green', marker='^', +# markersize=10, label='Mean', linewidth=0) +# median_patch = mlines.Line2D([], [], color='red', label='Median') +# plt.legend(handles=[mean_patch, median_patch]) + +# for aMean in bp['means']: +# plt.text(aMean._x+0.35, aMean._y -0.005, '%.3f' % aMean._y, +# horizontalalignment='center',size=18) + +# #plt.title('F1-Score') +# plt.ylabel('F1-Score') +# plt.xticks(indOther,modeOther) +# plt.ylim(0.70, 1.0) +# plt.tight_layout() +# plt.savefig("fig_f1_others.pdf", bbox_inches = 'tight', +# pad_inches = 0.02) +# plt.show() + + +# fig_time_only_cir = plt.figure() +# plt.axes().minorticks_on() +# plt.grid(b=True, which='both', color='k', linestyle='-', alpha=0.4) +# plt.grid(b=True, which='minor', color='k', linestyle='--', alpha=0.2) +# # plt.bar(mode,resultsAccuracy) +# bp=plt.boxplot(resultsExecutioinTimeOther.tolist(),showfliers=False, patch_artist=True, whis=[0,100],showmeans=True) +# for patch, color in zip(bp['boxes'], colorsOther): +# patch.set_facecolor(color) + +# mean_patch = mlines.Line2D([], [], color='green', marker='^', +# markersize=10, label='Mean', linewidth=0) +# median_patch = mlines.Line2D([], [], color='red', label='Median') +# plt.legend(handles=[mean_patch, median_patch]) + +# for aMean in bp['means']: +# plt.text(aMean._x+0.35, aMean._y -0.005, '%.1f' % aMean._y, +# horizontalalignment='center',size=18) + +# #plt.title('Training time') +# plt.ylabel('Time (s)') +# plt.xticks(indOther,modeOther) +# plt.tight_layout() +# plt.savefig("fig_time_others.pdf", bbox_inches = 'tight', +# pad_inches = 0.02) +# plt.show() \ No newline at end of file