Merge pull request #79 from loucerac/develop

Develop

drexml/cli/stab_explainer.py

@@ -23,9 +23,10 @@
     import shap
 
 from sklearn.model_selection import train_test_split
+from sklearn.pipeline import Pipeline
 
 from drexml.explain import compute_shap_fs, compute_shap_relevance, compute_shap_values_
-from drexml.models import get_model
+from drexml.models import extract_estimator, get_model
 from drexml.utils import convert_names, parse_stab
 
 if __name__ == "__main__":
@@ -83,10 +84,15 @@
 
         if n_splits == 1:
             print(f"fit final model {i_split=} {n_splits=}")
+            use_imputer = features.isna().any(axis=None)
             this_model = get_model(
-                features.shape[1], targets.shape[1], n_cpus, debug, n_iters
+                features.shape[1], targets.shape[1], n_cpus, debug, n_iters, use_imputer
             )
-            this_model.set_params(n_jobs=n_cpus, random_state=this_seed)
+            if isinstance(this_model, Pipeline):
+                # set final estimator params if pipeline
+                this_model[-1].set_params(n_jobs=n_cpus, random_state=this_seed)
+            else:
+                this_model.set_params(n_jobs=n_cpus, random_state=this_seed)
             this_model.fit(features_learn, targets_learn)
         else:
             model_fname = f"model_{i_split}.jbl"
@@ -100,9 +106,6 @@
 
         def runner(model, bkg, new, check_add, use_gpu):
             gpu_id = queue.get()
-            if hasattr(model, "best_estimator_"):
-                # retrieve RF model if HP-optim has been carried out.
-                model = model.best_estimator_
 
             if use_gpu:
                 explainer = shap.GPUTreeExplainer(model, bkg)
@@ -123,10 +126,12 @@ def runner(model, bkg, new, check_add, use_gpu):
         else:
             features_bkg = features_learn
 
+        estimator = extract_estimator(this_model)
+
         with joblib.parallel_backend("multiprocessing", n_jobs=n_devices):
             shap_values = joblib.Parallel()(
                 joblib.delayed(runner)(
-                    model=this_model,
+                    model=estimator,
                     bkg=features_bkg,
                     new=gb,
                     check_add=add,

drexml/cli/stab_scorer.py

@@ -10,9 +10,10 @@
 import pandas as pd
 from sklearn.base import clone
 from sklearn.ensemble import RandomForestRegressor
-from sklearn.metrics import r2_score
 from sklearn.experimental import enable_halving_search_cv
-from sklearn.model_selection import RandomizedSearchCV, HalvingRandomSearchCV
+from sklearn.metrics import r2_score
+from sklearn.model_selection import HalvingRandomSearchCV, RandomizedSearchCV
+from sklearn.pipeline import Pipeline
 
 from drexml.explain import build_stability_dict
 from drexml.pystab import nogueria_test
@@ -54,6 +55,11 @@ def stab_i(estimator, X, Y, split_id, this_split):
                 fimp = estimator_.feature_importances_
             elif hasattr(estimator_, "best_estimator_"):
                 fimp = estimator_.best_estimator_.feature_importances_
+            elif hasattr(estimator_, "named_steps"):
+                if hasattr(estimator_[-1], "best_estimator_"):
+                    fimp = estimator_[-1].best_estimator_.feature_importances_
+                else:
+                    fimp = estimator_[-1].feature_importances_
             filt_i[fimp.argmax()] = True
 
         features_train_filt = features_train.loc[:, filt_i]
@@ -65,6 +71,11 @@ def stab_i(estimator, X, Y, split_id, this_split):
                 estimator_filt.max_features = 1.0
             elif isinstance(estimator, (RandomizedSearchCV, HalvingRandomSearchCV)):
                 estimator_filt = clone(estimator)
+            elif isinstance(estimator, Pipeline):
+                estimator_filt = clone(estimator)
+                if isinstance(estimator[-1], RandomForestRegressor):
+                    estimator_filt[-1].max_features = 1.0
+
                 # estimator_filt.set_params(**{"max_features": 1.0, "random_state": 42})
             # sub_model.set_params(**{"max_depth": 32, "max_features": filt_i.sum()})
             estimator_filt.fit(features_train_filt, targets_train)

drexml/datasets.py

@@ -84,7 +84,7 @@ def load_disgenet():
 
 
 def get_gda(disease_id, k_top=40):
-    """Retrieve the list of genes associated to a disese according to the Disgenet
+    """Retrieve the list of genes associated to a disease according to the Disgenet
     curated list of gene-disease associations.
 
     Parameters
@@ -169,7 +169,7 @@ def fetch_file(key, env, version="latest"):
 
 
 def load_df(path, key=None):
-    """Load dataframe from file. At the moment: stv, tsv crompressed or feather.
+    """Load dataframe from file. At the moment: stv, tsv compressed or feather.
 
     Parameters
     ----------
@@ -244,7 +244,7 @@ def get_index_name_options(key):
 
 def preprocess_frame(res, env, key):
     """
-    Preprocesses the input data frame.
+    Preprocess the input data frame.
 
     Parameters
     ----------
@@ -298,7 +298,7 @@ def preprocess_frame_(res, key):
 
 def preprocess_gexp(frame):
     """
-    Preprocesses a gene expression data frame.
+    Preprocess a gene expression data frame.
 
     Parameters
     ----------
@@ -331,7 +331,7 @@ def preprocess_gexp(frame):
 
 def preprocess_activities(frame):
     """
-    Preprocesses an activities data frame.
+    Preprocess an activities data frame.
 
     Parameters
     ----------
@@ -366,7 +366,7 @@ def preprocess_map(
     frame, disease_seed_genes, circuits_column, use_physio, circuits_dict=None
 ):
     """
-    Preprocesses a map data frame.
+    Preprocess a map data frame.
 
     Parameters
     ----------
@@ -428,7 +428,7 @@ def preprocess_map(
 
 def preprocess_genes(frame, genes_column):
     """
-    Preprocesses a gene expression data frame.
+    Preprocess a gene expression data frame.
 
     Parameters
     ----------

drexml/explain.py

@@ -177,13 +177,13 @@ def get_quantile_by_circuit(model, X, Y, threshold=0.5):
     Parameters
     ----------
     model : sklearn.base.BaseEstimator
-        Fited model.
+        Fitted model.
     X : pandas.DataFrame [n_samples, n_features]
         The feature dataset to explain.
     Y : pandas.DataFrame [n_samples, n_tasks]
         The task dataset to explain.
     threshold : float, optional
-        Theshold to use to dicriminate ill-conditioned circuits when performing feature
+        Threshold to use to discriminate ill-conditioned circuits when performing feature
         selection, by default 0.5
 
     Returns

drexml/models.py

@@ -7,24 +7,28 @@
 import numpy as np
 from sklearn.ensemble import RandomForestRegressor
 from sklearn.experimental import enable_halving_search_cv  # noqa
-from sklearn.model_selection import HalvingRandomSearchCV
+from sklearn.impute import KNNImputer
+from sklearn.model_selection import HalvingRandomSearchCV, RandomizedSearchCV
+from sklearn.pipeline import Pipeline, make_pipeline
 
 
-def get_model(n_features, n_targets, n_jobs, debug, n_iters=0):
+def get_model(n_features, n_targets, n_jobs, debug, n_iters=0, use_imputer=False):
     """Create a model.
 
     Parameters
     ----------
     n_features : int
-        Number of features (KDTs).
+        Number of features (KDTs / gene input targets).
     n_targets : int
         Number of targets (circuits).
     n_jobs : int
         The number of jobs to run in parallel.
     debug : bool
         Debug flag.
     n_iters : int, optional
-        Number of ietrations for hyperparatemer optimization, by default None
+        Number of iterations for hyperparameter optimization, by default 0.
+    use_imputer : bool, optional
+        Flag to fit an imputer, by default False.
 
     Returns
     -------
@@ -79,16 +83,46 @@ def get_model(n_features, n_targets, n_jobs, debug, n_iters=0):
         #     refit=True,
         # )
 
+    if use_imputer:
+        model = make_pipeline(KNNImputer(), model)
+
     print(f"Predicting {n_targets} circuits with {n_features} KDTs")
 
     return model
 
 
 def get_rf_space():
-    """Retrieve minimal hyperparameter space for a Ranndom Forest whose number of base
+    """Retrieve minimal hyperparameter space for a Random Forest whose number of base
     learners are going to be used as an expandable resource while optimizing."""
 
     return {
         "max_depth": np.arange(2, 9, 1),
         "max_features": np.arange(0.1, 0.6, 0.1),
     }
+
+
+def extract_estimator(model):
+    """Extract the final estimator from a sklearn pipeline.
+
+    Parameters
+    ----------
+    model : sklearn Pipeline, Estimator, Optimizer
+        Fitted model.
+
+    Returns
+    -------
+    sklearn Estimator
+        The final estimator.
+    """
+    if isinstance(model, Pipeline):
+        estimator = model[-1]
+        if isinstance(estimator, (HalvingRandomSearchCV, RandomizedSearchCV)):
+            estimator = estimator.best_estimator_
+    elif isinstance(model, (HalvingRandomSearchCV, RandomizedSearchCV)):
+        estimator = model.best_estimator_
+    elif isinstance(model, RandomForestRegressor):
+        estimator = model
+    else:
+        raise NotImplementedError("Model no yet implemented.")
+
+    return estimator

drexml/plotting.py

@@ -244,7 +244,7 @@ def plot_metrics(self, width=2.735, output_folder=None):
         """
         Read the drexml results TSV file and plot it. The R^2 confidence interval for the
         mean go to y-axis, whereas the x-axis shows the 95% interval for the Nogueiras's
-        stability stimate.
+        stability estimate.
 
         Parameters
         ----------

drexml/postprocess.py

@@ -47,7 +47,7 @@ def fdr(pvalues):
 
 
 def test_ora_drugs_from_files(shap_sel_path, results_path, db_path=None, atc_path=None):
-    """Helper function fo over representation analysis of ATC classifiction across
+    """Helper function fo over representation analysis of ATC classification across
       levels for SHAP selected drugs.
 
     Parameters
@@ -88,7 +88,7 @@ def test_ora_drugs_from_files(shap_sel_path, results_path, db_path=None, atc_pat
 
 
 def test_ora_drugs_from_frames(shap_selection_df, drugbank_df, atc_df):
-    """Over representation analysis of ATC classifiction across levels for SHAP selected drugs.
+    """Over representation analysis of ATC classification across levels for SHAP selected drugs.
 
     Parameters
     ----------

drexml/pystab.py

@@ -18,9 +18,9 @@
 
 def nogueria_test(pop_mat, alpha=0.05, as_dict=False):
     """Let X be a feature space of dimension `n_features` and `pop_mat` a binary matrix
-    of dimension `(n_samples, n_festures)` representing `n_samples` runs of a feature
+    of dimension `(n_samples, n_features)` representing `n_samples` runs of a feature
     selection algorithm over X (with respect to a response). This function computes the
-    Nogueira stability estimate, error, variance and confindece interval.
+    Nogueira stability estimate, error, variance and confidence interval.
 
     Parameters
     ----------

drexml/utils.py

@@ -109,7 +109,7 @@ def get_stab(data_folder, n_splits, n_cpus, debug, n_iters):
     debug : bool
         Debug flag, by default False.
     n_iters : int
-        Number of hyperparemeter optimization iterations.
+        Number of hyperparameter optimization iterations.
 
     Returns
     -------
@@ -125,6 +125,8 @@ def get_stab(data_folder, n_splits, n_cpus, debug, n_iters):
     features_orig_fpath = data_folder.joinpath("features.jbl")
     features_orig = joblib.load(features_orig_fpath)
 
+    use_imputer = features_orig.isna().any(axis=None)
+
     targets_orig_fpath = data_folder.joinpath("target.jbl")
     targets_orig = joblib.load(targets_orig_fpath)
 
@@ -141,7 +143,9 @@ def get_stab(data_folder, n_splits, n_cpus, debug, n_iters):
     n_features = features_orig.shape[1]
     n_targets = targets_orig.shape[1]
 
-    model = get_model(n_features, n_targets, n_cpus, debug, n_iters)
+    model = get_model(
+        n_features, n_targets, n_cpus, debug, n_iters, use_imputer=use_imputer
+    )
 
     return model, stab_cv, features_orig, targets_orig
 
@@ -226,7 +230,7 @@ def get_cuda_version():  # pragma: no cover
 
 
 def check_gputree_availability():  # pragma: no cover
-    """Check if GPUTree has been corectly compiled."""
+    """Check if GPUTree has been correctly compiled."""
     try:
         shap.utils.assert_import("cext_gpu")
         return True
@@ -356,7 +360,7 @@ def read_seed_genes(config):
     Returns
     -------
     dict
-        Updated conig dict.
+        Updated config dict.
 
     Raises
     ------

pyproject.toml

@@ -21,7 +21,7 @@ dependencies = [
     "pystow>=0.5.0",
 ]
 name = "drexml"
-version = "1.0.3"
+version = "1.0.4"
 description = "(DRExM³L) Drug REpurposing using and eXplainable Machine Learning and Mechanistic Models of signal transduction\""
 readme = "README.md"
 

tests/example_withnan.env

@@ -0,0 +1,3 @@
+seed_genes=9901
+use_physio=true
+activity_normalizer=true

tests/shap_selection_withnan.tsv

@@ -0,0 +1,3 @@
+	18	19	25	43	47	55	87	100	124	125	126	134	135	136	140	142	146	147	148	150	151	152	153	154	155	185	210	213	217	231	238	240	251	279	291	292	293	301	302	306	307	308	331	345	351	354	367	462	471	476	486	495	496	526	528	552	553	554	558	590	595	596	613	624	632	645	657	659	673	683	695	715	716	718	720	721	727	759	760	761	762	766	771	774	775	776	777	778	779	781	782	783	784	785	786	796	797	799	801	819	836	846	847	886	887	915	916	917	919	920	930	931	933	941	942	943	945	960	974	978	1019	1021	1043	1080	1128	1129	1131	1132	1133	1135	1136	1137	1139	1141	1143	1152	1158	1160	1181	1233	1234	1244	1268	1269	1277	1280	1281	1312	1361	1373	1374	1376	1436	1438	1439	1441	1493	1583	1584	1586	1588	1595	1621	1633	1636	1644	1646	1719	1723	1756	1786	1800	1803	1806	1812	1813	1814	1815	1816	1890	1892	1901	1903	1909	1910	1938	1950	1956	1969	1991	2057	2064	2066	2099	2100	2110	2146	2147	2149	2152	2153	2155	2157	2158	2159	2160	2161	2162	2165	2205	2206	2207	2209	2212	2213	2214	2215	2224	2243	2244	2246	2247	2260	2261	2263	2264	2266	2280	2321	2322	2324	2335	2339	2342	2346	2444	2475	2492	2495	2512	2520	2548	2550	2554	2555	2556	2557	2558	2559	2560	2561	2562	2566	2593	2595	2618	2642	2677	2690	2692	2740	2741	2742	2798	2864	2890	2902	2903	2904	2906	2908	2915	2936	2937	2977	2984	3028	3033	3039	3043	3053	3061	3062	3065	3066	3140	3156	3161	3242	3251	3269	3274	3290	3350	3351	3352	3355	3356	3357	3358	3359	3360	3363	3383	3416	3417	3418	3454	3455	3458	3459	3460	3480	3551	3553	3554	3559	3560	3561	3563	3566	3567	3568	3569	3570	3590	3593	3605	3614	3615	3643	3674	3676	3683	3685	3688	3690	3695	3716	3717	3718	3725	3736	3737	3738	3739	3741	3742	3743	3744	3745	3746	3747	3748	3750	3751	3752	3757	3758	3760	3764	3767	3768	3776	3777	3778	3783	3784	3791	3815	3818	3849	3859	3932	3939	3945	3953	3973	3992	4023	4049	4070	4074	4128	4129	4130	4133	4134	4137	4151	4157	4158	4160	4214	4233	4306	4311	4352	4360	4363	4535	4543	4544	4547	4548	4594	4731	4790	4837	4842	4843	4846	4852	4860	4881	4882	4914	4915	4916	4921	4985	4986	4988	5021	5025	5053	5133	5139	5141	5142	5143	5144	5156	5159	5163	5222	5228	5241	5290	5293	5294	5320	5327	5340	5406	5422	5423	5426	5427	5465	5467	5468	5478	5535	5564	5578	5594	5595	5600	5604	5605	5618	5624	5627	5689	5690	5693	5696	5698	5699	5724	5731	5732	5733	5734	5737	5739	5740	5742	5743	5745	5746	5894	5914	5915	5916	5972	5979	6093	6098	6121	6122	6240	6241	6256	6257	6258	6261	6262	6323	6324	6326	6327	6328	6329	6330	6331	6334	6335	6336	6337	6338	6339	6340	6344	6389	6403	6462	6476	6510	6524	6528	6529	6530	6531	6532	6535	6546	6550	6557	6558	6559	6560	6571	6608	6616	6646	6713	6714	6715	6716	6718	6751	6752	6753	6754	6755	6785	6833	6843	6844	6850	6869	7010	7035	7037	7040	7048	7054	7067	7068	7084	7097	7099	7124	7132	7134	7150	7153	7155	7166	7172	7173	7201	7253	7276	7277	7283	7296	7298	7299	7357	7421	7422	7423	7442	7450	7498	7514	7525	7846	7852	8001	8399	8513	8566	8600	8639	8654	8841	8843	8856	8911	8912	8913	8972	8989	9056	9254	9312	9340	9356	9376	9415	9453	9475	9568	9607	9734	9817	9900	9971	10013	10038	10039	10060	10105	10203	10226	10266	10267	10268	10280	10381	10383	10544	10673	10800	10846	11188	11255	11280	23193	23475	23620	23632	23657	25824	27010	27032	29126	29881	50484	50632	50964	51174	51175	51185	51284	51305	51561	53637	54106	54107	54363	55107	55312	55800	55867	56655	57053	57105	57468	57823	59340	64127	64805	79001	79054	79581	79714	81027	84649	84889	116085	116443	116444	116447	121278	127833	134864	154807	162514	203068	255738	338442	338557	548596
+P.hsa03320.28	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0
+P.hsa04920.43	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	1	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0