#!/usr/bin/env python
"""
survivalPredictor - Survival prediction uing gene expression data 
Version: 0.3
@author: Aziz Khan
"""
import sys
import os
from string import lower
import numpy as np
from scipy import interp
import matplotlib
matplotlib.use('Agg')
import pylab as pl
import pandas as pd
from sklearn.metrics import roc_curve, auc, precision_recall_curve
from sklearn.cross_validation import StratifiedKFold
from sklearn.preprocessing import Imputer
import datetime
from optparse import OptionParser
import warnings
warnings.filterwarnings('ignore')
from sklearn import preprocessing
from sklearn.svm import SVC
import survivalPredictor
#from sklearn.model_selection import StratifiedKFold
from sklearn.feature_selection import RFECV

__version__ = '0.3'


#data_path = "/Users/azizkhan/NORBIS/survivalPredictor/survivalPredictor"
data_path = os.path.dirname(survivalPredictor.__file__)
   
def feature_combinations(train_df, feature_comb):
     #X = train_df.values
    if feature_comb[0] == 'Top 10':
        X = train_df.iloc[:,[1400,15905,6836,4387,14160,3674,15120,2442,7860,1986]]

    elif feature_comb[0] == 'Top 50':
        X = train_df.iloc[:,[1400,15905,6836,4387,14160,3674,15120,2442,7860,1986,16,503,3392,6018,5366,15599,1294,16412,11877,636,12564,5023,6237,9097,1668,15060,3066,10345,16728,14200,4126,12955,14412,7604,15078,8220,12958,9131,10677,7688,9164,1518,1410,12051,3304,10589,17543,13633,189,1305]]

    elif feature_comb[0] == 'Top 100':
        X = train_df.iloc[:,[1400,15905,6836,4387,14160,3674,15120,2442,7860,1986,16,503,3392,6018,5366,15599,1294,16412,11877,636,12564,5023,6237,9097,1668,15060,3066,10345,16728,14200,4126,12955,14412,7604,15078,8220,12958,9131,10677,7688,9164,1518,1410,12051,3304,10589,17543,13633,189,1305,16209,16959,8006,9540,11882,926,12277,3287,15082,10286,1089,4206,13340,17582,7231,6315,11315,497,3318,1911,1387,7697,11363,14560,11539,17706,5594,14648,8804,11558,8476,6934,9466,2191,6720,6722,6096,16185,7297,8498,10489,11652,17586,9069,2734]]
    
    elif feature_comb[0] == 'Top 200':
        X = train_df.iloc[:,[1400,15905,6836,4387,14160,3674,15120,2442,7860,1986,16,503,3392,6018,5366,15599,1294,16412,11877,636,12564,5023,6237,9097,1668,15060,3066,10345,16728,14200,4126,12955,14412,7604,15078,8220,12958,9131,10677,7688,9164,1518,1410,12051,3304,10589,17543,13633,189,1305,16209,16959,8006,9540,11882,926,12277,3287,15082,10286,1089,4206,13340,17582,7231,6315,11315,497,3318,1911,1387,7697,11363,14560,11539,17706,5594,14648,8804,11558,8476,6934,9466,2191,6720,6722,6096,16185,7297,8498,10489,11652,17586,9069,2734,16953,14815,7016,14323,755,1466,1804,4797,9077,16063,17536,11242,10959,12073,15699,11843,3944,12675,16299,7040,16907,15304,7091,11057,8507,16466,13655,6940,10512,16086,13361,3893,11639,9436,1563,3757,14061,9769,8065,12725,10359,16613,13955,50,9431,15897,8894,2656,157,9454,1453,2058,7268,12311,2936,13527,1642,9139,15713,16917,1440,15129,4663,13738,4337,10498,4648,14818,11084,960,11586,5407,1904,10381,7797,12043,750,7587,16302,3668,1519,11990,3924,14874,10125,16864,11259,14036,11905,14238,53,16087,14614]]
    
    elif feature_comb[0] == 'Top 500':
        X = train_df.iloc[:,[1400,15905,6836,4387,14160,3674,15120,2442,7860,1986,16,503,3392,6018,5366,15599,1294,16412,11877,636,12564,5023,6237,9097,1668,15060,3066,10345,16728,14200,4126,12955,14412,7604,15078,8220,12958,9131,10677,7688,9164,1518,1410,12051,3304,10589,17543,13633,189,1305,16209,16959,8006,9540,11882,926,12277,3287,15082,10286,1089,4206,13340,17582,7231,6315,11315,497,3318,1911,1387,7697,11363,14560,11539,17706,5594,14648,8804,11558,8476,6934,9466,2191,6720,6722,6096,16185,7297,8498,10489,11652,17586,9069,2734,16953,14815,7016,14323,755,1466,1804,4797,9077,16063,17536,11242,10959,12073,15699,11843,3944,12675,16299,7040,16907,15304,7091,11057,8507,16466,13655,6940,10512,16086,13361,3893,11639,9436,1563,3757,14061,9769,8065,12725,10359,16613,13955,50,9431,15897,8894,2656,157,9454,1453,2058,7268,12311,2936,13527,1642,9139,15713,16917,1440,15129,4663,13738,4337,10498,4648,14818,11084,960,11586,5407,1904,10381,7797,12043,750,7587,16302,3668,1519,11990,3924,14874,10125,16864,11259,14036,11905,14238,53,16087,14614,8161,16563,7366,16215,10432,7434,10336,5738,9386,11337,3853,5351,5718,1494,7383,9732,14053,1578,6736,15935,10302,2437,2601,15296,2769,378,7372,14220,17589,1664,14403,1537,5449,14495,3277,7063,1612,12695,10361,12881,12507,4704,14143,8077,14882,6648,11030,15347,8084,13534,6990,7189,15859,8522,5524,12538,11451,11310,9451,1912,14181,6918,5543,16990,14281,6092,8212,4272,6630,826,1850,2410,4587,8941,7151,3139,4146,15972,408,6325,15904,6810,1731,13461,3589,11692,2869,2319,7755,15953,8233,15223,6130,3171,6903,5382,17226,6117,2956,7149,13604,10277,13656,2341,13029,7994,13575,11597,3492,3264,15362,17423,8024,10654,6533,17012,2597,5960,5208,2419,2534,14100,13062,9020,1971,4686,4522,9614,1640,17214,10561,3571,5372,1558,2626,8390,12977,17387,16106,1735,7388,14751,5354,10155,10311,3051,4822,15167,5833,17751,1241,2007,1801,1345,127,4057,9032,11695,14610,2902,9616,4378,17376,5773,5309,3018,1593,3777,4602,52,12007,12366,6098,5401,2743,12184,15561,8317,11258,16517,12634,10080,7964,17735,9360,11761,12196,4278,14478,6304,1706,7614,572,14173,2029,10825,15328,15233,16231,5138,3138,2924,5932,13406,3562,16644,12603,8189,566,718,15698,9149,258,7895,934,8294,2687,8683,4322,16076,8442,16080,6654,13498,9758,9900,9621,14769,13958,17199,10299,2974,108,17293,15392,11846,7801,5381,14006,14185,5064,17771,11631,3923,6289,2194,8305,7356,5072,4111,13338,12275,2492,11521,6183,12868,13687,9004,9458,10113,2934,4802,7564,1562,4088,11479,17812,2028,1437,11814,15760,4607,14576,17145,10524,13920,14397,3340,3918,9826,1401,11190,16248,12090]]    
    
    elif feature_comb[0] == 'Top 1000':

        X = train_df.iloc[:,[1400,15905,6836,4387,14160,3674,15120,2442,7860,1986,16,503,3392,6018,5366,15599,1294,16412,11877,636,12564,5023,6237,9097,1668,15060,3066,10345,16728,14200,4126,12955,14412,7604,15078,8220,12958,9131,10677,7688,9164,1518,1410,12051,3304,10589,17543,13633,189,1305,16209,16959,8006,9540,11882,926,12277,3287,15082,10286,1089,4206,13340,17582,7231,6315,11315,497,3318,1911,1387,7697,11363,14560,11539,17706,5594,14648,8804,11558,8476,6934,9466,2191,6720,6722,6096,16185,7297,8498,10489,11652,17586,9069,2734,16953,14815,7016,14323,755,1466,1804,4797,9077,16063,17536,11242,10959,12073,15699,11843,3944,12675,16299,7040,16907,15304,7091,11057,8507,16466,13655,6940,10512,16086,13361,3893,11639,9436,1563,3757,14061,9769,8065,12725,10359,16613,13955,50,9431,15897,8894,2656,157,9454,1453,2058,7268,12311,2936,13527,1642,9139,15713,16917,1440,15129,4663,13738,4337,10498,4648,14818,11084,960,11586,5407,1904,10381,7797,12043,750,7587,16302,3668,1519,11990,3924,14874,10125,16864,11259,14036,11905,14238,53,16087,14614,8161,16563,7366,16215,10432,7434,10336,5738,9386,11337,3853,5351,5718,1494,7383,9732,14053,1578,6736,15935,10302,2437,2601,15296,2769,378,7372,14220,17589,1664,14403,1537,5449,14495,3277,7063,1612,12695,10361,12881,12507,4704,14143,8077,14882,6648,11030,15347,8084,13534,6990,7189,15859,8522,5524,12538,11451,11310,9451,1912,14181,6918,5543,16990,14281,6092,8212,4272,6630,826,1850,2410,4587,8941,7151,3139,4146,15972,408,6325,15904,6810,1731,13461,3589,11692,2869,2319,7755,15953,8233,15223,6130,3171,6903,5382,17226,6117,2956,7149,13604,10277,13656,2341,13029,7994,13575,11597,3492,3264,15362,17423,8024,10654,6533,17012,2597,5960,5208,2419,2534,14100,13062,9020,1971,4686,4522,9614,1640,17214,10561,3571,5372,1558,2626,8390,12977,17387,16106,1735,7388,14751,5354,10155,10311,3051,4822,15167,5833,17751,1241,2007,1801,1345,127,4057,9032,11695,14610,2902,9616,4378,17376,5773,5309,3018,1593,3777,4602,52,12007,12366,6098,5401,2743,12184,15561,8317,11258,16517,12634,10080,7964,17735,9360,11761,12196,4278,14478,6304,1706,7614,572,14173,2029,10825,15328,15233,16231,5138,3138,2924,5932,13406,3562,16644,12603,8189,566,718,15698,9149,258,7895,934,8294,2687,8683,4322,16076,8442,16080,6654,13498,9758,9900,9621,14769,13958,17199,10299,2974,108,17293,15392,11846,7801,5381,14006,14185,5064,17771,11631,3923,6289,2194,8305,7356,5072,4111,13338,12275,2492,11521,6183,12868,13687,9004,9458,10113,2934,4802,7564,1562,4088,11479,17812,2028,1437,11814,15760,4607,14576,17145,10524,13920,14397,3340,3918,9826,1401,11190,16248,12090,1006,17077,11241,1874,1583,8540,2768,17788,4568,8969,16723,14461,5618,6579,5229,7771,14469,12037,8283,8877,5901,11209,15035,2491,11798,9603,303,11414,14018,3847,6166,17144,11858,7869,12057,5315,7908,3564,12944,12869,7234,16869,10153,16159,7371,5008,10434,10695,7679,15527,14824,16371,16385,12088,10764,16788,13422,15083,17391,5084,5099,13740,13482,7726,2481,17737,549,13265,9588,14844,463,9201,367,15931,2907,3215,13809,4882,5603,3133,16332,138,16963,160,6361,11255,17078,9810,9881,2478,4732,17043,13573,3367,17555,12520,4074,16975,1782,949,777,6271,17540,17413,5857,14339,11925,16479,9080,5203,1085,5757,12578,1316,4550,11236,17699,3956,16342,16793,7393,14083,11769,10312,4405,1023,13334,14059,10358,11413,11887,16534,17053,6248,16705,8825,16504,4043,1899,14189,9902,2617,12023,4462,11453,13725,10393,4012,11976,16245,2632,5385,6991,16152,11391,4784,7346,17416,17457,7893,6761,15031,3462,8559,7341,7753,8836,505,7478,17337,2655,11674,6209,9586,14473,1081,12405,12042,16581,13162,16577,12539,3844,14331,10241,4685,9600,12162,12844,15261,693,5736,9906,5624,15271,17316,6419,6240,7240,4414,3596,6589,9015,11890,14158,14401,10391,17155,17224,12083,15734,15426,7642,12565,10457,10908,6737,1997,13717,15647,8347,16395,15102,418,14548,7068,5611,6145,15683,11362,16773,16849,660,17741,9643,11366,14341,14787,8751,10562,8808,6256,1173,4511,7313,16350,7403,448,14889,9778,4776,12127,8802,15977,11177,11313,14667,5677,7310,10972,5247,5250,5221,14842,12113,3494,11644,8526,13262,659,11464,15721,13791,16179,3915,10569,6474,6915,9401,5343,13237,16219,8982,12465,5188,16205,15427,17814,7281,692,17591,4231,5038,8384,12600,15072,3667,3047,16677,407,840,10459,13514,7899,10357,16692,16242,5087,13295,17530,16056,518,5912,13305,2962,13076,7073,1309,5505,16611,17521,12128,3252,16388,4892,7416,11611,7862,16982,9776,5200,14021,6532,1968,16747,15497,15810,13579,1516,14000,1653,15023,8020,8686,4761,5937,11042,10397,5320,3543,14489,12742,3291,6572,9057,332,13762,1487,10953,11856,17015,17120,15650,5046,550,891,14953,3916,6185,40,16043,16938,17270,9274,12890,14963,15002,2444,16503,5483,15165,9479,9723,1098,2021,1065,17552,1106,4833,5985,15550,3325,6549,6041,13987,4534,1281,9148,2275,11907,8910,1783,10203,14272,1648,3910,3232,10201,14233,16351,11197,11500,12835,6094,11063,7300,13609,1373,996,6278,5586,1004,13643,5678,8215,1219,14356,7136,14686,6290,16066,5948,15070,14076,12914,8794,14533,547,1784,8418,7831,11238,8685,6776,14501,6719,16228,16840,8518,17480,672,2685,12919,12707,90,11305,7541,15811,5092,16471,15478,15966,6779,13462]]
    
    elif feature_comb[0] == 'Top 5000':
        X = train_df.iloc[:,[1400,15905,6836,4387,14160,3674,15120,2442,7860,1986,16,503,3392,6018,5366,15599,1294,16412,11877,636,12564,5023,6237,9097,1668,15060,3066,10345,16728,14200,4126,12955,14412,7604,15078,8220,12958,9131,10677,7688,9164,1518,1410,12051,3304,10589,17543,13633,189,1305,16209,16959,8006,9540,11882,926,12277,3287,15082,10286,1089,4206,13340,17582,7231,6315,11315,497,3318,1911,1387,7697,11363,14560,11539,17706,5594,14648,8804,11558,8476,6934,9466,2191,6720,6722,6096,16185,7297,8498,10489,11652,17586,9069,2734,16953,14815,7016,14323,755,1466,1804,4797,9077,16063,17536,11242,10959,12073,15699,11843,3944,12675,16299,7040,16907,15304,7091,11057,8507,16466,13655,6940,10512,16086,13361,3893,11639,9436,1563,3757,14061,9769,8065,12725,10359,16613,13955,50,9431,15897,8894,2656,157,9454,1453,2058,7268,12311,2936,13527,1642,9139,15713,16917,1440,15129,4663,13738,4337,10498,4648,14818,11084,960,11586,5407,1904,10381,7797,12043,750,7587,16302,3668,1519,11990,3924,14874,10125,16864,11259,14036,11905,14238,53,16087,14614,8161,16563,7366,16215,10432,7434,10336,5738,9386,11337,3853,5351,5718,1494,7383,9732,14053,1578,6736,15935,10302,2437,2601,15296,2769,378,7372,14220,17589,1664,14403,1537,5449,14495,3277,7063,1612,12695,10361,12881,12507,4704,14143,8077,14882,6648,11030,15347,8084,13534,6990,7189,15859,8522,5524,12538,11451,11310,9451,1912,14181,6918,5543,16990,14281,6092,8212,4272,6630,826,1850,2410,4587,8941,7151,3139,4146,15972,408,6325,15904,6810,1731,13461,3589,11692,2869,2319,7755,15953,8233,15223,6130,3171,6903,5382,17226,6117,2956,7149,13604,10277,13656,2341,13029,7994,13575,11597,3492,3264,15362,17423,8024,10654,6533,17012,2597,5960,5208,2419,2534,14100,13062,9020,1971,4686,4522,9614,1640,17214,10561,3571,5372,1558,2626,8390,12977,17387,16106,1735,7388,14751,5354,10155,10311,3051,4822,15167,5833,17751,1241,2007,1801,1345,127,4057,9032,11695,14610,2902,9616,4378,17376,5773,5309,3018,1593,3777,4602,52,12007,12366,6098,5401,2743,12184,15561,8317,11258,16517,12634,10080,7964,17735,9360,11761,12196,4278,14478,6304,1706,7614,572,14173,2029,10825,15328,15233,16231,5138,3138,2924,5932,13406,3562,16644,12603,8189,566,718,15698,9149,258,7895,934,8294,2687,8683,4322,16076,8442,16080,6654,13498,9758,9900,9621,14769,13958,17199,10299,2974,108,17293,15392,11846,7801,5381,14006,14185,5064,17771,11631,3923,6289,2194,8305,7356,5072,4111,13338,12275,2492,11521,6183,12868,13687,9004,9458,10113,2934,4802,7564,1562,4088,11479,17812,2028,1437,11814,15760,4607,14576,17145,10524,13920,14397,3340,3918,9826,1401,11190,16248,12090,1006,17077,11241,1874,1583,8540,2768,17788,4568,8969,16723,14461,5618,6579,5229,7771,14469,12037,8283,8877,5901,11209,15035,2491,11798,9603,303,11414,14018,3847,6166,17144,11858,7869,12057,5315,7908,3564,12944,12869,7234,16869,10153,16159,7371,5008,10434,10695,7679,15527,14824,16371,16385,12088,10764,16788,13422,15083,17391,5084,5099,13740,13482,7726,2481,17737,549,13265,9588,14844,463,9201,367,15931,2907,3215,13809,4882,5603,3133,16332,138,16963,160,6361,11255,17078,9810,9881,2478,4732,17043,13573,3367,17555,12520,4074,16975,1782,949,777,6271,17540,17413,5857,14339,11925,16479,9080,5203,1085,5757,12578,1316,4550,11236,17699,3956,16342,16793,7393,14083,11769,10312,4405,1023,13334,14059,10358,11413,11887,16534,17053,6248,16705,8825,16504,4043,1899,14189,9902,2617,12023,4462,11453,13725,10393,4012,11976,16245,2632,5385,6991,16152,11391,4784,7346,17416,17457,7893,6761,15031,3462,8559,7341,7753,8836,505,7478,17337,2655,11674,6209,9586,14473,1081,12405,12042,16581,13162,16577,12539,3844,14331,10241,4685,9600,12162,12844,15261,693,5736,9906,5624,15271,17316,6419,6240,7240,4414,3596,6589,9015,11890,14158,14401,10391,17155,17224,12083,15734,15426,7642,12565,10457,10908,6737,1997,13717,15647,8347,16395,15102,418,14548,7068,5611,6145,15683,11362,16773,16849,660,17741,9643,11366,14341,14787,8751,10562,8808,6256,1173,4511,7313,16350,7403,448,14889,9778,4776,12127,8802,15977,11177,11313,14667,5677,7310,10972,5247,5250,5221,14842,12113,3494,11644,8526,13262,659,11464,15721,13791,16179,3915,10569,6474,6915,9401,5343,13237,16219,8982,12465,5188,16205,15427,17814,7281,692,17591,4231,5038,8384,12600,15072,3667,3047,16677,407,840,10459,13514,7899,10357,16692,16242,5087,13295,17530,16056,518,5912,13305,2962,13076,7073,1309,5505,16611,17521,12128,3252,16388,4892,7416,11611,7862,16982,9776,5200,14021,6532,1968,16747,15497,15810,13579,1516,14000,1653,15023,8020,8686,4761,5937,11042,10397,5320,3543,14489,12742,3291,6572,9057,332,13762,1487,10953,11856,17015,17120,15650,5046,550,891,14953,3916,6185,40,16043,16938,17270,9274,12890,14963,15002,2444,16503,5483,15165,9479,9723,1098,2021,1065,17552,1106,4833,5985,15550,3325,6549,6041,13987,4534,1281,9148,2275,11907,8910,1783,10203,14272,1648,3910,3232,10201,14233,16351,11197,11500,12835,6094,11063,7300,13609,1373,996,6278,5586,1004,13643,5678,8215,1219,14356,7136,14686,6290,16066,5948,15070,14076,12914,8794,14533,547,1784,8418,7831,11238,8685,6776,14501,6719,16228,16840,8518,17480,672,2685,12919,12707,90,11305,7541,15811,5092,16471,15478,15966,6779,13462,6343,8479,2022,11275,9184,17615,9299,5935,4680,15260,9366,3830,1613,7910,11399,17246,2032,16544,14344,6954,10797,5512,12017,12393,2296,7929,14460,7622,12982,4576,17532,14885,12299,12752,15383,17223,2812,15228,15105,12375,987,6833,3581,10000,9185,14209,12376,1467,7354,11947,6297,6785,5606,1512,7878,10010,8209,4747,16017,11523,15306,8655,10346,13961,9055,4213,3037,13787,5158,13564,17541,5699,10572,14713,7542,4191,6503,17219,12486,270,13969,16457,6827,2628,8302,6970,10861,1817,3540,6400,8112,13927,5209,1649,7851,1509,3024,15662,14674,10890,2326,11927,13394,2162,17080,5097,13072,621,10767,16296,17036,8402,15805,14270,2804,1747,1616,16288,13947,9268,4085,16126,5911,11678,4564,14613,5762,13537,16672,7760,16171,16202,14789,14131,4301,4110,16029,17673,12990,11267,1191,11619,6927,288,17619,10177,6644,10862,13234,11854,13137,5965,9706,998,4843,9583,1365,9357,17495,6619,1634,7898,17616,15660,758,915,13205,4251,12617,11748,3097,4743,11930,1159,15700,17188,13997,16257,11730,3611,158,13850,1539,7512,3025,10857,16749,4007,5946,4352,10735,2728,3425,8252,4363,15312,7552,9297,12818,17421,6616,756,15658,12962,10438,6389,4165,10412,9576,1244,14479,901,13810,5849,3451,13525,9059,482,14349,12472,13362,15870,12703,9587,10413,12351,15505,5252,14106,15239,16852,4805,17103,13556,10086,2759,15666,17770,7055,8660,4531,1431,3914,3927,3353,13098,3008,9898,11140,1535,12161,972,8906,17501,48,5119,1248,7636,769,4790,16635,10119,13396,10380,17472,8535,6502,12811,15522,2730,9864,922,8933,4513,15058,552,89,16855,7029,15361,13672,9924,5329,13288,4699,2543,6824,17412,17207,5583,9642,12706,1548,17276,795,13910,9960,12379,15089,984,15879,14319,3389,7199,9891,3411,1626,2361,8785,12900,15173,13884,15313,16941,2229,6668,6259,13607,5027,9585,11400,11012,2248,8868,13479,15502,17313,1377,10485,15483,7773,10518,12888,14166,3662,16049,4425,10420,14922,1072,589,17705,11859,16423,12168,6740,8815,4309,9474,13471,14454,2832,9376,12552,1381,16918,4503,100,12722,4448,4145,12395,13771,5689,374,3892,647,310,9923,5125,13572,9715,10294,6019,12574,2148,1,3093,6801,2910,17733,9560,3827,16111,12243,3290,16436,249,7858,5242,2762,12518,15867,1739,11085,15779,3011,2640,3946,6786,9533,1600,7344,10842,6681,3906,5539,12830,2978,3169,7309,619,15791,8666,11327,10193,8732,17023,4053,16151,16325,6807,1873,1255,2653,2949,11458,17592,17329,16055,10429,4250,15534,1835,5920,14617,10968,14603,14486,14101,15073,2512,4786,16470,10588,2797,1432,6428,14598,5785,5186,5798,8606,3653,4455,12458,13083,16971,1458,17483,14696,578,5532,10753,13503,7969,13176,5147,14596,13090,8398,711,9821,16442,4210,12026,15445,16310,17200,16606,14607,8134,15022,6152,1983,16099,13515,1220,17280,14572,9813,7987,12738,10009,3397,16208,13457,2082,13906,11452,12870,9909,7414,5628,10289,13070,12605,5172,14735,5772,6977,762,17157,2438,5768,14287,6408,16767,15563,17128,2625,6121,4591,7561,6262,1655,15338,13990,15667,7963,14074,228,3227,824,15376,1011,8746,16144,6257,8053,8256,12577,14600,4209,12987,298,10732,3843,3070,16900,17220,7435,3516,3384,13445,16787,15681,2802,11694,17716,14927,10970,16870,12339,15148,13096,858,14013,3535,16552,7891,15956,5507,10079,16921,14744,13466,11324,13853,10099,6705,1800,15055,9993,1095,15415,9636,17549,11113,4824,5843,16300,5124,17424,4171,17196,9927,14211,3209,15454,7510,17409,17162,9522,14790,181,7375,17638,3497,11837,16197,15558,16853,14097,5600,12147,16204,13571,4965,13610,1851,3421,8491,14669,10087,625,6923,16803,17165,13491,11908,3156,3140,6545,13360,1153,2090,13064,16033,2995,15287,10821,968,10253,15780,9335,10796,9800,16295,12313,14277,9958,8690,8742,1484,2067,5406,6435,8411,1682,1352,8610,6471,10595,6971,7314,14969,15554,5802,4501,5791,13569,10506,6596,12543,3108,6916,3740,12563,15661,12292,1499,10556,5599,13093,3124,12776,10168,5981,6171,11006,8887,13159,15597,16757,6896,14583,9048,13264,16335,2428,4982,3715,2206,16970,17330,7708,17055,14904,9351,10283,8552,14971,8709,12532,14858,16883,9820,3538,4597,11314,5936,5257,2149,1047,14048,14241,11485,15508,13006,9653,11055,7051,12506,17650,6109,1193,8642,3031,1791,10178,11036,17590,5818,5118,17089,14841,7081,2002,7160,5192,13023,13904,5786,9028,862,2705,1752,25,16743,14867,12318,12194,4514,7765,7792,2678,17464,1036,3182,2527,8427,17576,10700,13299,14540,8771,4852,5953,2574,13679,16085,10431,4498,2482,5302,10297,785,10543,5288,7126,2383,14475,2898,4985,9145,5460,8466,14733,13051,2329,16387,5739,12155,4879,8974,10626,11647,8532,7436,1872,16752,7936,6353,8704,13855,14917,14278,1629,7813,8684,16536,9134,1770,3900,14620,6207,639,6931,16143,17559,13261,1953,4927,11172,14041,15359,2487,13065,10552,8180,15174,13152,6796,1651,12293,6110,9575,10067,9779,1020,12860,12021,17641,8678,2208,2980,6411,13697,12664,8198,17781,17106,9972,3548,14065,6438,9123,15059,11892,5106,982,12015,5194,13785,5807,17228,1328,7875,14770,2834,7392,13731,4856,6937,10644,15200,16871,14402,12733,7977,4370,7347,11855,12137,5282,10924,14279,4775,13323,1636,4181,3036,15503,2216,15645,3339,11588,9031,17563,14849,1104,9320,1183,8564,5264,10162,1673,14032,3998,4060,3704,17468,1793,9054,2213,16119,17319,8738,2253,2868,9107,3515,7607,10343,12312,445,3424,14831,12802,13730,3307,10964,5143,3815,14647,14520,13352,5036,9932,16604,5766,2158,13100,16954,9181,10839,16670,12654,2831,17159,1194,6553,3196,13293,661,811,9133,8595,7437,7299,5919,14680,499,5499,6739,15747,13021,13198,2039,3315,801,6381,16262,110,10585,4125,9651,11328,13630,16735,9001,9488,1991,2129,14219,3489,11776,1032,17273,12640,13108,2749,17461,7432,4073,3789,13822,632,2477,15656,14827,2888,11950,38,8333,9529,2522,4160,3631,6466,7387,9971,12678,5515,7671,11033,6742,2827,13019,8493,4556,2412,15386,676,13235,4469,7381,1946,1569,7143,6581,339,3764,13828,16339,3010,10611,2379,4999,5380,15121,11243,11765,1256,15482,14883,16892,745,1137,6061,11928,6317,15319,8791,9928,13577,14774,9994,3300,12570,8122,8038,13917,1988,26,15012,5743,825,4897,10043,8027,7345,12214,9613,2711,5083,2726,11936,14391,1013,7724,10849,14814,14137,17797,16032,1299,11603,10516,1659,2819,9569,3444,1443,13529,7274,2580,3298,5685,15634,15848,7694,9169,11772,14120,11013,1703,14984,17169,15423,2576,8647,5134,5463,12153,8091,4683,3507,5675,6412,12871,12934,10446,1740,1901,8374,10248,7362,12248,14422,17436,8586,10250,1144,7966,8404,3846,8279,17652,6234,16091,11721,2219,15252,14161,8335,5976,16328,4032,17695,3547,10268,449,5411,4228,6642,3744,2552,9460,15547,10238,2057,8121,14057,15090,10452,5614,17425,17488,10905,13541,4943,12531,11820,5625,8872,15028,9108,2399,15676,13142,6658,13654,12081,8019,9772,12895,11756,134,1287,6539,15240,4906,3422,13736,3151,9719,16729,8858,13153,1896,14619,12807,15929,4631,7218,10973,15222,17763,11556,7599,9974,14450,9629,11685,7399,12031,13833,837,16779,2984,2572,4593,4004,15630,3693,15590,15831,41,15572,13001,13407,16258,13429,12573,3453,5473,16124,6598,14426,9700,13544,17248,7843,4285,5707,17107,12242,8640,12250,11753,12078,1088,7291,5010,3997,13120,14203,13595,3936,8054,16326,895,7238,680,15569,13468,4537,14973,10671,7284,9448,11680,7750,1195,4080,12709,6286,15933,17325,16498,16000,12452,9589,2694,16972,13532,16386,772,10629,9662,6792,11446,15471,16805,15007,2312,2124,7591,11767,6978,4294,5529,595,1212,17595,861,9506,14590,10792,15065,9265,3935,17453,10604,10266,7263,13238,8344,2188,8468,4938,15034,6695,13398,2630,16792,11827,12002,2345,1214,4167,14499,4002,8101,1421,9679,11591,12418,11871,8976,15282,3075,3938,17553,360,9086,12462,14911,4672,5861,11791,506,9110,11607,4297,658,7617,5349,13662,13150,5290,6279,11279,13337,7287,15369,15549,4372,7759,15609,4701,5467,12772,2809,9290,13964,6274,13554,2777,16580,14371,10663,16021,7115,9321,7394,6947,7531,15891,8094,7853,8534,10884,9818,14456,15825,15000,5459,10020,17098,10711,16501,9842,15594,5884,16357,12889,3959,6467,8755,925,16308,17494,9049,9977,380,14365,7825,14333,10166,10014,5057,9216,12307,7183,14019,11474,14665,1576,16101,15189,1302,939,12592,2756,4665,9999,10719,2455,5215,3856,5560,2781,7864,14793,7719,11513,3666,5327,7924,727,8123,16191,928,8553,17070,4183,12569,1505,430,14129,3120,16762,8057,7468,4295,5234,11946,15677,9591,4151,11934,5384,8840,3931,17309,5291,5565,3798,13981,11835,2280,16439,1777,8323,13970,10591,9013,11906,13857,10148,674,17640,9125,232,2826,13913,7254,3445,9223,5272,9160,8114,4029,14920,1349,7628,15069,17545,271,942,7629,3559,2381,903,1296,8556,8572,10775,11145,17577,11329,17284,7947,722,16141,1819,9244,6650,14464,8228,1861,6784,3302,10321,3183,13170,12029,14348,10836,6667,12301,4689,7882,12773,8973,1831,8944,11633,11099,12621,11922,3174,2991,9043,9626,6125,3201,16038,3330,11175,12933,16273,8780,10606,11116,9670,5897,12349,13570,1279,15795,9599,13427,10971,14025,2735,4333,7191,7323,9918,5268,3638,15485,15921,8711,3065,3331,2953,5518,8138,3651,11222,16196,9089,13651,9868,12645,13983,13183,13134,6211,10930,1182,12734,14795,7777,1482,2074,13889,226,13956,2232,4961,489,3455,4090,7662,5847,10076,6219,10011,102,16996,8030,8176,13892,17789,2700,4828,14833,11201,9190,10422,7426,3184,17350,15307,6891,10280,15847,17614,5735,9660,16006,7467,4803,15111,10635,16047,11110,15487,11735,12323,11283,13741,5668,1041,515,12536,4135,12527,6592,14782,14745,3160,4123,148,11123,2795,15441,16897,6113,4189,12971,1358,4612,353,2867,13637,254,2958,16904,831,9877,8861,12404,2825,7406,5133,10885,5256,11423,7577,13914,2392,14661,635,9844,2878,17331,5666,11965,17604,14183,5061,12985,13037,11083,17813,8966,6703,9558,13954,4641,7739,17240,9796,5418,4416,8854,7465,3594,9608,15695,7380,11334,17297,13122,9526,7734,1107,12482,9595,12412,6332,13500,5674,3828,10016,10900,17490,9552,1120,15818,16589,8977,5988,13283,11804,4169,15430,17777,10727,12635,10725,10035,323,3393,364,15025,4142,3553,666,366,15983,10339,15009,16203,16903,10399,132,9006,4912,9472,12130,1732,10895,8806,7338,14677,379,11066,11388,5885,9317,16174,14320,6850,6294,5519,13803,412,10978,12060,11870,12433,12133,13526,17632,5630,17486,15926,17798,4748,16816,3701,7014,8967,847,11832,9283,10648,14668,3778,15821,9200,9412,2212,1837,5006,14434,2558,7545,14250,13670,8199,4069,511,9014,8499,2269,3282,12526,371,16379,13871,12619,1472,6221,13621,15580,710,8878,8496,4419,8635,1834,13267,16689,16737,4860,12535,2308,14915,6420,16405,16138,7276,5126,8792,7747,12679,13014,390,4211,10167,17796,8070,13975,12125,12103,9689,10586,4771,11382,1411,10975,16131,13753,13411,6773,7235,13418,8758,1787,12547,10832,1492,8352,2,4199,10110,11444,14712,4488,10550,9468,14633,7721,15785,10303,1759,6682,3286,6625,17039,3410,2050,5860,15593,6943,10783,13073,12639,17285,1290,4793,3785,8080,17606,9388,13512,6013,6609,15781,5795,5593,11886,1763,16170,1622,9106,5375,2773,1257,2954,1857,12049,6798,11788,7207,9914,2659,2218,5378,12580,10887,3189,4581,10333,9590,11697,7821,16488,6382,872,11296,10403,8336,3319,16979,6067,12826,8784,9934,33,16083,6001,1527,8855,7114,2083,197,17011,13190,15258,17149,4305,9547,11711,471,14517,13270,4254,5141,16720,2131,17726,1247,11308,11501,131,11515,16924,2398,11774,1423,13705,14515,11763,15968,14493,9193,13111,5667,12911,12368,13481,13060,12643,3745,13224,17094,8743,10643,3259,4640,5883,4728,9411,4539,15688,4249,12616,906,6498,779,5974,17058,10211,13674,14966,8470,9494,13351,9850,6307,17396,2209,11559,13501,17388,8715,16525,15314,5155,17557,791,2036,9756,8620,484,14589,17263,5144,12256,17644,2011,3592,7691,14872,8451,9067,6795,8125,9661,5838,6046,14020,1823,10287,11538,289,12708,5746,6191,1780,13393,3060,13202,587,1422,12020,5609,810,17807,13765,3220,13860,789,13105,15244,812,7605,8482,13938,12177,13931,13642,4763,13530,6528,9828,14636,15757,15149,10121,15671,6496,4639,13823,4902,9896,15003,10220,13442,14167,16448,1278,16626,11129,17323,6085,1709,9503,15584,15947,142,1705,10763,7246,9246,9711,13048,4707,12753,15,7794,1258,7304,400,9570,938,2560,3161,7155,1430,6752,6743,7877,4463,11365,167,4652,2033,13940,5871,8272,3542,3202,83,4395,4523,15963,7093,3681,4059,4559,4466,2479,3428,6543,12803,11245,17645,15621,7229,14366,11618,7735,10567,8222,8143,16740,17353,9682,9728,5353,14879,11154,10632,743,16833,8612,3661,14086,8829,17384,13558,136,383,4573,12750,3067,12528,5696,15276,3035,1798,8724,11520,11350,12735,8037,6684,5171,14997,8940,3021,2434,2524,14253,1848,11569,3316,10170,2380,3032,2446,11573,17482,3126,12625,6228,15861,14635,9738,4713,1809,8929,16770,16795,7979,10897,17371,10684,7214,14099,17806,9691,8769,13478,9281,16210,10607,5899,17704,12963,14092,12235,4196,11842,6689,7176,11239,16078,7391,17613,10491,14246,10656,9539,1288,4039,8475,2608,8927,12005,12995,344,790,4808,5770,16176,6631,13038,10141,8692,3266,8706,5873,12956,13749,3192,16403,3234,9695,15168,4143,15005,13840,11926,15576,6832,12092,14362,9177,5404,1538,6816,6672,13370,5998,6066,15955,1150,2602,15577,9559,6610,9175,553,4144,8072,4956,5969,9051,1027,13847,11863,15979,4804,13922,2501,7112,7913,1564,5653,15644,4010,11395,8174,2709,13385,14551,6714,13602,17038,12139,16419,5550,11563,971,6908,6814,9467,10672,12407,3141,8444,6933,16574,9962,13049,14231,1327,17612,1730,15726,4052,14978,10405,784,14201,17478,16992,1907,9884,3205,2933,15284,8917,7828,11213,8288,2137,3641,9456,9913,4246,16212,3809,14188,16655,9425,2986,15705,17310,17599,1887,10580,1897,3881,11396,9233,8672,6334,16497,4351,12102,8473,2499,2616,6567,5429,3107,13192,14410,9251,13356,7938,8863,11418,4914,12994,5754,8194,17398,13663,342,5545,11288,1449,7330,5394,1017,571,15136,15951,1379,1674,10101,3373,9476,15432,14012,974,9350,6538,5498,15552,7084,9508,6568,7054,5540,16669,13354,5822,12213,10999,7163,13249,1016,3231,330,8668,109,4876,11303,10765,12494,12891,370,3517,17193,6367,10637,1980,16287,10941,6735,1350,12928,4960,16507,11570,1996,2896,1665,15331,17288,4049,15548,7833,10228,466,8120,5464,7511,5332,6853,16353,3609,14118,17132,5214,3208,7786,5616,12394,9997,6510,1314,11818,15194,15293,6456,15710,5197,1860,9975,9094,15428,5021,10028,9937,61,5226,8218,6035,707,9755,5152,3157,878,10912,3013,4255,914,5956,4287,5925,6014,14127,12343,10046,294,9154,15745,11655,15088,3348,283,3823,2485,4779,10307,17007,15605,7382,7861,12252,3213,10265,7815,13402,2134,4717,5727,13306,5701,3320,14764,8567,17506,5346,7796,2214,1378,4490,5308,11021,5266,7780,6613,16399,13505,14332,17312,6141,1855,16404,11894,4262,13508,9339,7897,2078,4270,1623,17029,742,4992,7874,16044,6716,637,1001,796,11385,7499,2977,6957,2127,4258,9839,10624,17274,17025,5262,3551,11341,12372,3644,14062,7536,1671,7556,5547,13282,9296,8330,1541,8354,7725,6548,16854,2439,17760,5018,16890,15322,13439,14994,7729,11000,6603,9514,3366,2873,11090,8874,11278,7698,9130,14247,7101,9836,1059,17002,145,3595,2241,8644,15315,17130,17713,3711,4529,7472,11,2054,16452,5474,17092,10197,4200,15291,6164,6169,7169,10630,9857,16605,16586,7921,908,7293,4380,1658,4814,1058,13372,16628,12825,15017,11677,15812,7469,8789,14302,1154,4884,9440,4840,11654,5951,2079,9637,8345,17328,376,286,10985,10937,10819,13706,10563,4164,13112,7130,17753,14931,11584,1592,5490,14130,13808,16701,6504,6111,10660,34,11821,7824,10720,13359,4443,14329,11344,14322,559,3233,15564,13703,6583,2863,2549,1708,10194,1322,97,3811,4453,13389,15738,2610,17630,1608,8565,7693,6306,5117,12813,5,5771,14918,12916,5074,4475,5520,1155,783,3826,4896,4321,2973,9534,9551,4252,5012,1164,4800,11249,4398,7409,14128,1940,12357,6967,1461,3706,16217,428,16761,2393,5019,4339,8439,2114,13882,14210,1409,590,5224,6973,4477,9424,1371,2645,6476,11122,4283,10377,5090,17318,12608,438,11864,10779,11649,8396,17430,9118,3817,9851,14511,5276,4866,5436,17322,3204,730,7522,17618,5287,4312,6866,8244,1948,1497,9546,14588,8127,11097,11264,2476,12227,11713,17018,11325,893,3080,16719,15964,9631,5451,15740,10981,10356,6811,13908,11929,4706,7072,15809,603,8337,7581,15333,4100,11205,10418,3731,9061,6962,721,12800,12910,2880,1666,8441,5240,13841,2557,3593,10974,11272,12839,11101,1692,6675,15207,4402,13475,11372,2889,6635,10366,12923,12937,4244,16991,4891,15567,12148,16778,4412,6670,14286,2498,318,17275,10368,1734,2929,7631,13770,5191,2210,5676,5042,7320,16912,13164,14886,7563,17476,14908,16554,8261,11028,7124,6395,6362,11482,16673,11759,2292,8300,9684,13298,7767,16129,17503,16024,8561,8560,9291,3984,8623,4749,12027,16200,16882,3536,2421,3869,11196,4482,6089,11260,13996,5139,5774,5284,6942,2854,17222,7684,5279,3568,11545,5585,11499,11080,686,4690,654,15874,9224,17148,11534,16316,12390,5178,9694,16930,10681,2245,11483,1639,457,15918,7779,3447,11940,14491,16661,5322,9363,8098,2405,1669,7700,10855,2414,6564,17050,12220,1209,10156,1617,10666,3278,12420,1859,9304,8851,6083,2629,16014,16172,14380,11600,13226,12714,1779,13294,7998,11672,1560,14423,12737,16863,6525,10949,4905,9942,9555,7045,14577,10531,3727,2806,9822,10094,13431,5892,15906,10925,11155,16763,16850,7999,14442,15107,13693,6763,8998,3920,4174,1025,17243,14812,13084,2440,2037,13510,15381,16449,12732,5497,15204,13115,4921,12011,14783,11546,8472,218,554,13535,11357,8492,2016,6939,15062,6944,6975,12145,12831,4737,11011,8425,534,15455,12691,9463,14136,12710,12558,5750,13364,2339,13949,16015,13321,4830,9349,16146,8581,17528,1839,10204,16584,7378,6080,2169,15323,882,2171,8717,634,1337,7077,2163,9625,12012,16946,15250,5360,856,13613,17381,14178,9416,12475,1243,11416,3101,4048,10987,12794,10984,7906,5859,7271,5514,11102,5201,1342,15390,8951,13185,4920,9248,15997,3968,10189,5661,6527,14912,9035,3583,10300,16711,2598,16715,15134,6483,16742,7603,4025,6477,16082,709,7,9165,5942,2088,7373,1300,6460,740,5123,2075,1972,11269,14594,6119,4241,4711,2923,6846,11469,17510,2649,12922,11986,15213,14628,14967,13489,7441,12951,17491,6508,1956,5952,11488,9073,15171,9940,6065,4240,5782,10131,17663,9729,14034,3365,3000,14747,4072,10103,4153,1573,10100,15592,5447,17648,4785,3733,1397,10486,12202,1217,1392,8032,345,6726,12237,359,3849,1180,2951,3899,1445,5400,15853,1196,11687,3885,9444,477,3335,10534,1502,9780,12589,3257,4533,11708,4887,11347,7657,16318,1045,4389,10150,17745,8383,9697,1372,2987,14109,10453,3945,2758,16437,9282,16462,12135,1021,16468,10455,1233,8885,1389,5231,10774,11403,1717,4600,15923,14810,7978,15470,268,2368,14951,16292,17626,8956,3403,4208,671,10593,16464,14150,8224,11472,6557,16260,7677,8147,1070,7288,3427,1886,4811,13708,14888,5972,5016,13859,187,14697,7909,509,9302,17179,11525,2164,14898,9492,9447,8126,11879,2106,13339,11747,13160,9258,5626,5797,5789,14571,14964,9519,7210,3670,13673,9861,14145,15912,6620,16520,4746,4269,8953,16117,2899,6074,12834,12061,15991,16615,2737,4234,4230,17579,10680,12509,12719,3783,5471,9709,4821,4077,15865,5992,1113,1503,12268,17766,1488,14354,768,13877,3625,12972,16688,11944,8617,2606,17451,6499,11919,11460,3971,1318,12965,11375,3328,3402,2177,10553,11166,11162,4791,3550,11621,17171,14714,3200,4340,15878,14387,12516,16177,1333,14340,14240,8422,6100,9610,3129,12115,14550,14361,10154,1005,5634,17693,15959,3633,9102,5570,7803,12430,12413,15318,13691,8074,4558,9341,10945,13547,1945,1506,5430,13041,12119,8087,1587,13800,17227,17690,4935,15796,8558,11355,5165,1274,8295,11144,15363,12065,8485,16993,3242,16822,12028,5858,11900,12154,17102,2279,3145,12228,1567,2810,10065,22,5043,13926,214,14749,10634,4557,13269,16025,11511,11168,17231,8149,12660,9725,12699,13863,1480,14117,2014,13640,17345,11354,5633,5551,8043,12356,13551,14107,10683,13918,8888,16070,4478,1122,1524,16281,5289,11547,12904,4422,16654,13012,4554,5588,16173,4712]]
    
    elif feature_comb[0] == 'All':
        X = train_df

    #Top200
    else:
        X = train_df.iloc[:,[1400,15905,6836,4387,14160,3674,15120,2442,7860,1986,16,503,3392,6018,5366,15599,1294,16412,11877,636,12564,5023,6237,9097,1668,15060,3066,10345,16728,14200,4126,12955,14412,7604,15078,8220,12958,9131,10677,7688,9164,1518,1410,12051,3304,10589,17543,13633,189,1305,16209,16959,8006,9540,11882,926,12277,3287,15082,10286,1089,4206,13340,17582,7231,6315,11315,497,3318,1911,1387,7697,11363,14560,11539,17706,5594,14648,8804,11558,8476,6934,9466,2191,6720,6722,6096,16185,7297,8498,10489,11652,17586,9069,2734,16953,14815,7016,14323,755,1466,1804,4797,9077,16063,17536,11242,10959,12073,15699,11843,3944,12675,16299,7040,16907,15304,7091,11057,8507,16466,13655,6940,10512,16086,13361,3893,11639,9436,1563,3757,14061,9769,8065,12725,10359,16613,13955,50,9431,15897,8894,2656,157,9454,1453,2058,7268,12311,2936,13527,1642,9139,15713,16917,1440,15129,4663,13738,4337,10498,4648,14818,11084,960,11586,5407,1904,10381,7797,12043,750,7587,16302,3668,1519,11990,3924,14874,10125,16864,11259,14036,11905,14238,53,16087,14614]]

    return X

def get_data(feature_comb, sampling = None, feature_set = 'all'):
    '''
    Get the training data.
    '''
    
    train_df = pd.read_table(data_path+'/data/x.train.txt')

    y = pd.read_table(data_path+'/data/y.train.txt')
    y = y['Class'].values

    X = feature_combinations(train_df, feature_comb)

    print X.shape

    X = X.values

    #X = preprocessing.scale(X)

    #Normalize
    X = preprocessing.normalize(X, norm='l2')
    
    labels = list(train_df.columns.values)
   
    imp = Imputer(missing_values='NaN', strategy='mean', axis=0)
    imp.fit(X)
    imp.transform(X)

    return X, y

def create_dir(dir_path):

    if not os.path.exists(dir_path):
        try:
            os.makedirs(dir_path)
        except:
            sys.exit( "Output directory (%s) could not be created." % dir_path )
    return dir_path

def rfe(X,y):
    """
    
    """
    print y[1,]

    # Create the RFE object and compute a cross-validated score.
    svc = SVC(kernel="linear")
    # The "accuracy" scoring is proportional to the number of correct
    # classifications
    rfecv = RFECV(estimator=svc, step=1, cv=StratifiedKFold(2),
                  scoring='accuracy')
    rfecv.fit(X, y)

    print("Optimal number of features : %d" % rfecv.n_features_)

    # Plot number of features VS. cross-validation scores
    plt.figure()
    plt.xlabel("Number of features selected")
    plt.ylabel("Cross validation score (nb of correct classifications)")
    plt.plot(range(1, len(rfecv.grid_scores_) + 1), rfecv.grid_scores_)
    plt.show()

#get model     
def get_model(model, options):
    '''
    Get the model from six stat-of-the-art machine learning models.
    '''
    if options.depth:
       max_depth = int(options.depth)
    else:
        max_depth = None

    if model=='svm':
        from sklearn.svm import SVC
        names = ["Linear SVM"]
        classifiers = [
        SVC(kernel='linear', probability=True)    
        ]
    elif model=='ab':
        from sklearn.ensemble import AdaBoostClassifier
        names = ["AdaBoost"]
        classifiers = [
        AdaBoostClassifier() 
        ]
    elif model=='knn':
        from sklearn.neighbors import KNeighborsClassifier
        names = ["K-Nearest Neighbors"]
        classifiers = [
        KNeighborsClassifier(4)
        ]
    elif model=='dt':
        from sklearn.tree import DecisionTreeClassifier
        names = ["Decision Tree"]
        classifiers = [
        DecisionTreeClassifier(max_depth=3)   
        ]
    elif model=='nb':
        from sklearn.naive_bayes import GaussianNB
        names = ["Naive Bayes"]
        classifiers = [
         GaussianNB()   
        ]
    
    elif model == 'all':
        
        from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier
        from sklearn.naive_bayes import GaussianNB
        from sklearn.neighbors import KNeighborsClassifier
        from sklearn.tree import DecisionTreeClassifier
        from sklearn.svm import SVC
        from sklearn.linear_model import LogisticRegression, Lasso
    
        
        names = ["Random Forest", "Linear SVM", "AdaBoost", "K-Nearest Neighbors", "Naive Bayes", "Decision Tree", "Logistic Regression"]
        classifiers = [
        RandomForestClassifier(max_depth=max_depth,n_estimators=int(options.estimator)),
        SVC(kernel='linear', probability=True),  
        AdaBoostClassifier(),
        KNeighborsClassifier(4),
        GaussianNB(),
        DecisionTreeClassifier(max_depth=3),
        LogisticRegression()]
    else:
        from sklearn.ensemble import RandomForestClassifier
        names = ["Random Forest"]
        classifiers = [
        RandomForestClassifier(max_depth=max_depth,n_estimators=int(options.estimator))
        ]
    
    return names, classifiers

# Classification and ROC analysis
def claf_roc(X,y, Clabel, classifier, count, feature_comb, options, test_set=None):
    '''
    Train the model and test it using CV and draw the ROC plot.
    '''
    #X, y = X[y != 2], y[y != 2]
    #n_samples, n_features = X.shape
    
    # Run classifier with cross-validation and plot ROC curves
    cv = StratifiedKFold(y, n_folds=options.cv)
    mean_precision = []
    mean_recall = []
    mean_prc = []
    mean_tpr = 0.0
    mean_fpr = np.linspace(0, 1, 100)
    #all_tpr = []
   
    if options.test:
        test_df = pd.read_csv(test_set)
        
        #perform down sampling
        test_df = downsample(test_df)

        imp = Imputer(missing_values='NaN', strategy='mean', axis=0)
        test_X = test_df[feature_comb].values

        imp.fit(test_X)
        imp.transform(test_X)

        test_yy = test_df["Class"].values

        test_y = np.zeros(len(test_X))
        test_y[test_yy == 'SE'] = 1
        test_y[test_yy == 'TE'] = 0

        
        print "Testing the model using feature set "+Clabel
        probas_ = classifier.fit(X, y).predict_proba(test_X)
        #clr.predict_proba(X_test)[:, 1]
 
       # Compute ROC curve and area the curve
        #print report1
        fpr, tpr, thresholds = roc_curve(test_y, probas_[:, 1])
        mean_tpr += interp(mean_fpr, fpr, tpr)
        mean_tpr[0] = 0.0
        #roc_auc = auc(fpr, tpr)
        # Compute Precision-Recall and plot curve
        precision, recall, thresh = precision_recall_curve(test_y, probas_[:, 1])
        mean_precision.append(np.mean(precision))
        mean_recall.append(np.mean(recall))
        area = auc(recall, precision)
        mean_prc.append(area)

        #mean_tpr /= len(cv)
        mean_tpr[-1] = 1.0
        mean_auc = auc(mean_fpr, mean_tpr)
        pl.plot([0, 1], [0, 1], '--', color=(0.6, 0.6, 0.6), lw=0.5) #good luck line

    else:

        for i, (train, test) in enumerate(cv):
            
            print "Fold-"+str(i+1)+" for feature set "+Clabel
            probas_ = classifier.fit(X[train], y[train]).predict_proba(X[test])
            #clr.predict_proba(X_test)[:, 1]
     
           # Compute ROC curve and area the curve
            #print report1
            fpr, tpr, thresholds = roc_curve(y[test], probas_[:, 1])
            mean_tpr += interp(mean_fpr, fpr, tpr)
            mean_tpr[0] = 0.0
            #roc_auc = auc(fpr, tpr)
            # Compute Precision-Recall and plot curve
            precision, recall, thresh = precision_recall_curve(y[test], probas_[:, 1])
            mean_precision.append(np.mean(precision))
            mean_recall.append(np.mean(recall))
            area = auc(recall, precision)
            mean_prc.append(area)
            #pl.plot(fpr, tpr, lw=2, label= Clabel +' = %0.2f' % roc_auc)

        mean_tpr /= len(cv)
        mean_tpr[-1] = 1.0
        mean_auc = auc(mean_fpr, mean_tpr)
        pl.plot([0, 1], [0, 1], '--', color=(0.6, 0.6, 0.6), lw=0.5) #good luck line

    #if type[Clabel] is list:
    #if not isinstance(Clabel, str):
    #    Clabel = str(Clabel)
    
    if count > 7:
        pl.plot(mean_fpr, mean_tpr, '--', label= Clabel +' (%0.2f)' % mean_auc, lw=1.0)
    else:
        pl.plot(mean_fpr, mean_tpr, label= Clabel +' (%0.2f)' % mean_auc, lw=1.0)
       
    return mean_auc, np.mean(mean_recall), np.mean(mean_precision), np.mean(mean_prc)


def model_compare(X,y, names,classifiers,output_dir,options):
    X, y = X[y != 2], y[y != 2]
    #n_samples, n_features = X.shape 
    # Divide the data into folds

    file_name = str(output_dir)+"/survivalPredictor_model_compare_results.txt"
    file_out_stat = open(file_name,'w')
    out_string =  "These results are generated using survivalPredictor version 1.0 on "+str(datetime.datetime.now())+"\nRead more about survivalPredictor at https://github.com/asntech/survivalPredictor\n"
    file_out_stat.write(out_string)

    cv = StratifiedKFold(y, n_folds=int(options.cv))
    
    out_string =  "\n\nComparing models with "+str(options.cv)+"-fold cross-validation\n"
    file_out_stat.write(out_string)

    out_string = "\t".join(['Model', 'Precision',"Recall", "F1-score", "AUC", "PRC"]) + "\n"
    file_out_stat.write(out_string)
    # iterate over classifiers
    for name, clf in zip(names, classifiers):
        print "Running classifier "+name 
        mean_tpr = 0.0
        mean_precision = []
        mean_recall = []
        mean_pr = []
        mean_fpr = np.linspace(0, 1, 100)
        #all_tpr = []
        
        # Run each classifier with 10-fold stratified cross-validation   
        for i, (train, test) in enumerate(cv):
            #print "------> Cross validation for classifier "+name 
            probas_ = clf.fit(X[train], y[train]).predict_proba(X[test])
        
           # Compute ROC curve and area the curve
            fpr, tpr, thresholds = roc_curve(y[test], probas_[:, 1])
            
            # Compute Precision-Recall and plot curve
            precision, recall, thresh = precision_recall_curve(y[test], probas_[:, 1])
            mean_precision.append(np.mean(precision))
            mean_recall.append(np.mean(recall))
            area = auc(recall, precision)
            mean_pr.append(area)
            
            mean_tpr += interp(mean_fpr, fpr, tpr)
            mean_tpr[0] = 0.0
            #roc_auc = auc(fpr, tpr)
                   
            #pl.plot(fpr, tpr, lw=2, label= Clabel +' = %0.2f' % roc_auc)
        
        #print ((np.mean(mean_precision)*np.mean(mean_recall))/((np.mean(mean_precision)+np.mean(mean_recall))
              
        # Plot the decision boundaries
       
        #pl.plot([1, 0], [0, 1], '--', color=(0.6, 0.6, 0.6), lw=0.5)
        #pl.plot(recall, precision, label= name +' = %0.2f' % area, lw=1.5)
        
        mean_tpr /= len(cv)
        mean_tpr[-1] = 1.0
        mean_auc = auc(mean_fpr, mean_tpr)
        
        pl.plot([0, 1], [0, 1], '--', color=(0.6, 0.6, 0.6), lw=0.5)
        pl.plot(mean_fpr, mean_tpr, label= name +' (%0.2f)' % mean_auc, lw=1.5)

        f1s = 2*((np.mean(mean_precision)*np.mean(mean_recall)) / (np.mean(mean_precision)+np.mean(mean_recall)))
        
        #print("AUC Curve : %0.2f" % mean_auc)
        out_string = "\t".join([str(name),str(float(np.round(np.mean(mean_precision),2))), str(float(np.round(np.mean(mean_recall),2))), str(float(np.round(f1s,2))), str(float(np.round(mean_auc,2))),  str(float(np.round(np.mean(mean_pr),2)))]) + "\n"
        file_out_stat.write(out_string)
               
    #'ROC for all features with 10-fold CV'
    pl.xlim([0.0, 1.0])
    pl.ylim([0.0, 1.0])
    pl.xlabel('False Positive Rate (1-Specificity)')
    pl.ylabel('True Positive Rate (Sensitivity)')
    pl.title("Model comparision")
    pl.legend(fancybox=True, frameon=False, loc="lower right")
    #pl.legend(frameon=False, loc="lower right")

    #pl.xlabel('Recall')
    #pl.ylabel('Precision')
    #pl.title('Precision-Recall Curve')
    #pl.legend(fancybox=True, fontsize='medium', loc="lower left")

    #pl.show()
    #pl.savefig('fig/'+clf_name+'.eps', format='eps', dpi=1000)
    pl.savefig(output_dir+'/model_comparision.pdf', format='pdf', dpi=300)
    #pl.savefig('fig/'+heading+'.png', format='png', dpi=300)


def run_test(feature_list,classifiers,names,output_dir, options, sampling, feature_compare=False):
    '''
    Run on the training data and test the models using 10-fold cross-validation or test set.
    '''
    file_name = str(output_dir)+"/survivalPredictor_results.txt"
    file_out_stat = open(file_name,'w')
    out_string =  "These results are generated using survivalPredictor version 1.0 on "+str(datetime.datetime.now())+"\nRead more about survivalPredictor  at https://github.com/asntech/survivalPredictor\n"
    file_out_stat.write(out_string)
    
    for name, clf in zip(names, classifiers):

        clf_name = name
        if options.test:
            print "\nTraining "+clf_name
        else:
            print "\nRunning "+clf_name+" with "+str(options.cv)+"-fold cross-validation..."
        roc_score = []
        recall = []
        precision = []
        pr_curve = []
        f1_score = []
        
        count = 1
        if options.test:
            out_string =  "\n\n"+clf_name+" with validation using test data "+options.input+"\n"
        else:
            out_string =  "\n\n"+clf_name+" with "+str(options.cv)+"-fold cross-validation\n"
        file_out_stat.write(out_string)

        out_string = "\t".join(['Features', 'Precision',"Recall", "F1-score", "AUC", "PRC"]) + "\n"
        file_out_stat.write(out_string)

        if feature_compare and options.test:
            #if list contains another list
            if any(isinstance(el, list) for el in feature_list):
                feature_list = sum(feature_list, [])        
            #features_lable = '+'.join(feature_list)

            X, y = get_data(feature_list, sampling, options.feature)

            test_sets = options.input.split(',')

            for test_set in test_sets:

                features_lable = str(str(test_set).split('.')[0])

                roc, rc, pr, prc = claf_roc(X,y, features_lable, clf, count, feature_list, options, test_set)                
                roc_score.append(np.round(roc,2))
                recall.append(np.round(rc,2))
                precision.append(np.round(pr,2))
                pr_curve.append(np.round(prc,2))
                
                f1s = 2*((pr*rc) / (pr+rc))
                f1_score.append(np.round(f1s,2))
                #pl.grid(linestyle='--', lw=0.04)
               #draw ROC CURVE
                out_string = "\t".join([str(features_lable),str(float(np.round(pr,2))), str(float(np.round(rc,2))), str(float(np.round(f1s,2))), str(float(np.round(roc,2))),  str(float(np.round(prc,2)))]) + "\n"
                file_out_stat.write(out_string)
                count = count+1
        
        elif feature_compare:
            for feature_comb in feature_list:
                X, y = get_data(feature_comb, sampling, options.feature)
                #X = recursive_feature_selection(X,y)
                 #get roc, precision, recall....
                features_lable = '+'.join(feature_comb)

                roc, rc, pr, prc = claf_roc(X,y, features_lable, clf, count, feature_comb, options)
                
                roc_score.append(np.round(roc,2))
                recall.append(np.round(rc,2))
                precision.append(np.round(pr,2))
                pr_curve.append(np.round(prc,2))
                
                f1s = 2*((pr*rc) / (pr+rc))
                f1_score.append(np.round(f1s,2))
                #pl.grid(linestyle='--', lw=0.04)
               #draw ROC CURVE
                out_string = "\t".join([str(features_lable),str(float(np.round(pr,2))), str(float(np.round(rc,2))), str(float(np.round(f1s,2))), str(float(np.round(roc,2))),  str(float(np.round(prc,2)))]) + "\n"
                file_out_stat.write(out_string)
                count = count+1
        
        else:
            
            #if list contains another list
            if any(isinstance(el, list) for el in feature_list):
                feature_list = sum(feature_list, [])
        
            features_lable = '+'.join(feature_list)
            #features_lable = "Features"

            X, y = get_data(feature_list, sampling, options.feature)
            
            #get roc, precision, recall....
            roc, rc, pr, prc = claf_roc(X,y, features_lable, clf, count,feature_list, options)
            
            roc_score.append(np.round(roc,2))
            recall.append(np.round(rc,2))
            precision.append(np.round(pr,2))
            pr_curve.append(np.round(prc,2))
            
            f1s = 2*((pr*rc) / (pr+rc))
            f1_score.append(np.round(f1s,3))
            #pl.grid(linestyle='--', lw=0.04)
            #draw ROC CURVE
            out_string = "\t".join([features_lable,str(float(np.round(pr,2))), str(float(np.round(rc,2))), str(float(np.round(f1s,2))), str(float(np.round(roc,2))),  str(float(np.round(prc,2)))]) + "\n"
            file_out_stat.write(out_string)
            count = count+1

        pl.xlim([0.0, 1.0])
        pl.ylim([0.0, 1.0])
        pl.xlabel('False Positive Rate (1-Specificity)')
        pl.ylabel('True Positive Rate (Sensitivity)')
        
        #pl.title('ROC - Receiver Operating Characteristic')
        pl.title("ROC - "+clf_name)
        pl.legend(fancybox=True,frameon=False, fontsize='medium', loc="lower right",)

        #Save figure
        if options.figuretype:
            pl.savefig(str(output_dir)+"/"+clf_name+'.'+options.figuretype, format=options.figuretype, dpi=options.dpi)

        else:
            pl.savefig(str(output_dir)+"/"+clf_name+'.pdf', format='pdf', dpi=300)
        pl.close()


def make_prediction(X, y, feature_list, input_file, names, classifiers, output_dir, prob=0.5):
    '''
    Make prediction on the test data
    '''

    if not input_file:
        test_df = pd.read_table(data_path+'/data/test_data.txt')
    else:
        test_df = pd.read_table(input_file)

    test_df = feature_combinations(test_df, feature_list)

    labels = list(test_df.columns.values)


    test_X = test_df.values

    print (test_X.shape)
    
    imp = Imputer(missing_values='NaN', strategy='mean', axis=0)
    
    imp.fit(test_X)
    imp.transform(test_X)

    test_X = preprocessing.scale(test_X)

    labels.append("Probability")
    labels.append("Class")
    
    test_df["Probability"] = 0
    test_df["Class"] = 0
    
    #test_df["Probability"] = '0'

    for name, clf in zip(names, classifiers):
        file_name = output_dir+"/survivalPredictor_"+name+"_predictions.txt"
        file_ob = open(file_name,'w')
        
        print "Training and predicting usind "+name+"..."
        
        clf = clf.fit(X,y)

        pred_prob = clf.predict_proba(test_X)
        file_ob.write("Probability\n")
        file_ob.write(str(pd.Series(pred_prob[:,1])))

        #print predictions
        test_df["Probability"] = pd.Series(pred_prob[:,1])
    
        pred_class = np.zeros(len(test_X))
        pred_class[pred_prob[:,1] >= prob ] = 1
        pred_class[pred_prob[:,1] < prob ] = 0

        #print(pd.Series(pred_class))

        file_ob.write("Class\n")
        file_ob.write(str(pd.Series(pred_class)))

        test_df["Class"] = pd.Series(pred_class)

        file_ob.close()

        test_df.to_csv(output_dir+"/survivalPredictor_"+name+"_predictions.csv", columns=labels, index=False)

def main():
    '''
    This is main function for survivalPredictor
    '''
    usage = """
    survivalPredictor [options]

    To run example with default parameters type:

        %s --demo

    Read more about survivalPredictor https://github.com/asntech/survivalPredictor
    """ % sys.argv[0]
    parser = OptionParser(usage = usage)    
    #The following parameters are required to run the program. 
    parser.add_option("--demo", dest="demo", action='store_true',
                    help = "Run Demo using Random Forest 5-fold cross-validation. \nYou can set --model all, if you want to run the test on all six models. Note: This will take few minutes.")
    parser.add_option("-m","--model", dest="model", nargs = 1, type=str, default="rf",
                    help = "Select model. [rf, svm, knn, ab, dt, bn, all] (default=rf)")

    parser.add_option("-f","--feature", dest="feature", nargs = 1,  type=str, default=None,
                    help = "Select features of features type to train the model and make predictions. Feature names should be comma separated. \nIf you are want to check the combinatorial predictive power of features, separate them using + sypbol.\n[H3K27ac,Brd4 or Chromatin,TFs etc ]. (Default=all)")
    
    parser.add_option("--compare", dest="compare", action='store_true',
                    help = "Compare the listed features in one ROC plot.")

    parser.add_option("--mcomp", dest="model_compare", action='store_true',
                    help = "Compare the model in one ROC plot.")
    
    parser.add_option("-i","--input", dest="input", nargs = 1, type=str, default=None,
                    help = "Input file to use as test data or to make predictions. Please provide a CSV file with computed features and a 'Class' label if its test data.")

    parser.add_option("-t","--test", dest="test", action='store_true',
                    help = "Set if input file is a test file.")

    parser.add_option("-p","--pred", dest="pred", action='store_true',
                    help = "Set if input file is for prediction.")

    parser.add_option("--prob", dest="probability", nargs=1, type=float, default=0.5,
                    help = "Set the probability cutoff (default=0.5).")

    parser.add_option("-c","--cv", nargs = 1, type=int, default=5,
                    help = "Set Cross-validation folds. (default=5)")

    parser.add_option("-o","--output",  nargs = 1, type=str, default=None,
                    help = "Enter output folder path. By default results will be saved in the current working directory.")    
      
    parser.add_option("--figuretype", dest="figuretype", nargs = 1, type=str, default="pdf",
                    help = "Figure type [pdf, eps, jpg, png] (default=pdf)")

    parser.add_option("-e","--estimator", nargs = 1, type=int, default=20,
                    help = "The number of trees in the Random Forest. (default=20)")

    parser.add_option("-d","--depth", nargs = 1, type=int,
                    help = "The maximum depth of the Random Forest tree. (default=None)")

    parser.add_option("--dpi", dest="dpi", nargs = 1, type=int, default=300,
                    help = "DPI(Dots per inch) of figure. (default=300)")

    parser.add_option("-v","--version", dest="version", action='store_true',
                    help = "Print version and exit.")

    #parser.add_argument('--version', action='version', version='%(prog)s 1.0')

    #checking the parameters
    (options,args) = parser.parse_args()

    if options.version:
        print "survivalPredictor version "+__version__ 
        exit()

    if not options.pred and not options.test and not options.demo and not options.model_compare:
        parser.print_help()
        sys.exit(1)

    #making the out folder if it doesn't exist
    if options.output:
        output_dir = create_dir(options.output)
    else:
        output_dir = create_dir(os.getcwd()+"/survivalPredictor_results")

    #Original or SMOTE or Undersampling
    sampling = " "

    if options.compare:
        feature_compare = True
    else:
        feature_compare = False

    feature_list = []
    if options.feature == None:
        feature_list = [["Top 10"],["Top 50"],["Top 100"],["Top 200"],["Top 500"],["Top 1000"], ["Top 5000"], ["All"]]
        #feature_list = ["top10","top100","top500","top1000","top5000","all"]

        
    else:
        features = options.feature.split(',')
        #feature_list = list(feature_list)
        for feature in features:
            #if len(feature.split("+")) > 1:
                #print feature
                #exit()
            feature_list.append(feature.split("+"))
            #else:
             #   feature_list.append(feature)
        print feature_list
        #exit()
   
    #Get model
    names, classifiers = get_model(options.model, options)

    if options.model_compare:
        feature_list = sum(feature_list, [])
        print feature_list
        X, y = get_data(feature_list, sampling, options.feature)
        model_compare(X,y, names,classifiers,output_dir,options)

    elif options.demo:

        run_test(feature_list,classifiers,names,output_dir, options, sampling, feature_compare)

    elif options.input:
        print('Preaparing the data...')
        
        feature_list = sum(feature_list, [])
        X, y = get_data(feature_list, sampling, options.feature)

        if os.path.exists(options.input):

            if options.pred:
                print('\nTraining the model and making predictions')
                make_prediction(X, y, feature_list, options.input , names, classifiers, output_dir, options.probability);
            elif options.test:
                print('\nTraining the model and testing it using provided data')
                run_test(feature_list,classifiers,names,output_dir, options, sampling, feature_compare)
            else:
                print('\nYou forgot to mention weather the input file is to use as test or to make prediction.')
                parser.print_help()
                exit()
        else:
            print("\nThe input file does not exit. Please check it again.")
            exit()
    else:
        print('\nPlease make sure you provided all the required parameters')
        parser.print_help()
        exit()

    print '\nYou are done! Please check your results @ '+output_dir+'. \nThank you for using survivalPredictor!\n'

if __name__ == '__main__':
    try:
        main()
    except KeyboardInterrupt:
        sys.stderr.write("I got interrupted. :-( Bye!\n")
        sys.exit(0)
