import pydendroheatmap as pdh
from __future__ import print_function
#Sklearn
from sklearn.datasets import make_blobs
from sklearn.cluster import KMeans,DBSCAN
from sklearn.metrics import silhouette_samples, silhouette_score
from sklearn import metrics
from sklearn.decomposition import PCA
from sklearn.preprocessing import StandardScaler
import time
import warnings

import numpy as np
import matplotlib.pyplot as plt

from sklearn import cluster, datasets, mixture
from sklearn.neighbors import kneighbors_graph
from sklearn.preprocessing import StandardScaler
from itertools import cycle, islice
import sklearn
from sklearn.ensemble import RandomForestClassifier,RandomForestRegressor
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score
from scipy.spatial.distance import pdist, squareform
from scipy.cluster.hierarchy import linkage, dendrogram
from scipy.cluster.hierarchy import cophenet
from scipy.cluster.hierarchy import inconsistent
from scipy.cluster.hierarchy import fcluster
import scipy.spatial as sp, scipy.cluster.hierarchy as hc


#Plot
import seaborn as sns
import matplotlib.pyplot as plt
import matplotlib.cm as cm
import numpy as np
import pandas as pd
print(__doc__)
from IPython.display import HTML
path = '/media/cotran/New Volume/REU_data'
small = '/CNCAU_1403-1509_R_v1_03-03-2016/'
new_features = '/New features computing/'
folder = '/Computing_features/'
df_final = pd.read_csv(path+folder+'Courses_DataFrame.csv')
from IPython.display import Latex
#Configure Visualization Defaults
#%matplotlib inline = show plots in Jupyter Notebook browser
#Visualization
import matplotlib as mpl
import matplotlib.pylab as pylab
from pandas.tools.plotting import scatter_matrix
%matplotlib inline
plt.style.use('ggplot')
sns.set_style('white')
pylab.rcParams['figure.figsize'] = 20,12
import numpy as np
from scipy.cluster.hierarchy import linkage, _LINKAGE_METHODS
from scipy.spatial.distance import pdist
from fastcluster import linkage as linkage_fc

HTML('''<script>
code_show=true; 
function code_toggle() {
 if (code_show){
 $('div.input').hide();
 } else {
 $('div.input').show();
 }
 code_show = !code_show
} 
$( document ).ready(code_toggle);
</script>
<form action="javascript:code_toggle()"><input type="submit" value="Click here to toggle on/off the raw code."></form>''')

df_mm = pd.read_csv('/media/cotran/New Volume/REU_data/Computing_features/Courses_minmax.csv')
df_std = pd.read_csv('/media/cotran/New Volume/REU_data/Computing_features/Courses_Standard.csv')
df_mm['course_id'] = df_final['course_id']
df_std['course_id'] = df_final['course_id']
df_mm['discipline'] = df_mm['discipline']*10
df_std['discipline'] = df_std['discipline']*10

df.sample(5)

df.describe()

df = df_mm.drop(columns=['discipline','course_id']).copy()
Z = linkage(df, 'ward')

c, coph_dists = cophenet(Z, pdist(df))
print('cophenet score for the ward linkage : %f' %c)

for n in np.arange(0.5,3,0.1):
    max_d = n
    clusters = fcluster(Z, max_d, criterion='distance')
    silhouette_score = metrics.silhouette_score(df, clusters,
                                      metric='euclidean')
    c, coph_dists = cophenet(Z, pdist(df))
    print('max_distance = %f,number of clusters = %f, silhouette_score = %f' %
          (max_d,clusters.max(),silhouette_score))

max_d = 1.4
clusters = fcluster(Z, max_d, criterion='distance')
df_ward = df.copy()
df_ward['cluster'] = clusters
%matplotlib inline
mpl.style.use('ggplot')
sns.set_style('white')
pylab.rcParams['figure.figsize'] = 12,9
g = sns.countplot(df_ward['cluster'])
g.set_title('Number of courses per cluster using Hierarchical Clustering')
df_ward['cluster'].value_counts()

DF_corr = df.T.corr()
DF_dism = 1 - DF_corr   # distance matrix
linkage = hc.linkage(sp.distance.squareform(DF_dism), method='ward')
dis_heat_map = sns.clustermap(DF_dism, row_linkage=linkage, col_linkage=linkage)
dis_heat_map.fig.suptitle('Distance Similarity heat map using Hierarchical Clustering with Ward linkage method') 

df_ward = df_ward.sort_values(by=['cluster'])
DF_corr = df_ward.T.corr()
DF_dism = 1 - DF_corr 
ax = sns.heatmap(DF_dism)
ax.set_title('HAC with 1.4 cut off line')

for n in np.arange(3,20,1):
    estimator = KMeans(init='k-means++', n_clusters=n)
    estimator.fit(df)
    y_predict = estimator.predict(df)
    silhouette_score = metrics.silhouette_score(df, estimator.labels_,
                                      metric='euclidean')
    print('number of clusters = %f, silhouette_score = %f' %(n,silhouette_score))

estimator = KMeans(init='k-means++', n_clusters=7)
estimator.fit(df)
y_predict = estimator.predict(df)
df_k_mean = df.copy()
df_k_mean['cluster'] = y_predict
df_k_mean = df_k_mean.sort_values(by=['cluster'])
df_corr = df_k_mean.T.corr()
df_dism = 1 - df_corr   # distance matrix
ax = sns.heatmap(df_dism)
ax.set_title('K Means clustering with 7 clusters')

%matplotlib inline
mpl.style.use('ggplot')
sns.set_style('white')
pylab.rcParams['figure.figsize'] = 12,9
g = sns.countplot(df_k_mean['cluster'])
df_k_mean['cluster'].value_counts()

for m in range(3,6,1):
    for n in np.arange(0.1,0.5,0.05):
        min_samples = m
        estimator = DBSCAN(eps=n, min_samples=min_samples)
        y_predict = estimator.fit_predict(df)
        silhouette_score = metrics.silhouette_score(df, estimator.labels_,
                                          metric='euclidean')
        print('eps = %f, silhouette_score = %f, number of clusters = %d,min_samples = %d' 
              %(n,silhouette_score,len(np.unique(y_predict)),min_samples))

estimator = DBSCAN(eps=0.2, min_samples=3)
y_predict = estimator.fit_predict(df)
df_db = df.copy()
df_db['cluster'] = y_predict
df_db = df_db.sort_values(by=['cluster'])
df_db_corr = df_db.T.corr()
df_db_dism = 1 - df_db_corr   # distance matrix
sns.set(font="monospace")
ax = sns.heatmap(df_db_dism)
ax.set_title('DBSCAN with eps=0.2, min_samples=3 and 3 clusters')

%matplotlib inline
mpl.style.use('ggplot')
sns.set_style('white')
pylab.rcParams['figure.figsize'] = 12,9
g = sns.countplot(df_db['cluster'])
df_db['cluster'].value_counts()

estimator = DBSCAN(eps=0.15, min_samples=5)
y_predict = estimator.fit_predict(df)
df_db = df.copy()
df_db['cluster'] = y_predict
df_db = df_db.sort_values(by=['cluster'])
df_db_corr = df_db.T.corr()
df_db_dism = 1 - df_db_corr   # distance matrix
sns.set(font="monospace")
ax = sns.heatmap(df_db_dism)
ax.set_title('DBSCAN with eps=0.15, min_samples=5 and 4 clusters')

%matplotlib inline
mpl.style.use('ggplot')
sns.set_style('white')
pylab.rcParams['figure.figsize'] = 12,9
g = sns.countplot(df_db['cluster'])
df_db['cluster'].value_counts()
g.set_title('DBSCAN with eps=0.15, min_samples=5 and 4 clusters, Courses per cluster distribution')