투빅스 11기&12기 5주차 PCA - 12기 김태한

by 12기김태한 posted Aug 27, 2019

단축키

Prev이전 문서

Next다음 문서

ESC닫기

import numpy as npimport numpy.linalg as lin
import matplotlib.pyplot as plt
import pandas as pd
from sklearn.datasets import fetch_mldata
from scipy import io
%matplotlib inline
from mpl_toolkits.mplot3d import Axes3D
mnist = io.loadmat('mnist-original.mat') 
X = mnist['data'].T
y = mnist['label'].T
# data information
​
# 7만개의 작은 숫자 이미지
# 행 열이 반대로 되어있음 -> 전치
# grayscale 28x28 pixel = 784 feature
# 각 picel은 0~255의 값
# label = 1~10 label이 총 10개인거에 주목하자
# data를 각 픽셀에 이름붙여 표현
feat_cols = [ 'pixel'+str(i) for i in range(X.shape[1]) ]
df = pd.DataFrame(X,columns=feat_cols)
df.head()
pixel0    pixel1    pixel2    pixel3    pixel4    pixel5    pixel6    pixel7    pixel8    pixel9    ...    pixel774    pixel775    pixel776    pixel777    pixel778    pixel779    pixel780    pixel781    pixel782    pixel783
0    0    0    0    0    0    0    0    0    0    0    ...    0    0    0    0    0    0    0    0    0    0
1    0    0    0    0    0    0    0    0    0    0    ...    0    0    0    0    0    0    0    0    0    0
2    0    0    0    0    0    0    0    0    0    0    ...    0    0    0    0    0    0    0    0    0    0
3    0    0    0    0    0    0    0    0    0    0    ...    0    0    0    0    0    0    0    0    0    0
4    0    0    0    0    0    0    0    0    0    0    ...    0    0    0    0    0    0    0    0    0    0
5 rows × 784 columns
 
# df에 라벨 y를 붙여서 데이터프레임 생성
df['y'] = y
# 자 먼저 mnist data들을 train 8 test 2 의 비율로 분리를 하자
from sklearn.model_selection import train_test_split
train, test = train_test_split(df, test_size=0.2)
# 잘 분리가 된 것을 확인할 수 있다
print(train.shape)
print(test.shape)
(56000, 785)
(14000, 785)
### 이제 한번 원본 data, PCA data, LDA data를 구해보고 비교를 해보자
from sklearn.decomposition import PCA
​
pca = PCA(n_components=3)
pca_result = pca.fit_transform(df[feat_cols].values)
df['pca-one'] = pca_result[:,0]
df['pca-two'] = pca_result[:,1] 
df['pca-three'] = pca_result[:,2]
print('Explained variation per principal component: {}'.format(pca.explained_variance_ratio_))
Explained variation per principal component: [0.09746116 0.07155445 0.06149531]
fig = plt.figure(figsize = (10,10))
ax = fig.add_subplot(1,1,1) 
ax.set_xlabel('Principal Component 1', fontsize = 15)
ax.set_ylabel('Principal Component 2', fontsize = 15)
ax.set_title('First and Second Principal Components colored by digit', fontsize = 20)
​
for i in range(10):
    ax.scatter(df['pca-one'][df['y']==i]
               , df['pca-two'][df['y']==i]
               , s = 10)
​
ax.legend(range(10))
ax.grid()
Colored by Color Scripter
cs

# 가시적인 효과는 위의 scatter plot으로 확인 할 수 있으나 적당히 몇차원에서 stop해야 하는지 모르므로 다음의 룰들을 따르기로 했다# 1. elbow point
# 2. kaiser's rule
# 3. 누적 설명률 80~90%
# pca는 설명변수들의 분포에 따른 축소이므로 target변수인 y값을 지워주도록 한다
train_1 = train.drop(['y'], axis=1)
train_label = pd.DataFrame(train['y'])
# 먼저 2번의 규칙을 기준으로 eigenvalue의 값이 1이 되는 근처 지점을 찾아보자
# 손글씨 데이터의 픽셀값들이므로 구석진 곳같이 언제나 0이 되는 지점을 제외한 픽셀들은 대부분 의미를 갖는 거 같다
# 차원을 784에서 650정도로 줄여야 eigenvalue가 1을 기준으로 왔다갔다 하는 것을 찾았다.
for i in range(645,650):
    pca = PCA(n_components=i)
    pca.fit(train_1)
    print(pca.explained_variance_.min())
1.3211235492011624
1.211493311905986
1.18122238115841
1.1609173552217211
0.9383812933863515
# 그럼 1번의 방법을 적용하기 위해 약 좌우로 25개의 범주에서 plot을 한번 해보자
x_list = range(625,675)
y_list = list()
for i in x_list:
    pca = PCA(n_components=i)
    pca.fit(train_1)
    y_list.append(pca.explained_variance_.min())
plt.plot(x_list,y_list)
plt.xlabel('n_components')
plt.ylabel('eigenvalue')
Text(0, 0.5, 'eigenvalue')
Colored by Color Scripter
cs

# elbow인지는 잘 모르겠지만 그래도 약 647~8정도에서 급락이후 다시 완만한 굴곡으로 내려간다 생각되었다# 지금은 elbow로 안보이지만 만약 저부분만 확대한다면 elbow로 봐도 되지 않을까
# 그래서 n_components를 648로 잡았다.
# 3번째 룰을 적용해보려 하였는데 거의 누적설명률이 100프로라고 봐도 무방해 보인다
pca = PCA(n_components=648)
pca.fit(train_1)
pca.explained_variance_ratio_.sum()
0.9999946547076656
train_1 = pca.transform(train_1)
### LDA를 해보자 ###
# 개인적인 생각이지만 LDA란 label의 개수-1 차원(베르누이 분포라면 단 1차원밖에 안된다)으로 차원을 축소시켜주는데
# 각 data들의 label을 반영하여 축소된 차원에서 data들의 label간 군집의 mean값의 거리가 최대가되고 label간 군집내의 편차는 최소가 되도록
# 분류를 한다.
# 여기서 pca하기전의 원본data를 사용한다면 데이터분포의 큰 영향을 주지 않는 축이 포함되고 그러면 라벨의 분포에 따른 해당 축의 영향력이 적어
# 오히려 차원축소후 투영시 bias 혹은 error로 작용할 것 같다
# 그래서 pca로 줄인 차원에서 lda로 축소를 하기로 생각하였다
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
​
lda = LinearDiscriminantAnalysis(n_components=2)
lda_result = lda.fit_transform(df[feat_cols].values, df['y'])
/home/kimtaehan/anaconda3/lib/python3.7/site-packages/sklearn/discriminant_analysis.py:388: UserWarning: Variables are collinear.
  warnings.warn("Variables are collinear.")
fig = plt.figure(figsize = (10,10))
ax = fig.add_subplot(1,1,1) 
ax.set_xlabel('LDA 1', fontsize = 15)
ax.set_ylabel('LDA 2', fontsize = 15)
ax.set_title('First and Second LDA colored by digit', fontsize = 20)
​
for i in range(10):
    ax.scatter(lda_result[df['y']==i,0]
            ,lda_result[df['y']==i,1]
            , s = 10)
​
ax.legend(range(10))
ax.grid()
Colored by Color Scripter
cs

# pca이전 데이터를 가지고 lda를 적용하여 2차원으로 줄여 plot한 그래프이다# 이것으로는 잘 모르겠다
# label개수 만큼의 축 즉 dimension을 잡아주어야 각 label의 구분 속성이 다 반영될 것이라 생각하여 max인 10으로 잡았다
lda = LinearDiscriminantAnalysis(n_components=10)
lda_result = lda.fit(train_1, train_label)
/home/kimtaehan/anaconda3/lib/python3.7/site-packages/sklearn/utils/validation.py:761: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().
  y = column_or_1d(y, warn=True)
train_after_lda = lda.transform(train_1)
# 648의 제곱근은 약 25이므로 깊이를 25정도로 잡아보자
from sklearn.ensemble import RandomForestClassifier
RF = RandomForestClassifier(n_estimators=3000, max_depth=25,random_state=1, criterion='gini')
import datetime as pydatetime
time_old = pydatetime.datetime.now().timestamp()
RF.fit(train_after_lda, train_label)
time_new = pydatetime.datetime.now().timestamp()
print(time_new-time_old) # second로 출력 약 11분가량 걸린다
/home/kimtaehan/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:3: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples,), for example using ravel().
  This is separate from the ipykernel package so we can avoid doing imports until
650.3023781776428
test_1 = pd.DataFrame(test.drop(['y'],axis=1))
test_1 = pca.transform(test_1)
test_after_lda = lda.transform(test_1)
test_label = pd.DataFrame(test['y'])
RF.score(test_after_lda, test_label)
# 2000 n_estimator accuracy 0.9128
# 3000 n_estimator accuracy 0.913357
0.9133571428571429
### 이제 그럼 original data를 가지고 RF를 해보자
from sklearn.ensemble import RandomForestClassifier
RF_origin = RandomForestClassifier(n_estimators=3000, max_depth=25,random_state=0, criterion='gini')
import datetime as pydatetime
from sklearn.ensemble import RandomForestClassifier
time_old = pydatetime.datetime.now().timestamp()
RF_origin.fit(train.drop(['y'], axis=1), pd.DataFrame(train['y']))
time_new = pydatetime.datetime.now().timestamp()
/home/kimtaehan/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:4: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples,), for example using ravel().
  after removing the cwd from sys.path.
print(time_new-time_old) # origin은 약 4분이 걸린다 시간이 짧은 것은 n_estimator를 3000에서 2000, max_depth를 25에서 5로 줄여서 그렇다
249.38196897506714
RF_origin.score(test.drop(['y'], axis=1), pd.DataFrame(test['y'])) # 그래도 그냥 한거보다는 성능이 좋아서 다행이다 ㅎㅎ
0.865
### 2번째 방법 knn을 사용해보자
# knn의 경우 distance를 바탕으로하는 instance_based model로 현재 각 feature 즉 독립변수의 value들이 0-255로 딱히 scaling을
# 하지 않아도 규격이 정해져 distance로 구분하기가 어쩌면 좋은 data set이지 않을까 하는 생각을 하였다
from sklearn.neighbors import KNeighborsClassifier
neigh = KNeighborsClassifier(n_neighbors=10, algorithm='auto')
time_old = pydatetime.datetime.now().timestamp()
neigh.fit(train_after_lda, train_label)
time_new = pydatetime.datetime.now().timestamp()
print(time_new-time_old) # 0.05초 걸린다 ㄷㄷ
0.057225942611694336
/home/kimtaehan/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:4: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().
  after removing the cwd from sys.path.
test_1 = pd.DataFrame(test.drop(['y'],axis=1))
test_1 = pca.transform(test_1)
test_after_lda = lda.transform(test_1)
test_label = pd.DataFrame(test['y'])
neigh.score(test_after_lda, test_label) # accuracy 약 91.6퍼센트 나온다
0.9162857142857143
# 그냥 원래 데이터로 한번 해보자
from sklearn.neighbors import KNeighborsClassifier
neigh = KNeighborsClassifier(n_neighbors=10, algorithm='auto')
time_old = pydatetime.datetime.now().timestamp()
neigh.fit(train.drop(['y'], axis=1), pd.DataFrame(train['y']))
time_new = pydatetime.datetime.now().timestamp()
print(time_new-time_old) # 20초 걸린다 꽤나 길다
/home/kimtaehan/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:5: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().
  """
20.481126070022583
neigh.score(test.drop(['y'], axis=1), pd.DataFrame(test['y']))
0.9707142857142858
# 그냥 knn한것은 약 25분~30분이 걸렸고 정확도가 97퍼센트나 나온것을 볼 수 있다
# 결론
# pca lda는 너무 많은 독립변수들을 차원을 낮혀 차원의 저주 혹은 고차원으로 인한 기하급수적으로 늘어나는 필요 데이터 량을 줄여주는데 큰
# 역할을 하는 방법이다
# mnist data set에서는 이미지 픽셀마다 값이 0-255 정해진 value들이 독립변수로 선택이 된다
# 이떄 random forest같은 경우는 bagging의 한 종류로써 boostrapping을 하게되는데 여기서 독립변수 종류를 선택할 때 label에 관계없이
# 언제나 0인 data set들만 선택이 된다던가 이들로 인해서 각 base model 즉 decision tree에서 이들이 섞여들어가면 gini계수 판별로
# 영향력 없는 data종류는 선택이 되지 않아 하지않아도 되는 process를 하게되므로 영향력이 있는 feature들만 뽑아 이를 특정 depth의 깊이로
# 어느정도 깊은 model이 나오도록하여 boosting으로 variance가 높은 DT들을 학습하도록 하면 더 좋은 성능을 발휘한다고 생각한다
# 반면 knn의 경우 distance를 기반으로 하는데 현재 mnist의 data set은 상당히 distance를 구하기 좋은 set으로 나와있다고 생각한다
# 이를 pca와 lda를 거침으로 인해 각 feature의 scale까지 완벽한 data들이 차원축소가 일어나면서 새로운 차원에서 feature의 분포가 변화가
# 생기고 이로인해 distance가 영향을 받아 더 안좋은 결과를 내놓았다고 생각한다
# 이를 종합해보면 instance_based classifier보다 model_based classifier에 pca lda를 적용하는 것이 적합하다고 생각이 든다.
​
Colored by Color Scripter
cs

« Prev 투빅스 11기&12기 5주차 SVM - 12기 이홍정

투빅스 11기&12기 5주차 SVM - 12기 이홍정 2019.08.27by 올타임넘버원메시 〈

투빅스 11기&12기 4주차 Ensemble - 12기 배유나 Next »

투빅스 11기&12기 4주차 Ensemble - 12기 배유나 2019.08.26by 배유나 〉

Trackback http://www.datamarket.kr/xe/board_BoGi29/61321/1e6/trackback

Articles

2 3 4 5 6 7 8 9 10 11

나눔글꼴 설치 안내

이 PC에는 나눔글꼴이 설치되어 있지 않습니다.

이 사이트를 나눔글꼴로 보기 위해서는
나눔글꼴을 설치해야 합니다.

✔ 설치 취소