param_search_cluster.py

import os

import numpy as np
import pandas as pd
from sklearn.cluster import DBSCAN, AgglomerativeClustering, AffinityPropagation
from sklearn.cluster import KMeans, Birch
from sklearn.decomposition import NMF, IncrementalPCA, PCA
from sklearn.preprocessing import StandardScaler

from cluster_performance_evaluate import evaluate_cluster_performance
from prepare_dataset import load_nn_stage2_features, load_stage2_tf_idf, load_clustering_statics_files


def train_cluster(data_type=0, dimension_reduction=0, cluster_way=0, n_components=50, threshold=2, n_clusters=210,
                  branching_factor=50, linkage=0, max_iter=500, eps=1.0):
    if data_type == 0:
        train_data = load_stage2_tf_idf("")
    elif data_type == 1:
        train_data = load_stage2_tf_idf("")
        nn_data = load_nn_stage2_features()
        train_data = pd.merge(train_data, nn_data, 'left', on="file_name")
    elif data_type == 2:
        train_data = load_nn_stage2_features()
    elif data_type == 3:
        train_data = load_stage2_tf_idf("1000")
        nn_data = load_nn_stage2_features()
        train_data = pd.merge(train_data, nn_data, 'left', on="file_name")
        dll = load_stage2_tf_idf("_dll")
        train_data = pd.merge(train_data, dll, 'left', on="file_name")
        dll = load_stage2_tf_idf("_hkey", "first")
        train_data = pd.merge(train_data, dll, 'left', on="file_name")
        dll = load_stage2_tf_idf("_hkey", "last")
        train_data = pd.merge(train_data, dll, 'left', on="file_name")
        train_data.fillna(0, inplace=True)
    elif data_type == 4:
        train_data = load_stage2_tf_idf("1000")
        nn_data = load_nn_stage2_features()
        train_data = pd.merge(train_data, nn_data, 'left', on="file_name")
        dll = load_stage2_tf_idf("_dll")
        train_data = pd.merge(train_data, dll, 'left', on="file_name")
        dll = load_stage2_tf_idf("_hkey", "first")
        train_data = pd.merge(train_data, dll, 'left', on="file_name")
        dll = load_stage2_tf_idf("_hkey", "last")
        train_data = pd.merge(train_data, dll, 'left', on="file_name")
        dll = load_clustering_statics_files()
        train_data = pd.merge(train_data, dll, 'left', on="file_name")
        train_data.fillna(0, inplace=True)

    file_name = train_data["file_name"]
    train_data.drop(columns=["file_name"], inplace=True)
    X = StandardScaler(with_mean=False).fit_transform(train_data)
    origin_data = X

    if dimension_reduction == 0:
        pass
    elif dimension_reduction == 1:
        model = IncrementalPCA(n_components=n_components)
        X = model.fit_transform(X)
    elif dimension_reduction == 2:
        model = NMF(n_components=n_components, init='random', random_state=0, max_iter=max_iter)
        X = model.fit_transform(X)
    elif dimension_reduction == 3:
        model = PCA(n_components=n_components)
        X = model.fit_transform(X)

    print(len(X[0]))
    if cluster_way == 0:
        mode = ["ward", "complete", "average", "single"]
        db = AgglomerativeClustering(n_clusters=n_clusters, linkage=mode[linkage]).fit(X)
        labels = db.labels_
        pd.DataFrame(data={"id": file_name, "family_id": db.labels_}).to_csv(
            os.path.join("predictions", "aggcl" + "_" + str(n_clusters) + "_" + str(data_type) + "_" + str(
                dimension_reduction) + "_" + str(n_components) + ".csv"), index=False)
        print(len(set(labels)))
    elif cluster_way == 1:
        db = Birch(branching_factor=branching_factor, n_clusters=n_clusters, threshold=threshold).fit(X)
        labels = db.predict(X)
        pd.DataFrame(data={"id": file_name, "family_id": db.labels_}).to_csv(
            os.path.join("predictions", "birch" + ".csv"),
            index=False)
        print(len(set(labels)))
    elif cluster_way == 2:
        db = hdbscan.HDBSCAN(min_cluster_size=40)
        db.fit(X)
        labels = db.labels_
        pd.DataFrame(data={"id": file_name, "family_id": db.labels_}).to_csv(
            os.path.join("predictions", "hdb_40" + ".csv"),
            index=False)
        print(len(set(labels)))
    elif cluster_way == 3:
        db = DBSCAN(eps=eps, n_jobs=-1).fit(X)
        labels = db.labels_
        pd.DataFrame(data={"id": file_name, "family_id": db.labels_}).to_csv(
            os.path.join("predictions", "db" + "_" + str(eps) + "_" + str(dimension_reduction) + ".csv"),
            index=False)
        print(len(set(labels)))
    elif cluster_way == 4:
        labels = np.zeros((len(file_name),))
        pd.DataFrame(data={"id": file_name, "family_id": np.zeros((len(file_name),))}).to_csv(
            os.path.join("predictions", "zeros" + ".csv"),
            index=False)
    elif cluster_way == 5:
        db = KMeans(n_clusters=n_clusters, random_state=0).fit(X)
        labels = db.labels_
        pd.DataFrame(data={"id": file_name, "family_id": db.labels_}).to_csv(
            os.path.join("predictions", "kmeans" + str(n_clusters) + ".csv"),
            index=False)
        print(len(set(labels)))
    elif cluster_way == 6:
        db = AffinityPropagation()

    # Number of clusters in labels, ignoring noise if present.
    n_clusters_ = len(set(labels)) - (1 if -1 in labels else 0)
    n_noise_ = list(labels).count(-1)

    print('Estimated number of clusters: %d' % n_clusters_)
    print('Estimated number of noise points: %d' % n_noise_)

    scores = evaluate_cluster_performance(origin_data, labels)
    evaluate_cluster_performance(X, labels)
    return scores


def connect_params(params):
    full = {}
    for i in params:
        full.update(i)
    return full