【数据挖掘】《数据分析与数据挖掘》--天津大学公开课

    数据处理大致可以分成两大类：联机事务处理OLTP（on-line transaction processing）、联机分析处理OLAP（On-Line Analytical Processing）。OLTP是传统的关系型数据库的主要应用，主要是基本的、日常的事务处理，例如银行交易。OLAP是数据仓库系统的主要应用，支持复杂的分析操作，侧重决策支持，并且提供直观易懂的查询结果。

    // An highlighted block
    """
    # Python 2.7
    # Filename: apriori.py
    # Author: llhthinker
    # Email: hangliu56[AT]gmail[DOT]com
    # Blog: http://www.cnblogs.com/llhthinker/p/6719779.html
    # Date: 2017-04-16
    """
    
    
    def load_data_set():
    """
    Load a sample data set (From Data Mining: Concepts and Techniques, 3th Edition)
    Returns: 
        A data set: A list of transactions. Each transaction contains several items.
    """
    data_set = [['l1', 'l2', 'l5'], ['l2', 'l4'], ['l2', 'l3'],
            ['l1', 'l2', 'l4'], ['l1', 'l3'], ['l2', 'l3'],
            ['l1', 'l3'], ['l1', 'l2', 'l3', 'l5'], ['l1', 'l2', 'l3']]
    return data_set
    
    
    def create_C1(data_set):
    """
    Create frequent candidate 1-itemset C1 by scaning data set.
    Args:
        data_set: A list of transactions. Each transaction contains several items.
    Returns:
        C1: A set which contains all frequent candidate 1-itemsets
    """
    C1 = set()
    for t in data_set:
        for item in t:
            item_set = frozenset([item])
            C1.add(item_set)
    return C1
    
    
    def is_apriori(Ck_item, Lksub1):
    """
    Judge whether a frequent candidate k-itemset satisfy Apriori property.
    Args:
        Ck_item: a frequent candidate k-itemset in Ck which contains all frequent
                 candidate k-itemsets.
        Lksub1: Lk-1, a set which contains all frequent candidate (k-1)-itemsets.
    Returns:
        True: satisfying Apriori property.
        False: Not satisfying Apriori property.
    """
    for item in Ck_item:
        sub_Ck = Ck_item - frozenset([item])
        if sub_Ck not in Lksub1:
            return False
    return True
    
    
    def create_Ck(Lksub1, k):
    """
    Create Ck, a set which contains all all frequent candidate k-itemsets
    by Lk-1's own connection operation.
    Args:
        Lksub1: Lk-1, a set which contains all frequent candidate (k-1)-itemsets.
        k: the item number of a frequent itemset.
    Return:
        Ck: a set which contains all all frequent candidate k-itemsets.
    """
    Ck = set()
    len_Lksub1 = len(Lksub1)
    list_Lksub1 = list(Lksub1)
    for i in range(len_Lksub1):
        for j in range(1, len_Lksub1):
            l1 = list(list_Lksub1[i])
            l2 = list(list_Lksub1[j])
            l1.sort()
            l2.sort()
            if l1[0:k-2] == l2[0:k-2]:
                Ck_item = list_Lksub1[i] | list_Lksub1[j]
                # pruning
                if is_apriori(Ck_item, Lksub1):
                    Ck.add(Ck_item)
    return Ck
    
    
    def generate_Lk_by_Ck(data_set, Ck, min_support, support_data):
    """
    Generate Lk by executing a delete policy from Ck.
    Args:
        data_set: A list of transactions. Each transaction contains several items.
        Ck: A set which contains all all frequent candidate k-itemsets.
        min_support: The minimum support.
        support_data: A dictionary. The key is frequent itemset and the value is support.
    Returns:
        Lk: A set which contains all all frequent k-itemsets.
    """
    Lk = set()
    item_count = {}
    for t in data_set:
        for item in Ck:
            if item.issubset(t):
                if item not in item_count:
                    item_count[item] = 1
                else:
                    item_count[item] += 1
    t_num = float(len(data_set))
    for item in item_count:
        if (item_count[item] / t_num) >= min_support:
            Lk.add(item)
            support_data[item] = item_count[item] / t_num
    return Lk
    
    
    def generate_L(data_set, k, min_support):
    """
    Generate all frequent itemsets.
    Args:
        data_set: A list of transactions. Each transaction contains several items.
        k: Maximum number of items for all frequent itemsets.
        min_support: The minimum support.
    Returns:
        L: The list of Lk.
        support_data: A dictionary. The key is frequent itemset and the value is support.
    """
    support_data = {}
    C1 = create_C1(data_set)
    L1 = generate_Lk_by_Ck(data_set, C1, min_support, support_data)
    Lksub1 = L1.copy()
    L = []
    L.append(Lksub1)
    for i in range(2, k+1):
        Ci = create_Ck(Lksub1, i)
        Li = generate_Lk_by_Ck(data_set, Ci, min_support, support_data)
        Lksub1 = Li.copy()
        L.append(Lksub1)
    return L, support_data
    
    
    def generate_big_rules(L, support_data, min_conf):
    """
    Generate big rules from frequent itemsets.
    Args:
        L: The list of Lk.
        support_data: A dictionary. The key is frequent itemset and the value is support.
        min_conf: Minimal confidence.
    Returns:
        big_rule_list: A list which contains all big rules. Each big rule is represented
                       as a 3-tuple.
    """
    big_rule_list = []
    sub_set_list = []
    for i in range(0, len(L)):
        for freq_set in L[i]:
            for sub_set in sub_set_list:
                if sub_set.issubset(freq_set):
                    conf = support_data[freq_set] / support_data[freq_set - sub_set]
                    big_rule = (freq_set - sub_set, sub_set, conf)
                    if conf >= min_conf and big_rule not in big_rule_list:
                        # print freq_set-sub_set, " => ", sub_set, "conf: ", conf
                        big_rule_list.append(big_rule)
            sub_set_list.append(freq_set)
    return big_rule_list
    
    
    if __name__ == "__main__":
    """
    Test
    """
    data_set = load_data_set()
    L, support_data = generate_L(data_set, k=3, min_support=0.2)
    big_rules_list = generate_big_rules(L, support_data, min_conf=0.7)
    for Lk in L:
        print "="*50
        print "frequent " + str(len(list(Lk)[0])) + "-itemsets\t\tsupport"
        print "="*50
        for freq_set in Lk:
            print freq_set, support_data[freq_set]
    print
    print "Big Rules"
    for item in big_rules_list:
        print item[0], "=>", item[1], "conf: ", item[2]

    // An highlighted block
    class treeNode:
    def __init__(self, nameValue, numOccur, parentNode):
        self.name = nameValue #节点名字
        self.count = numOccur #节点计数值
        self.nodeLink = None #用于链接相似的元素项
        self.parent = parentNode      #needs to be updated
        self.children = {} #子节点
    
    def inc(self, numOccur):
        '''
        对count变量增加给定值
        '''
        self.count += numOccur
    
    def disp(self, ind=1):
        '''
        将树以文本形式展示
        '''
        print ('  '*ind, self.name, ' ', self.count)
        for child in self.children.values():
            child.disp(ind+1)

    // An highlighted block
    def createTree(dataSet, minSup=1):
    '''
    创建FP树
    '''
    headerTable = {}
    #第一次扫描数据集
    for trans in dataSet:#计算item出现频数
        for item in trans:
            headerTable[item] = headerTable.get(item, 0) + dataSet[trans]
    headerTable = {k:v for k,v in headerTable.items() if v >= minSup}
    freqItemSet = set(headerTable.keys())
    #print ('freqItemSet: ',freqItemSet)
    if len(freqItemSet) == 0: return None, None  #如果没有元素项满足要求，则退出
    for k in headerTable:
        headerTable[k] = [headerTable[k], None] #初始化headerTable
    #print ('headerTable: ',headerTable)
    #第二次扫描数据集
    retTree = treeNode('Null Set', 1, None) #创建树
    for tranSet, count in dataSet.items():  
        localD = {}
        for item in tranSet:  #put transaction items in order
            if item in freqItemSet:
                localD[item] = headerTable[item][0]
        if len(localD) > 0:
            orderedItems = [v[0] for v in sorted(localD.items(), key=lambda p: p[1], reverse=True)]
            updateTree(orderedItems, retTree, headerTable, count)#将排序后的item集合填充的树中
    return retTree, headerTable #返回树型结构和头指针表
    
    def updateTree(items, inTree, headerTable, count):
    if items[0] in inTree.children:#检查第一个元素项是否作为子节点存在
        inTree.children[items[0]].inc(count) #存在，更新计数
    else:   #不存在，创建一个新的treeNode,将其作为一个新的子节点加入其中
        inTree.children[items[0]] = treeNode(items[0], count, inTree)
        if headerTable[items[0]][1] == None: #更新头指针表
            headerTable[items[0]][1] = inTree.children[items[0]]
        else:
            updateHeader(headerTable[items[0]][1], inTree.children[items[0]])
    if len(items) > 1:#不断迭代调用自身，每次调用都会删掉列表中的第一个元素
        updateTree(items[1::], inTree.children[items[0]], headerTable, count)
    
    def updateHeader(nodeToTest, targetNode):
    '''
    this version does not use recursion
    Do not use recursion to traverse a linked list!
    更新头指针表，确保节点链接指向树中该元素项的每一个实例
    '''
    while (nodeToTest.nodeLink != None):    
        nodeToTest = nodeToTest.nodeLink
    nodeToTest.nodeLink = targetNode

    // An highlighted block
    def ascendTree(leafNode, prefixPath): #迭代上溯整棵树
    if leafNode.parent != None:
        prefixPath.append(leafNode.name)
        ascendTree(leafNode.parent, prefixPath)
    
    def findPrefixPath(basePat, treeNode): #treeNode comes from header table
    condPats = {}
    while treeNode != None:
        prefixPath = []
        ascendTree(treeNode, prefixPath)
        if len(prefixPath) > 1: 
            condPats[frozenset(prefixPath[1:])] = treeNode.count
        treeNode = treeNode.nodeLink
    return condPats

    // An highlighted block
    def mineTree(inTree, headerTable, minSup, preFix, freqItemList):
    bigL = [v[0] for v in sorted(headerTable.items(), key=lambda p: p[1][0])]# 1.排序头指针表
    for basePat in bigL:  #从头指针表的底端开始
        newFreqSet = preFix.copy()
        newFreqSet.add(basePat)
        print ('finalFrequent Item: ',newFreqSet)    #添加的频繁项列表
        freqItemList.append(newFreqSet)
        condPattBases = findPrefixPath(basePat, headerTable[basePat][1])
        print ('condPattBases :',basePat, condPattBases)
        # 2.从条件模式基创建条件FP树
        myCondTree, myHead = createTree(condPattBases, minSup)
    #         print ('head from conditional tree: ', myHead)
        if myHead != None: # 3.挖掘条件FP树
            print ('conditional tree for: ',newFreqSet)
            myCondTree.disp(1)            
            mineTree(myCondTree, myHead, minSup, newFreqSet, freqItemList)

    // An highlighted block
    # D = {(x1,y1)、(x2,y2)......(xm,yn)} 是数据集
    # A = {a1、a2、a3.} 是划分节点的属性集
    # 节点node有两个主要属性：content代表该节点需要分类的训练集，type代表叶节点的决策类型
    def generateTree(D,A):
    newNode = 空 #生成新的节点
    # 如果当前数据集都为一个种类，则设为一个叶节点并返回
    if D 中数据皆属于类别 C:
        newNode.content = D
        newNode.type = C
        return  
    # 如果已经没有属性了或者数据集在剩余属性中表现相同（属性无法区分）
    if A = 空集 or D中数据在A中取值相同:
        newNode.content = D
        newNode.type = D中最多的类
        return
    #从A中选取最优的属性a
    a=selectBestPorperty(A)
    #为a的每一个取值生成一个节点，递归进行处理
    for a的每一个取值 res[i]:
        生成新的分支节点 node[i]
        D[i] = D中取值为res[i]的数据
        node[i].content = D[i]
        if node[i].content == null:
            node[i].type = D中最多的类
        else:
            generateTree(D[i],A - {a})
    return

    // An highlighted block
    import  numpy as np
    import matplotlib.pyplot as plt
    
    '''
    AGNES层次聚类，采用自底向上聚合策略的算法。先将数据集的每个样本看做一个初始的聚类簇，然后算法运行的每一步中找出距离最近的两个
    类簇进行合并，该过程不断重复，直至达到预设的聚类簇的个数。
    '''
    #计算两个向量之间的欧式距离
    def calDist(X1 , X2 ):
    sum = 0
    for x1 , x2 in zip(X1 , X2):
        sum += (x1 - x2) *
    return sum ** 0.5
    def updateClusterDis(dataset,distance,sets,cluster_i):
    i=0
    while i<len(sets):
        dis = []
        for e in sets[i]:
            for ele in sets[cluster_i]:
                dis.append(calDist(dataset[e],dataset[ele]))
        distance[i,cluster_i]=max(dis)
        distance[cluster_i,i]=max(dis)
        i+=1
    #将每个簇和自身距离设为无穷大
    distance[np.diag_indices_from(distance)] = float('inf')
    return distance
    def agens(dataset,k):
    #初始化聚类簇:让每一个点都代表，一个类簇
    sets=[]
    for i in range(0,len(dataset)):
        sets.append({i})
    #初始化类簇间距离的矩阵
    delta = np.array(dataset[0] - dataset)
    for e in dataset[1:, :]:
        delta = np.vstack((delta, (e - dataset)))
    distance = np.sqrt(np.sum(np.square(delta), axis=1))
    distance = np.reshape(distance, (len(dataset), len(dataset)))
    distance[np.diag_indices_from(distance)]=float('inf')
    ####################################################
    while len(sets)>k:
        locations=np.argwhere(distance==np.min(distance))
        #将集合合并，删除被合并的集合
        locations=locations[locations[:,0]<locations[:,1]]
        cluster_i=locations[0,0]
        cluster_j=locations[0,1]
        for e in sets[cluster_j]:
            sets[cluster_i].add(e)
        del sets[cluster_j]
        #删除被合并的簇，distance矩阵对应的行和列，并更新距离矩阵
        distance=np.delete(distance,cluster_j,axis=0)#删除对应列
        distance=np.delete(distance,cluster_j,axis=1)#删除对应行
        distance=updateClusterDis(dataset,distance,sets,cluster_i)
    print(sets)
    return sets