贝叶斯的应用过滤垃圾邮件贝叶斯分类器的著名的应用就是垃圾邮件过滤了，这方面推荐想详细了解的可以去看看《黑客与画家》或是《数学之美》中对应的章节，贝叶斯的基础实现看这里数据集两个文件夹，分别是正常邮件和垃圾邮件，其中各有25封邮件测试方法从50封邮件中随机选取10封做为测试数据实现细节
1.首先我们需要将文本转成我们需要的向量的样子，这里需要使用一点正则表达式
2.由于采取交叉验证的方式，随机过程会导致每次的结果不尽相同
1#coding=utf-82fromnumpyimport*34#解析文档的函数5deftextParse(bigString):6importre7listOfTokens = re.split(r'\W*',bigString)8return[tok.lower()fortokinlistOfTokensiflen(tok) > 2]91011#创建一个带有所有单词的列表12defcreateVocabList(dataSet):13vocabSet =set([])14fordocumentindataSet:15vocabSet = vocabSet |set(document)16returnlist(vocabSet)1718defsetOfWords2Vec(vocabList, inputSet):19retVocabList = [0] *len(vocabList)20forwordininputSet:21ifwordinvocabList:22retVocabList[vocabList.index(word)] = 123else:24PRint'word',word ,'not in dict'25returnretVocabList2627#另一种模型28defbagOfWords2VecMN(vocabList, inputSet):29returnVec = [0]*len(vocabList)30forwordininputSet:31ifwordinvocabList:32returnVec[vocabList.index(word)] += 133returnreturnVec3435deftrainNB0(trainMatrix,trainCatergory):36numTrainDoc =len(trainMatrix)37numWords =len(trainMatrix[0])38pAbusive = sum(trainCatergory)/float(numTrainDoc)39#防止多个概率的成绩当中的一个为040p0Num =ones(numWords)41p1Num =ones(numWords)42p0Denom = 2.043p1Denom = 2.044foriinrange(numTrainDoc):45iftrainCatergory[i] == 1:46p1Num +=trainMatrix[i]47p1Denom +=sum(trainMatrix[i])48else:49p0Num +=trainMatrix[i]50p0Denom +=sum(trainMatrix[i])51p1Vect = log(p1Num/p1Denom)#处于精度的考虑，否则很可能到限归零52p0Vect = log(p0Num/p0Denom)53returnp0Vect,p1Vect,pAbusive5455defclassifyNB(vec2Classify, p0Vec, p1Vec, pClass1):56p1 = sum(vec2Classify * p1Vec) + log(pClass1)#element-wise mult57p0 = sum(vec2Classify * p0Vec) + log(1.0 -pClass1)58ifp1 >p0:59return160else:61return06263defspamTest(spamFloder, hamFloder):64docList =[]65classList =[]66fullText =[]67foriinrange(1,26):68wordList = textParse(open(spamFloder+str(i)+'.txt').read())69docList.append(wordList)70fullText.extend(wordList)71classList.append(1)72wordList = textParse(open(hamFloder+str(i)+'.txt').read())73docList.append(wordList)74fullText.extend(wordList)75classList.append(0)76vocabList =createVocabList(docList)77trainingSet = range(50)78testSet =[]79foriinrange(10):80randIndex =int(random.uniform(0,len(trainingSet)))81testSet.append(trainingSet[randIndex])82del(trainingSet[randIndex])83trainMat =[]84trianClasses =[]85printtrainingSet86fordocIndexintrainingSet:87trainMat.append(setOfWords2Vec(vocabList, docList[docIndex]))88#trainMat.append(bagOfWords2VecMN(vocabList, docList[docIndex]))89trianClasses.append(classList[docIndex])90p0V,p1V,pSpam =trainNB0(array(trainMat),array(trianClasses))91errorCount =092fordocIndexintestSet:#classify the remaining items93#wordVector = bagOfWords2VecMN(vocabList, docList[docIndex])94wordVector =setOfWords2Vec(vocabList, docList[docIndex])95ifclassifyNB(array(wordVector),p0V,p1V,pSpam) !=classList[docIndex]:96errorCount += 197print"classification error",docList[docIndex]98print'the error rate is:',float(errorCount)/len(testSet)99#return vocabList,fullText100101102defmain():103spamTest('email/spam/','email/ham/')104105if__name__=='__main__':106main()
从个人广告中获取地区倾向这个是从某个网站上提取了不同地区板块的信息，分析他们的用词是不是有某些规律数据集这里的数据使用rss获取的，用到了python的feedparse包，想了解可以看这里.这里分别获取了某网站两个地区板块中的信息测试方法交叉验证实现细节
1.这里有两种字符需要特别处理（其实他们有很大重合），一种是频率最高的一些，另一种是所谓的停用词（我的理解其实就是那些使用频率很高但没什么实际意义的），各种语言的停用词可以看这里。
我们需要移除这些词以使得结果更能体现出地区差异。
2.getTopWords函数实际上就是对这个概率统计了一下特征。对学习贝叶斯来说不是必要代码
3.除了数据来源不同实现细节和上面的很相似
1#coding=utf-82fromnumpyimport*34#解析文档的函数5deftextParse(bigString):6importre7listOfTokens = re.split(r'\W*',bigString)8return[tok.lower()fortokinlistOfTokensiflen(tok) > 2]91011#创建一个带有所有单词的列表12defcreateVocabList(dataSet):13vocabSet =set([])14fordocumentindataSet:15vocabSet = vocabSet |set(document)16returnlist(vocabSet)1718defsetOfWords2Vec(vocabList, inputSet):19retVocabList = [0] *len(vocabList)20forwordininputSet:21ifwordinvocabList:22retVocabList[vocabList.index(word)] = 123else:24print'word',word ,'not in dict'25returnretVocabList2627#另一种模型28defbagOfWords2VecMN(vocabList, inputSet):29returnVec = [0]*len(vocabList)30forwordininputSet:31ifwordinvocabList:32returnVec[vocabList.index(word)] += 133returnreturnVec3435deftrainNB0(trainMatrix,trainCatergory):36numTrainDoc =len(trainMatrix)37numWords =len(trainMatrix[0])38pAbusive = sum(trainCatergory)/float(numTrainDoc)39#防止多个概率的成绩当中的一个为040p0Num =ones(numWords)41p1Num =ones(numWords)42p0Denom = 2.043p1Denom = 2.044foriinrange(numTrainDoc):45iftrainCatergory[i] == 1:46p1Num +=trainMatrix[i]47p1Denom +=sum(trainMatrix[i])48else:49p0Num +=trainMatrix[i]50p0Denom +=sum(trainMatrix[i])51p1Vect = log(p1Num/p1Denom)#处于精度的考虑，否则很可能到限归零52p0Vect = log(p0Num/p0Denom)53returnp0Vect,p1Vect,pAbusive5455defclassifyNB(vec2Classify, p0Vec, p1Vec, pClass1):56p1 = sum(vec2Classify * p1Vec) + log(pClass1)#element-wise mult57p0 = sum(vec2Classify * p0Vec) + log(1.0 -pClass1)58ifp1 >p0:59return160else:61return06263defstopWords():64stopW =[]65f = open('stopwords.txt').readlines()66foreachLineinf:67stopW.append(eachLine[:-1])68returnstopW6970defcalcMostFreq(vocabList,fullText):71importOperator72freqDict ={}73fortokeninvocabList:74freqDict[token]=fullText.count(token)75sortedFreq = sorted(freqDict.iteritems(), key=operator.itemgetter(1), reverse=True)76returnsortedFreq[:30]7778deflocalWords(rss1,rss0):79importfeedparser80feed1 =feedparser.parse(rss1)81feed0 =feedparser.parse(rss0)82docList=[]; classList = []; fullText =[]83minLen = min(len(feed1['entries']),len(feed0['entries']))84foriinrange(minLen):85wordList = textParse(feed1['entries'][i]['summary'])86docList.append(wordList)87fullText.extend(wordList)88classList.append(1)#NY is class 189wordList = textParse(feed0['entries'][i]['summary'])90docList.append(wordList)91fullText.extend(wordList)92classList.append(0)93vocabList = createVocabList(docList)#create vocabulary94top30Words = calcMostFreq(vocabList,fullText)#remove top 30 words95forpairWintop30Words:96ifpairW[0]invocabList: vocabList.remove(pairW[0])97stopW =stopWords()98forpairWinstopW:99ifpairW[0]invocabList:100vocabList.remove(pairW[0])101trainingSet = range(2*minLen); testSet=[]#create test set102foriinrange(20):103randIndex =int(random.uniform(0,len(trainingSet)))104testSet.append(trainingSet[randIndex])105del(trainingSet[randIndex])106trainMat=[]; trainClasses =[]107fordocIndexintrainingSet:#train the classifier (get probs) trainNB0108trainMat.append(bagOfWords2VecMN(vocabList, docList[docIndex]))109trainClasses.append(classList[docIndex])110p0V,p1V,pSpam =trainNB0(array(trainMat),array(trainClasses))111errorCount =0112fordocIndexintestSet:#classify the remaining items113wordVector =bagOfWords2VecMN(vocabList, docList[docIndex])114ifclassifyNB(array(wordVector),p0V,p1V,pSpam) !=classList[docIndex]:115errorCount += 1116print'the error rate is:',float(errorCount)/len(testSet)117returnvocabList,p0V,p1V118119defgetTopWords(ny,sf):120importoperator121vocabList,p0V,p1V=localWords(ny,sf)122topNY=[]; topSF=[]123foriinrange(len(p0V)):124ifp0V[i] > -6.0: topSF.append((vocabList[i],p0V[i]))125ifp1V[i] > -6.0: topNY.append((vocabList[i],p1V[i]))126sortedSF = sorted(topSF, key=lambdapair: pair[1], reverse=True)127print"SF**SF**SF**SF**SF**SF**SF**SF**SF**SF**SF**SF**SF**SF**SF**SF**"128foriteminsortedSF:129printitem[0]130sortedNY = sorted(topNY, key=lambdapair: pair[1], reverse=True)131print"NY**NY**NY**NY**NY**NY**NY**NY**NY**NY**NY**NY**NY**NY**NY**NY**"132foriteminsortedNY:133printitem[0]134135defmain():136#print stopWords()137localWords('http://newyork.craigslist.org/stp/index.rss','http://sfbay.craigslist.org/stp/index.rss')138139if__name__=='__main__':140main()
机器学习笔记索引
来自为知笔记(Wiz)