Question

There are three documents and you need to return the top 3 important words in each document with their TF-IDF scores. Ignore all punctuations. Documents document1.dat, document2.dat, document3.dat

in python

i have to be able to run a program in python the opens 3 documents and outputs the 3 words that are used the most among the documents along with calculate their tf-idf scores

Answer 1

from collections import Counter
from heapq import nlargest
import math
#THIS FUNCTION RETURNS A DICTIONARY OF EACH FREQUENCY OF WORD
def wordfrequency(fname):
    l = [i.strip().split() for i in open(fname).readlines()]
    d=dict()
    for i in l:
        for j in i:
            if j not in d:
                d[j]=1
            else:
                d[j]+=1
    return d
#THIS FUNCTION RETURNS A LIST CONTAINING THE 3 LARGEST OCCURING WORDS
def threelargest(dct):
    res = nlargest(3, dct, key = dct.get)
    return res
#THIS FUNCTION RETURNS A TF-IDF OF A WORD
def calculate_tf_idf(l,dct,total_no_of_words,df):

    for i in dct:
        total_no_of_words+=dct[i]
    tf=dct[l]/total_no_of_words
    idf=math.log10(3.0/df)

    return (tf*idf)

d1=wordfrequency("f1.dat")
d2=wordfrequency("f2.dat")
d3=wordfrequency("f3.dat")
l1=threelargest(d1)
l2=threelargest(d2)
l3=threelargest(d3)
print("3 most occured word in file 1"+str(l1))
print("3 most occured word in file 2"+str(l2))
print("3 most occured word in file 3"+str(l3))
dt1={}
t1=0
for i in d1:
    t1+=d1[i]
for i in l1:
    df=0
    if i in d1:
        df+=1
    if i in d2:
        df+=1
    if i in d3:
        df+=1
    dt1[i]=calculate_tf_idf(i,d1,t1,df)
print("tfidf of 3 most occured words in file 1 "+str(dt1))
dt2={}
t2=0
for i in d2:
    t2+=d2[i]
for i in l2:
    df=0
    if i in d1:
        df+=1
    if i in d2:
        df+=1
    if i in d3:
        df+=1
    dt2[i]=calculate_tf_idf(i,d2,t2,df)
print("tfidf of 3 most occured words in file 2 "+str(dt2))
dt3={}
t3=0
for i in d3:
    t3+=d3[i]
for i in l3:
    df=0
    if i in d1:
        df+=1
    if i in d2:
        df+=1
    if i in d3:
        df+=1
    dt3[i]=calculate_tf_idf(i,d3,t3,df)
print("tfidf of 3 most occured words in file 3 "+str(dt3))
#d1,d2,d3 contains dictionary of words with frequency
#t1,t2,t3 contains total no of words in d1,d2,d3
#l1,l2,l3 has 3 mlargest occuring words in d1,d2,d3
#dt1,dt2,dt3 has tf-idf for l1,l2,,l3
#df contains the no of documents the word occur

There are dat files f1,f2,f3

OUTPUT

GIVE IT A LIKE IF YOU FIND IT USEFUL,THANKS!!!

C1. C:\WINDOWS\system32\cmd.exe C:\Users\Lenovo\Desktop>python u1.py 3 most occured word in file 1['-0.06', '0.12', '-0.99'] 3 most occured word in file 2['0.0', '1', '0.00'] 3 most occured word in file 3['333', '19', '336'] tfidf of 3 most occured words in file 1 {'-0.06': 0.003306780973304591, '0.12': 0.003306780973304591, '-0.99': 0.003306780973304591} tfidf of 3 most occured words in file 2 {'0.0': 0.011431030060991913, '1': 0.006958018297995077, '0.00': 0.0023845691330456834} tfidf of 3 most occured words in file 3 {'333': 0.0020972362844820327, '19': 0.0005805206342494986, '336': 0.0015729272133615244} C:\Users\Lenovo\Desktop>