import string
import math
class LanguageModel:
    def __init__ (self,filePath,gramNumber):
        self.filePath = filePath
        self.gramNumber = gramNumber
        self.state = 0
        self.V = 0
        self.hashCounts = []
        self.hashProbs = []
        self.totalCount = []
        self.stopList = ['\r','\n']
#        self.stopList = [';',',','.','(',')','-','%','/',':','?','!','[',']','\n','\t','\b',' ']
#        for i in range(10):
#            self.stopList.append(str(i))
            
        for i in range(gramNumber):
            self.hashCounts.append({})
            self.totalCount.append(0)
        
        for i in range(gramNumber):
            self.hashProbs.append({})
        
        
    def setFilePath(self,filePath):
        self.filePath = filePath
        self.state = 0
        
    def setGramNumber(self,gramNumber):
        self.gramNumber = gramNumber
        self.state = 0 
    
    def buildCountTables(self):
        for i in range(self.gramNumber):
            self.buildCount2(i+1)
        self.state = 1
    
    def buildCount(self,n):
        file = open(self.filePath,'r')
        while 1:
            line = file.readline()
#            line = line[0:len(line)-1]
            line = line.lower()
            if not line:
                break
            
            if len(line) < n:
                continue
            for i in range(len(line)-n+1):
                key = line[i:i+n];
                found = 0
                for pattern in self.stopList:
                    result = key.find(pattern)
                    if result > -1:
                        found = 1
                        break
                
                if found:
                    continue
                    
                if self.hashCounts[n-1].has_key(key):
                    self.hashCounts[n-1][key] = self.hashCounts[n-1][key]+1
                else:
                    self.hashCounts[n-1][key] = 1
                self.totalCount[n-1] +=1          
        file.close()
        if n==1:
            self.V = len(self.hashCounts[0].keys())
    
    
    
    def buildCount2(self,n):
        file = open(self.filePath,'r')
        totLine = ''
        while 1:
            line = file.readline()
            if not line:
                break
            line = line[0:len(line)-1]+' '
            line = line.lower()
            totLine +=line
        totLine = preProcessDoc(totLine)
        for i in range(len(totLine)-n+1):
            key = totLine[i:i+n];
            found = 0
            for pattern in self.stopList:
                result = key.find(pattern)
                if result > -1:
                    found = 1
                    break
            if found:
                continue
            if self.hashCounts[n-1].has_key(key):
                self.hashCounts[n-1][key] = self.hashCounts[n-1][key]+1
            else:
                self.hashCounts[n-1][key] = 1
            self.totalCount[n-1] +=1          
        file.close()
        if n==1:
            self.V = len(self.hashCounts[0].keys())
    
    
    def getPerplexityUnsmooth(self,filePath,n):
        if n > self.gramNumber:
            print "This Language Model just supports from unigram up to " +str(self.gramNumber)+" gram query!"
            return -1
        if self.state ==0:
            print "You should First Build Language Model then you can use it for evaluating you test set" 
            return -1
        file = open(filePath,'r')
        totLine = ''
        while 1:
            line = file.readline()
            if not line:
                break
            line = line.lower()
            line = line[0:len(line)-1]+' ' # deleting the newline character
            totLine += line
        
        totLine = preProcessDoc(totLine)
        invPerplex = 0
        totCount = 0    
        try:
            invPerplex += math.log(float(self.hashCounts[0][totLine[0]])/float(self.totalCount[0]))
            totCount += 1
        except:
            print "Character "+ totLine[0] +" is not in the Vocabulary!"
        for i in range(len(totLine)-1):
            idx = i+1
            minIdx = max(0,idx-n+1)
            w2 = totLine[minIdx:idx+1]
            w1 = totLine[minIdx:idx]
            try:
                C1 = 0
                C2 = 0
                if self.hashCounts[len(w2)-1].has_key(w2):
                    C2 = self.hashCounts[len(w2)-1][w2]
                if self.hashCounts[len(w1)-1].has_key(w1):
                    C1 = self.hashCounts[len(w1)-1][w1]
                if len(w1)==0:
                    # Handling the Uni-gram Case                      
                    C1 = self.totalCount[0]
                invPerplex += math.log(float(C2)/float(C1))
                totCount += 1
            except:
#                print "The "+ str(len(w2)) +"gram  \""+ w2 +"\"  is not in the training set and has zero count!"
                continue
                    
        perplex = -(1/float(totCount))*invPerplex
        return perplex
    
    
    def getPerplexityLaplace(self,filePath,n):
        if n > self.gramNumber:
            print "This Language Model just supports from unigram up to " +str(self.gramNumber)+" gram query!"
            return -1
        if self.state ==0:
            print "You should First Build Language Model then you can use it for evaluating you test set" 
            return -1
        
        file = open(filePath,'r')
        totLine = ''
        while 1:
            line = file.readline()
            if not line:
                break
            line = line.lower()
            line = line[0:len(line)-1]+' '
            
            totLine += line
        
        totLine = preProcessDoc(totLine)
        invPerplex = 0
        totCount = 0    
        try:
            p = LaplaceSmoothing(float(self.hashCounts[0][totLine[0]]),float(self.totalCount[0]),self.V)
            invPerplex += math.log(p)
            totCount += 1
        except:
            print "Character "+ totLine[0] +" is not in the Vocabulary!"
            
        for i in range(len(totLine)-1):
            idx = i+1
            minIdx = max(0,idx-n+1)
            w2 = totLine[minIdx:idx+1]
            w1 = totLine[minIdx:idx]
            try:
                C1 = 0
                C2 = 0
                if self.hashCounts[len(w2)-1].has_key(w2):
                    C2 = self.hashCounts[len(w2)-1][w2]
                if self.hashCounts[len(w1)-1].has_key(w1):
                    C1 = self.hashCounts[len(w1)-1][w1]
                if len(w1)==0:
                    # Handling the Uni-gram Case                      
                    C1 = self.totalCount[0]
                    
                p = LaplaceSmoothing(float(C2),float(C1),float(self.V))
                invPerplex += math.log(p)                
                totCount += 1
            except:
                print "The "+ str(len(w2)) +"gram  \""+ w2 +"\"  is not in the training set and has zero count!"
                continue
                    
        perplex = -(1/float(totCount))*invPerplex
        return perplex
        
    def getPerplexityBackoff(self,filePath,n):
        if n > self.gramNumber:
            print "This Language Model just supports from unigram up to " +str(self.gramNumber)+" gram query!"
            return -1
        if self.state ==0:
            print "You should First Build Language Model then you can use it for evaluating you test set" 
            return -1
        
        file = open(filePath,'r')
        totLine = ''
        while 1:
            line = file.readline()
            if not line:
                break
            line = line.lower()
            line = line[0:len(line)-1]+' '
            totLine += line
            
        totLine = preProcessDoc(totLine)
        invPerplex = 0
        totCount = 0    
        try:
            p = LaplaceSmoothing(float(self.hashCounts[0][totLine[0]]),float(self.totalCount[0]),self.V)
            invPerplex += math.log(p)
            totCount += 1
        except:
            print "Character "+ totLine[0] +" is not in the Vocabulary!"
            
        for i in range(len(totLine)-1):
            idx = i+1
            minIdx = max(0,idx-n+1)
            w2 = totLine[minIdx:idx+1]
            p = self.Backoff(w2)
            try:
                invPerplex += math.log(p)
                totCount += 1
            except:
                continue
                                
        perplex = -(1/float(totCount))*invPerplex
        return perplex
        
    
    def getPerplexityInterpolate(self,filePath,n):
        if n > self.gramNumber:
            print "This Language Model just supports from unigram up to " +str(self.gramNumber)+" gram query!"
            return -1
        if self.state ==0:
            print "You should First Build Language Model then you can use it for evaluating you test set" 
            return -1
        file = open(filePath,'r')
        totLine = ''
        while 1:
            line = file.readline()
            if not line:
                break
            line = line.lower()
            line = line[0:len(line)-1]+' '
            totLine += line
            
        totLine = preProcessDoc(totLine)
        invPerplex = 0
        totCount = 0    
        try:
            p = LaplaceSmoothing(float(self.hashCounts[0][totLine[0]]),float(self.totalCount[0]),self.V)
            invPerplex += math.log(p)
            totCount += 1
        except:
            print "Character "+ totLine[0] +" is not in the Vocabulary!"
            
        for i in range(len(totLine)-1):
            idx = i+1
            minIdx = max(0,idx-n+1)
            w2 = totLine[minIdx:idx+1]
            p = self.Interpolate(w2)
            try:
                invPerplex += math.log(p)
                totCount += 1
            except:
                continue
                                
        perplex = -(1/float(totCount))*invPerplex
        return perplex
    
    def getPerplexityInterpolateSmooth(self,filePath,n):
        if n > self.gramNumber:
            print "This Language Model just supports from unigram up to " +str(self.gramNumber)+" gram query!"
            return -1
        if self.state ==0:
            print "You should First Build Language Model then you can use it for evaluating you test set" 
            return -1
        file = open(filePath,'r')
        totLine = ''
        while 1:
            line = file.readline()
            if not line:
                break
            line = line.lower()
            line = line[0:len(line)-1]+' '
            totLine += line
            
        totLine = preProcessDoc(totLine)
        invPerplex = 0
        totCount = 0    
        try:
            p = LaplaceSmoothing(float(self.hashCounts[0][totLine[0]]),float(self.totalCount[0]),self.V)
            invPerplex += math.log(p)
            totCount += 1
        except:
            print "Character "+ totLine[0] +" is not in the Vocabulary!"
            
        for i in range(len(totLine)-1):
            idx = i+1
            minIdx = max(0,idx-n+1)
            w2 = totLine[minIdx:idx+1]
            p = self.InterpolateSmooth(w2)
            try:
                invPerplex += math.log(p)
                totCount += 1
            except:
                continue
                                
        perplex = -(1/float(totCount))*invPerplex
        return perplex
    
    
    def getPerplexityInterpolateFreq(self,filePath,n):
        if n > self.gramNumber:
            print "This Language Model just supports from unigram up to " +str(self.gramNumber)+" gram query!"
            return -1
        if self.state ==0:
            print "You should First Build Language Model then you can use it for evaluating you test set" 
            return -1
        file = open(filePath,'r')
        totLine = ''
        while 1:
            line = file.readline()
            if not line:
                break
            line = line.lower()
            line = line[0:len(line)-1]+' '
            totLine += line
            
        totLine = preProcessDoc(totLine)
        invPerplex = 0
        totCount = 0    
        try:
            p = LaplaceSmoothing(float(self.hashCounts[0][totLine[0]]),float(self.totalCount[0]),self.V)
            invPerplex += math.log(p)
            totCount += 1
        except:
            print "Character "+ totLine[0] +" is not in the Vocabulary!"
            
        for i in range(len(totLine)-1):
            idx = i+1
            minIdx = max(0,idx-n+1)
            w2 = totLine[minIdx:idx+1]
            p = self.InterpolateFreq(w2)
            try:
                invPerplex += math.log(p)
                totCount += 1
            except:
                continue
                                
        perplex = -(1/float(totCount))*invPerplex
        return perplex
    
    def Backoff(self,w2):
        n = len(w2)
        P = 0
        while w2 != '':
            w1 = w2[0:len(w2)-1]
            C1 = 0
            C2 = 0
            if self.hashCounts[len(w2)-1].has_key(w2):
                C2 = self.hashCounts[len(w2)-1][w2]
            if self.hashCounts[len(w1)-1].has_key(w1):
                C1 = self.hashCounts[len(w1)-1][w1]
                # Handling the Uni-gram Case
            if len(w1)==0:
                C1 = self.totalCount[0]
            if C2>0:
                P = float(C2)/float(C1)
                break
            w2 = w2[1:len(w2)]
        return P
    
    def Interpolate(self,w2):
        P = []
        n = len(w2)
        while w2 != '':
            w1 = w2[0:len(w2)-1]
            C1 = 0
            C2 = 0
            if self.hashCounts[len(w2)-1].has_key(w2):
                C2 = self.hashCounts[len(w2)-1][w2]
            if self.hashCounts[len(w1)-1].has_key(w1):
                C1 = self.hashCounts[len(w1)-1][w1]
                # Handling the Uni-gram Case
            if len(w1)==0:
                C1 = self.totalCount[0]
            w2 = w2[1:len(w2)]
            try:
                tmp = float(C2)/float(C1)
                P.append(tmp)
            except:
                continue
        
        res = 0
        for i in range(len(P)):  
            res +=P[i]
        return (float(res)/len(P))
    
    
    def InterpolateSmooth(self,w2):
        P = []
        n = len(w2)
        while w2 != '':
            w1 = w2[0:len(w2)-1]
            C1 = 0
            C2 = 0
            if self.hashCounts[len(w2)-1].has_key(w2):
                C2 = self.hashCounts[len(w2)-1][w2]
            if self.hashCounts[len(w1)-1].has_key(w1):
                C1 = self.hashCounts[len(w1)-1][w1]
                # Handling the Uni-gram Case
            if len(w1)==0:
                C1 = self.totalCount[0]
            w2 = w2[1:len(w2)]
            try:
                tmp = LaplaceSmoothing(float(C2),float(C1),float(self.V))
                P.append(tmp)
            except:
                continue
        
        res = 0
        for i in range(len(P)):  
            res +=P[i]
        return (float(res)/len(P))
    
    
    def InterpolateFreq(self,w2):
        P = []
        W = []
        n = len(w2)
        while w2 != '':
            w1 = w2[0:len(w2)-1]
            C1 = 0
            C2 = 0
            if self.hashCounts[len(w2)-1].has_key(w2):
                C2 = self.hashCounts[len(w2)-1][w2]
            if self.hashCounts[len(w1)-1].has_key(w1):
                C1 = self.hashCounts[len(w1)-1][w1]
                # Handling the Uni-gram Case
            if len(w1)==0:
                C1 = self.totalCount[0]
            w2 = w2[1:len(w2)]
            try:
                tmp = float(C2)/float(C1)
                W.append(C1**(len(w1)+1))
                P.append(tmp)
            except:
                continue
        
#        for i in range(len(P)):
#            W.append((len(P)-i)**2)
        res = 0
        for i in range(len(P)):  
            res += P[i]*W[i]
        return (float(res)/sum(W))
    
    def writeFile(self,filePath):
        file = open(filePath,'w')
        for i in range(self.gramNumber):
            file.write('----------------'+str(i+1)+'gram Counts'+'----------------\n')
            totProb = 0;
            for j in self.hashCounts[i]:
                key = j
                if i==0:
                    tot = self.totalCount[0]
                else:
                    key2 = key[0:len(key)-1]
                    tot = self.hashCounts[i-1][key2] 
                prob = float(self.hashCounts[i][j])/float(tot)
                totProb += prob
                file.write(str(j)+'    '+str(self.hashCounts[i][j])+'    '+ str(prob)+'\n')
            file.write('----------------Total Probabilities = '+str(totProb)+' for '+str(i+1)+'gram Model'+'----------------\n\n')
        



def preProcessDoc(T):
    stopList = [';',',','.','(',')','-','%','/',':','?','!','[',']','\n','\t','\b']
    for i in range(10):
        stopList.append(str(i))
    T2=''
    for i in range(len(T)):
        if T[i] in stopList:
            T2 += ' '
        else:
            T2 += T[i]
    T = T2
    
    i = 0
    j = 0
    newT= ''
    while i<len(T):
        if T[i]==' ':
            if (j>0) and (newT[j-1]==' '):
                i +=1
                continue
        newT += T[i]
        i +=1
        j +=1
    return newT
          
def LaplaceSmoothing(C1,C2,V):
    return float(C1+1)/float(C2+V)