#Homework 3
#Joseph Campisi III
#This pretty much yields the rest of hw3.  You simply change some of the
#file names to adjust what your actually testing on.  Look below you'll see.

from nltk.token import *
from nltk.probability import *
from nltk.draw.plot import Plot
import string

def p1 () :

        # A Location Beginning at position 5 and ending at position 6
        # having the unit w (word)

        location_a = Location(5, 6, unit='w', source='foo.txt')
        print location_a

        # A token "parrot" at the above location.

        token_a = Token('parrot', location_a)
        print token_a

        # The type of the above token.

        print token_a.type()

        # The location of this token

        print token_a.loc()


def p2 () :

        #This should be changed to the proper path
        #should you try to run it on unixs or another machine.
        #dos didn't like the ,'s in the file name so i had to
        #rename it in order to run this function.

        #Set the file handle.
        warmup = open('2a.ascii').read()

        #Read in and tolkenize the file.
        tokens_a = WSTokenizer().tokenize(warmup)

        print "1 token per line"

        for token in tokens_a:
                print "%s" % token


def wordstats ():

        #Open Full Corpus
        partb = open('2000-2002.ascii').read()

	#TESTING ONLY for testing with a smaller file.
	#partb = open('2a.ascii').read()
	#TESTING ONLY it takes much less running time.

        tokens = WSTokenizer().tokenize(partb)

        #Print the number of tokens.
        print "Number of tokens = %s" % len(tokens)

        #Need to store the unique tokens and a count in a dictionary.
        dict = {}

	#Needed to hold the word frequency.
	freq_dist = FreqDist()
	
	#Used to produce plot of word lengths
	freq_dist2 = FreqDist()

	#Used to produce plot of stop word frequency / content freq
	cfdist = ConditionalFreqDist()

	#Used to produce plot of content word frequency
	#cfdist_content = ConditionalFreqDist()

	#I used a dictionary to hold the stop words to making the checking
	#easier below.  One a word is read from stop words, we make it the
	#key for easier testing later, the 1 is pretty much not important.

	cfdist_dict = {}
	dict_file = open('stopwords.ascii').read()
	dict_tokens = WSTokenizer().tokenize(dict_file)

	for tokk in dict_tokens:
		cfdist_dict[tokk.type().lower()] = 1

        for token in tokens:

		#Build our dictionary to calculate word counts.
                if dict.has_key(token.type().lower()):
                        dict[token.type().lower()] += 1
                else:
                        dict[token.type().lower()] = 1


		#Build the frequncy distribution of word type.
		freq_dist.inc(token.type().lower())

		#Build the word length frequency distribution.
		freq_dist2.inc(len(token.type()))

		#Build the stop word frequency and content word frequency
		if cfdist_dict.has_key(token.type().lower()):
			#increment cfreq for stop
			context = 'stop'
			outcome = token.type().lower()
			cfdist[context].inc(outcome)

		else:
			#increment cfeq for content
			context = 'content'
			outcome = token.type().lower()
			cfdist[context].inc(outcome)
		

        #Total Unique Types.
        print "Unique types = %s" % len(dict)

        #Now we need some trickery to sort the dictionary
        #since we need to sort based on member rather then key.
        #So make a list, and invert the keys and members from the dictionary
        #Then sort them.
        #Then we reverse them to put the largest first.
        #Finally we could flip the list back to key,value form, but it
        #looks much nicer in value,key form.
        items = [(v, k) for k, v in dict.items()]
        items.sort()
        items.reverse()

        #If you want it back in key,value form uncomment this line.
        #items = [(k, v) for v, k in items]
	
	print "Word / Word Counts, sorted by count."
        for item in items:
                print item

	
	#Now print the frequency.
	foo = []
	wordlens = freq_dist.samples() 
	for word in wordlens:
		foo.append((freq_dist.freq(word), word))
	
	foo.sort()
	foo.reverse()
	print "Word / Word Frequencies, sorted by Frequency"
	for xx in foo:
		print xx
		
	
	#Now you need to plot the freq dist of word lengths.
	wordlens2 = freq_dist2.samples()
	wordlens2.sort()
	points = [(l, freq_dist2.freq(l)) for l in wordlens2]
	Plot(points)

	#Conditional freqeuncy graph for stop words.
	condit = cfdist['stop'].samples()
	condit.sort()
	g = 0
	points2 = []
	for ll in condit:
		points2.append((g, cfdist['stop'].freq(ll)))
		g = g+1
	Plot(points2)

	#Now print content dist freq for content words.
	condit2 = cfdist['content'].samples()
	condit2.sort()
	h = 0
	points3 = []
	for lll in condit2:
		points3.append((h, cfdist['content'].freq(lll)))
		h = h+1
	Plot(points3)

def linestats ():

	#Open Full Corpus
	part3 = open('2000-2002.ascii').read()

	ltokens = LineTokenizer().tokenize(part3)

	#Print the number of tokens.
	print "Number of tokens (sentences) = %s" % len(ltokens)

	#Used to produce plot of word lengths
	freq_dist_line = FreqDist()

	for ltoken in ltokens:
		
		#Build the word length frequency distribution.
		freq_dist_line.inc(len(ltoken.type()))

	#Now you need to plot the freq dist of word lengths.
	wordlensline = freq_dist_line.samples()
	wordlensline.sort()
	pointsline = [(m, freq_dist_line.freq(m)) for m in wordlensline]
	Plot(pointsline)


#Function Usage
#p1()
#p2()
#wordstats()
#linestats()