#!/usr/bin/perl
use strict;
print "Fun with Perl and the Shakespeare corpus\n";

# a set of perl programs, similar to each other, that do basic
# text analysis

# the shakespeare corpus is available on the course web site
# I count 37 plays

my @plays = <shakespeare/*>;
printf "Read %d files in the shakespeare directory\n",scalar(@plays);

foreach my $entry (@plays) {
    open(FH, $entry);
    my @lines = <FH>;
    chomp(@lines);
    print "First line:  @lines[0]\n";
    close(FH);
}

# how many times does the word "Forsooth" appear?
my $nF = 0;
foreach my $entry (@plays) {
    open(FH, $entry);
    my @lines = <FH>;
    my $playTitle = @lines[0];
    my $lineNum = 0;
    chomp(@lines);
    foreach my $line (@lines) {
	$lineNum += 1;
	my @words = split /\s+/, $line;
	foreach my $_ (@words) {
	    if (/forsooth/i) {
		$nF +=1;
#		printf "Found a Forsooth in %s, line %d\n", 
		$playTitle, $lineNum;
	    }
	}
    }
    close(FH);
}
print "Total forsooths found: $nF\n";

# what are the ten most common words in the Shakespeare corpus?
# build a frequency distribution of all the words
my %lexicon = ();
foreach my $entry (@plays) {
    open(FH, $entry);
    my @lines = <FH>;
    chomp(@lines);
    foreach my $line (@lines) {
	my @words = split /\s+/, $line;
	foreach (@words) {
	    $lexicon{$_} += 1;
	}
    }
    close(FH);
}

# at this point, %lexicon has each word and the number of times it appears
foreach my $word (sort(keys(%lexicon))) {
    if ($lexicon{$word} > 7950) {
	print "$word occurs $lexicon{$word} time(s)\n";
    }
}

my $i = 0;
foreach my $word (sort { $lexicon{$b} <=> $lexicon{$a} } keys %lexicon) {
    if ($i > 10) {
	last;
    } else {
	$i++;
    }
    print "Most frequent word $i is $word, which occurs $lexicon{$word} times\n";
}


   
