#!/usr/bin/perl 
#
# protoype of homework 6
# given a URL, find the URLs it links to and print the list
# 
# CKN another Perl example for CMSC 331
# CKN credit to King Beetle http://kingbeetle66.blogspot.com
# CKN blog entry /2009/10/using-perls-lwpuseragent-and-htmlform_3083.html
# CKN with comments added by CKN
#

use strict; 

use LWP::Simple;
use HTML::LinkExtor;

sub listURLs { 
    my ($targetURL, $p, $result);

    $p = HTML::LinkExtor->new(\&cb);
    sub cb {
	my($tag, %links) =@_;
	print "<$tag @{[%links]}>\n";
    }

# we assume a URL as the argument
# if none is given, assume www.umbc.edu
    if (@ARGV) {$targetURL = $ARGV[0]}  
    else {$targetURL = "http://www.umbc.edu"}  
    print "looking at $targetURL\n";
    
    my $content = get($targetURL);
    die "Couldn't get it!" unless defined $content;
    
    $p->parse($content);
}

sub compareWebPages {
#    use Compress::Bzip2;
    use Compress::Zlib;
    use List::Util qw(max min);

    my ($url1, $url2) = @_;
    print "First URL is $url1\n";
    my $content1 = get($url1) or die "Can't get $url1";
#    printf ("length of $url1 content is %d\n", length($content1));
#    my $zip1 = memBzip($content1);
    my $zip1 = compress($content1);
    my $zlen1 = length($zip1);
    printf ("length of $url1 content compressed is %d\n", $zlen1);

    print "Second URL is $url2\n";
    my $content2 = get($url2) or die "Can't get $url2";
#    printf ("length of $url2 content is %d\n", length($content2));
#    my $zip2 = memBzip($content2);
    my $zip2 = compress($content2);
    my $zlen2 = length($zip2);
    printf ("length of $url2 content compressed is %d\n", $zlen2);

# refer to normalized information distance 
# from Li, Chen, Li, Ma and Vitanyi, "The Similarity Metric"
# IEEE transactions on Information Theory, 50(12) Dec 2004
# NCD(x,y) = (C(xy) - min{C(x),C(y)})/max{C(x),C(y)}
    
    printf ("min{C(url1),C(url2)} is %d\n", min($zlen1,$zlen2));
    printf ("max{C(url1),C(url2)} is %d\n", max($zlen1,$zlen2));
#    my $cxy = memBzip($content1.$content2);
    my $cxy = compress($content1.$content2);
    printf ("length of C(url1+url2) is %d\n", length($cxy));
    printf ("NCD of $url1 and $url2 is %f\n", 
	    (length($cxy) - min($zlen1,$zlen2))/max($zlen1,$zlen2));

}

#listURLs;
compareWebPages("http://www.google.com","http://www.google.com");
compareWebPages("http://www.google.com","http://www.google.fr");
compareWebPages("http://www.google.com","http://www.google.fi");
compareWebPages("http://www.yahoo.com","http://www.yahoo.com");
compareWebPages("http://www.umbc.edu","http://www.umbc.edu");
