#!/usr/bin/perl -w
use strict;

my $ROOT = shift or die;
my $url = shift or die;
my $name = shift or die;

$|++;

my $urlNoTrail = $url;
$urlNoTrail =~ s/\/[^\/]*$//; $urlNoTrail .= '/';

my $urlHead = $url;
if ($url =~ /^(.*):\/\/([^\/]+)/) {
    $urlHead = $1 . "://" . $2 . '/';
}

my $indexname = $url;
$indexname =~ s/^.*:\/\///;
$indexname =~ s/\//___/g;
$indexname =~ s/[^A-Za-z0-9\.]/_/g;

#if (-e "$ROOT/index/$indexname") { exit; }

`touch $ROOT/index/$indexname`;
`mkdir $ROOT/tmp/$indexname`;

my %pdfs = ();
my $txt = '';
open F, "wget -q -U Mozilla -O - '$url' | ";
while (<F>) { $txt .= $_; }
close F;

my $href = "[hH][rR][eE][fF]";

my $pdfId = 0;
while ($txt =~ s/$href=\"([^\"]+.pdf)\"//) { 
    my $f = $1;
    if ($f =~ /^\//) { $f = $urlHead . $f; }
    elsif (not ($f =~ /:\/\//)) { $f = $urlNoTrail . $f; }
    if (not exists $pdfs{$f}) { $pdfs{$f} = $pdfId++; }
}
while ($txt =~ s/$href=([^>]+.pdf)>//) { 
    my $f = $1;
    if ($f =~ /^\//) { $f = $urlHead . $f; }
    elsif (not ($f =~ /:\/\//)) { $f = $urlNoTrail . $f; }
    if (not exists $pdfs{$f}) { $pdfs{$f} = $pdfId++; }
}

while ($txt =~ s/$href=\"(http:\/\/doi.acm.org\/[0-9\.\/]+)\"//) {
    if (not exists $pdfs{$1}) { $pdfs{$1} = $pdfId++; }
}
while ($txt =~ s/$href=(http:\/\/doi.acm.org\/[0-9\.\/]+)>//) {
    if (not exists $pdfs{$1}) { $pdfs{$1} = $pdfId++; }
}

while ($txt =~ s/$href=\"(http:\/\/doi.ieeecomputersociety.org\/[^\"]+)\"//) {
    if (not exists $pdfs{$1}) { $pdfs{$1} = $pdfId++; }
}
while ($txt =~ s/$href=(http:\/\/doi.ieeecomputersociety.org\/[^>]+)>//) {
    if (not exists $pdfs{$1}) { $pdfs{$1} = $pdfId++; }
}

while ($txt =~ s/$href=\"(http:\/\/dx.doi.org\/[^\"]+)\"//) {
    if (not exists $pdfs{$1}) { $pdfs{$1} = $pdfId++; }
}
while ($txt =~ s/$href=(http:\/\/dx.doi.org\/[^>]+)>//) {
    if (not exists $pdfs{$1}) { $pdfs{$1} = $pdfId++; }
}

while ($txt =~ s/$href=\"(http:\/\/.*.computer.org\/[^\"]+)\"//) {
    if (not exists $pdfs{$1}) { $pdfs{$1} = $pdfId++; }
}
while ($txt =~ s/$href=(http:\/\/.*.computer.org\/[^>]+)>//) {
    if (not exists $pdfs{$1}) { $pdfs{$1} = $pdfId++; }
}


print "Downloading PDFs...";
foreach my $f (keys %pdfs) {
    print ".";
    my $o = "$ROOT/tmp/" . $indexname . '/' . $pdfs{$f};
    if ((not -e $o) && (not -e "$o.txt")) {
        if ($f =~ /^http:\/\/doi.acm.org/) { getACM($f,$o); }
        elsif ($f =~ /^http:\/\/doi.ieeecomputersociety.org/) { getIEEE($f,$o); }
        elsif ($f =~ /^http:\/\/dx.doi.org/) { getSpringer($f,$o); }
        elsif ($f =~ /^http:\/\/.*.computer.org/) { getComputerOrg($f,$o); }
        else { `wget -q -U Mozilla -O $o '$f'`; }
    }
}
print "\n";

print "Converting to text...";
foreach my $f (keys %pdfs) {
    print ".";
    my $o = "$ROOT/tmp/" . $indexname . '/' . $pdfs{$f};
    if (not -e $o) { next; }
    if (-e "$o.txt") { next; }
    `pdftotext $o`;
}
print "\n";

open I, "> $ROOT/index/$indexname" or die;

print I $url . "\n";
print I $name . "\n";

print "Extracting reference text...";
foreach my $f (keys %pdfs) {
    print ".";
    my $txt = "$ROOT/tmp/" . $indexname . '/' . $pdfs{$f} . '.txt';
    if (not -e $txt) { next; }
    my %words = ();
    open F, "cat $txt | $ROOT/getreftext.pl | $ROOT/preprocess.perl -q |" or next;
    while (<F>) {
        chomp;
        $_ = lc($_);
        s/[^A-Za-z ]/ /g;
        my @w = split /[\s]+/, $_;
        foreach my $w (@w) { 
            if (length($w) > 3) { $words{$w}++; }
        }
    }
    close F;

    my $title = '';
    open F, $txt or next;
    $title = <F>; 
    if (not defined $title) { next; }
    chomp $title;
    if ($title =~ /^Journal of Machine Learning Research/) {
        $title = <F>; $title = <F>; $title = <F>; $title = <F>; chomp $title;
    } elsif (($title =~ /Track Poster/) || ($title =~ /Track Paper/)) {
        $title = <F>; $title = <F>; chomp $title;
    } elsif ($title =~ /^Proceedings of/) {
        while ($title = <F>) { chomp $title; if (not ($title =~ /^[\s]*$/)) { last; } }
    }
    close F;
    if (not defined $title) { next; }
    $title =~ s/[\s]+/ /g;

    if (scalar keys %words < 50) { next; }

    print I $f . "\t" . $pdfs{$f} . "\t" . $title . "\t";
    my $first = 1;
    foreach my $w (sort keys %words) {
        if (not $first) { print I ' '; }
        print I $w;
        $first = 0;
    }
    print I "\n";
}
close I or die;

print "\n";


sub getACM {
    my ($doi, $o) = @_;

    open DOI, "wget -q -U Mozilla -O - '$doi' |" or die;
    my $url = '';
    while (<DOI>) {
        if (/<A NAME=\"FullText\" HREF=\"(ft_gateway.cfm?.+)\"/i) {
            $url = $1;
            last;
        }
    }
    close DOI;
    if ($url ne '') { 
        `wget -q -U Mozilla --referer='$doi' -O $o 'http://portal.acm.org/$url'`;
        print STDERR "+";
    } else { 
        print STDERR "-";
    }
}

sub getIEEE {
    my ($doi, $o) = @_;

    open DOI, "wget -q -U Mozilla -O - '$doi' |" or die;
    my $txt = '';
    my $url = '';
    while (<DOI>) {
        chomp;
        if (/<A href=\"([^\"]+)\" class=\"bodyCopy\">PDF/) {
            $url = 'http://ieeexplore.ieee.org' . $1;
            last;
        }
        $txt .= $_ . ' ';
        if ($txt =~ /Full Article Text:[\s]*<a href=\"(http[^\"]+\.pdf)\">/) { $url = $1; last; }
        if (not ($txt =~ /Full Article Text:/)) { $txt = ''; }
    }
    close DOI;

    if ($url ne '') { 
        `wget -q -U Mozilla --referer='$doi' -O $o '$url'`;
        print STDERR "+";
    } else { 
        print STDERR "-";
    }
}

sub getSpringer {
    my ($doi, $o) = @_;

    open DOI, "wget -q -U Mozilla -O - '$doi' |" or die;
    my $url = '';
    while (<DOI>) {
        if (/<a target=\"_blank\" href=\"([^\"]+\.pdf)\">Entire document<\/a>/) {
            $url = 'http://www.springerlink.com' . $1;
            last;
        }
        if (/<A href=\"([^\"]+)\" class=\"bodyCopy\">PDF/) {
            $url = 'http://ieeexplore.ieee.org' . $1;
            last;
        }
    }
    close DOI;
    if ($url ne '') { 
        print STDERR "\n$url  ";
        `wget -q -U Mozilla --referer='$doi' -O $o '$url'`;
        print STDERR "+";
    } else { 
        print STDERR "-";
    }
}

sub getComputerOrg {
    my ($doi, $o) = @_;

    open DOI, "wget -q -U Mozilla -O - '$doi' |" or die;
    my $url = '';
    while (<DOI>) {
        if (/<a href=\"(http:\/\/.*\.computer\.org\/dl\/proceedings[^\"]+)\"/) {
            $url = $1;
            last;
        }
    }
    close DOI;
    if ($url ne '') { 
        `wget -q -U Mozilla --referer='$doi' -O $o '$url'`;
        print STDERR "+";
    } else { 
        print STDERR "-";
    }
}

