#!/usr/bin/perl

=head1 NAME

eval-TS-treceval.pl - Evaluate a run using trec_eval

=head1 SYNOPSIS

eval-treceval.pl [options] (type) run ...

  Options:
    -qrels           Specify which qrels file to use
    -track           Track root directory
    -level           Relevance level cutoff (default = 1)
    -opts            Options to trec_eval  ('-q -M1000' by default)
	-eqclasses       Location of the eqclasses file

  (type) should be one of {business, scitech, sport, us, world}.

=head1 DESCRIPTION

Re-sorts the run's input file correctly and puts it through trec_eval.
For documents that are part of an equivalence class, output only the
first document to appear in a class.

=head1 HISTORY

The trec_eval invocation is F<trec_eval -q -M1000>, which means to
output individual query results, and only look at retrieved documents
to rank 1000.  This latter limitation is implicit in many of the
measures, and some measures will change significantly if evaluated to
different ranks.

=head1 FILES

Requires trec_eval version 7.2 or later.  Additionally, this script 
expects the common UNIX and/or GNU tools mktemp, gzip, and sort to be
on the path.

The run is expected to be found in F<$root/results/RUN/$RUN.gz>.

Output is to stdout.  A control script (usually eval-all-tasks.pl or
some such) is used to catch the output and put it in the right place.

=cut

use strict;
use Pod::Usage;
use Getopt::Long;

# Change these to match your local setup.

my $root = "/trec/trec19/blog";
my $qrels_root = "$root/eval";
my $qrels_all = "$qrels_root/TS10_Binary.WHICH.prels";
my $eq_classes = "$qrels_root/TS10_eqclasses";

my $trec_eval = "$root/scripts/statAP_MQ_eval_v4.pl";
my $trec_eval_opts = "-q";

# Also change declaration of $runfile below to match your setup.

GetOptions("qrels=s" => \$qrels_all,
	   "root=s"  => \$root,
           "opts=s"  => \$trec_eval_opts,
	   "eqclasses=s" => \$eq_classes
	   ) or pod2usage(2);
my $which = shift or die pod2usage();
my $run = shift or die pod2usage();

$qrels_all =~ s/WHICH/$which/;
my $trec_eval = "$trec_eval $trec_eval_opts";

my %classof;
my %seen;

open CLASSES, $eq_classes or die "Can't read $eq_classes: $!\n";
while (<CLASSES>) {
    chomp;
    my ($day, $first, @rest) = split;

    $classof{$first}{$first} = 1;

    # note class of each page.
    for my $d (@rest) {
	$classof{$d}{$first} = 1;
    }
}
close CLASSES;

my $runfile = "$root/results/$run/$run.gz";
my $runtmp = `mktemp -q /tmp/eval.XXXXXX`;
die "Error: couldn't create temporary run file $runtmp\n" if ($? >> 8) != 0;

my $open = ($runfile =~ /.gz$/) ? "gzip -dc $runfile" : "cat";
open IN, "$open | sort -s -k 1,1n |" 
    or die "Can't cat/gunzip/sort $runfile: $!\n";
open OUT, ">$runtmp" or die "Can't open output file $runtmp: $!\n";
while (<IN>) {
    my ($topic, undef, $docid, $rank, $sim, $tag) = split;
    next unless $topic =~ /$which$/o;
    $topic =~ s/TS10-(\d+)-$which/\1/o;
    $topic =~ s/^0*//;

    if (exists $classof{$docid}) {
	my $chead = (keys %{ $classof{$docid} })[0];
	$docid = $chead;
    }
    next if exists $seen{$topic}{$docid};

    print OUT "$topic Q0 $docid $rank $sim $tag\n";
    $seen{$topic}{$docid} = 1;
}
close IN;
close OUT;

warn "CMDLINE $trec_eval $qrels_all $runtmp \n";
open(EVAL, "$trec_eval $qrels_all $runtmp |")
    or die "Can't run trec_eval on $runtmp: $!\n";

while (<EVAL>) {
    print;
}
close(EVAL);

qx(rm -f $runtmp);
die "Error: couldn't remove temporary run file $runtmp\n" if ($? >> 8) != 0;

