package MatchedRef;
require Exporter;

use mStageMatcher;

@ISA  = qw(Exporter);
@EXPORT = qw(computeMRef extractTestSetId);


# Extract the testsetid from a translation file
sub extractTestSetId{
        my $file = shift;
        open(FILE,"<$file") or die("Couldn't open file $file");
        my $line = '';
	print "Extracting test set id\n";
        while(($line = <FILE>) !~ /tstset\s*setid=/){next;};
        $line =~ /setid=\"([^\"]+)\" /;
	print "Found the id\n";
        return $1;
}

# compute Matched References
sub computeMRef{
	my $sgmlFile = shift;
	my $refFile = shift;
	my $outFile = shift;
	
	my %data = ();
	
	readSGMfile($sgmlFile,\%data,'hyp');
	readSGMfile($refFile,\%data,'ref');
	
	my @sortedkeys = sort {    # This function sorts the segments of a doc in correct order
		my @akey = split /::/, $a;
		my @bkey = split /::/, $b;
		if ( $akey[0] eq $bkey[0] ) {
			return $akey[1] <=> $bkey[1];
		}
		else {
			return $akey[0] cmp $bkey[0];
		}
	} keys %data;

	foreach my $outerKey (@sortedkeys){
		my $alignmentRef = getAlignment(${ $data{$outerKey} }{'hyp'}[0],${ $data{$outerKey} }{'ref'}[0]);
# 		print "Alignment: @{$alignmentRef}\n";
		my @hypTokens = split /\s+/,${ $data{$outerKey} }{'hyp'}[0];
		for(my $r = 0; $r <= $#{${ $data{$outerKey} }{'ref'}}; $r++){
			my @refTokens = split /\s+/,${ $data{$outerKey} }{'ref'}[$r];
			for(my $i = 0;$i <= $#$alignmentRef; $i++){
				next if (${$alignmentRef}[$i] == -1);
				$refTokens[$i] = $hypTokens[${$alignmentRef}[$i]];
			}
# 			print "@refTokens\n";
			push @{${$data{$outerKey}}{'mref'}}, "@refTokens";
		}
	}
	writeMREFFile($refFile,$outFile,\%data,'mref');
}

BEGIN {
#my $wn = WordNet::QueryData->new;
#my %wnValidForms         = ();
#my %wnSynsetOffsetHashes = ();
my @modules = ("exact","porter_stem","wn_synonymy");
my %inputHash = ();
$inputHash{"language"}  = 'en';
$inputHash{"maxComputations"} = 10000;
$inputHash{"modules"} = \@modules;    # this array comes from the commandline
$inputHash{"prune"} = 1;
$inputHash{"details"} = 0;

sub getAlignment{
	my $hyp = shift;
	my $ref = shift;
	
	$inputHash{"firstString"}  = $hyp;
	$inputHash{"secondString"} = $ref;

	match( \%inputHash );
	return ($inputHash{'alignment'});
}

}

# Create MTER refs
sub writeMREFFile {
	my $testFile      = shift;
	my $outFile       = shift;
	my $allDataRef = shift;
	my $keyType = shift;
	
	open( RESFILE, ">$outFile" ) or die ("Couldn't open the output file $outFile\n");
	
# Let us read the test file
	open( FILE, $testFile ) || die "Couldn't open $testFile\n";
	my $currentDocID  = "";
	my $currentSysID  = "";
	my $lastAutoSegID = "";
	
	while ( my $line = <FILE> ) {
		if ( $line =~ /<DOC/i ) {
			print RESFILE "</doc>\n" if($currentDocID ne "");
			if ( $line =~ /docid=\"(.*?)\"/i ) {
				$currentDocID = $1;
			}
			else {
				die "Couldn't read document id from line: $line\n";
			}
			if ( $line =~ /sysid=\"([^\"]*)/i ) {
				$currentSysID = $1;
			}
			else {
				if($testFile =~ /\-src/){
					$currentSysID = 'src';
				}
				else{
					die "Couldn't read system id from line: $line\n";
				}
			}
		
			$lastAutoSegID = 0;
			print RESFILE "<DOC docid=\"$currentDocID\" sysid=\"$currentSysID\">\n";
			next;
		}

		if ( $line =~ /<\/DOC>/i ) {
			print RESFILE $line;
			$currentDocID = "";
			next;
		}
	
		my $currentSegID = 0;
		if ( $line =~ /<\s*seg\s*id\s*=\s*\"?\s*(.+?)\s*\"?\s*>/i ) {
			$currentSegID = $1;
		}
		elsif ( $line =~ /<\s*seg\s*>/i ) {
			$currentSegID = $lastAutoSegID + 1;
			$lastAutoSegID++;
		}
		elsif ($line =~ /<\s*\/?refset/i or $line =~ /<\s*\/?tstset/i or $line =~ /<\s*\/?srcset/i) {
			print RESFILE $line;
			next;
		}
		else{
			next;
		}

# Collect all the data of this seg
		while ( ( $line !~ /<\s*\/seg\s*>/i ) && ( my $nextLine = <FILE> ) ) {
			$line .= $nextLine;
		}
		my $string = "";
		my $lineToMatch = $line;
		$lineToMatch =~ s/\s/ /g;
	
# Extract the text
		if ( $lineToMatch =~ /<\s*seg.*?>(.*)<\s*\/seg\s*>/i ) {
			$string = $1;
		}
		else {
			die "Having trouble reading in the segment information from:\n$line\n";
		}
	
# if not inside a document, something's gone wrong
		if ( $currentDocID eq "" ) {
			die "The following segment line in file $testFile seems to be outside a DOC block!: [$line]\n";
}
	
		my $outerHashKey = $currentDocID . "::" . $currentSegID;
		my $mterRef = ${${$allDataRef}{$outerHashKey}}{$keyType}[0];
		print RESFILE "<seg id=\"$currentSegID\" >$mterRef";
		print RESFILE "</seg>\n";
		shift(@{${${$allDataRef}{$outerHashKey}}{$keyType}});
		next;
}
    close(FILE);
    close RESFILE;
}

sub readSGMfile {
    my $fileName   = shift;
    my $allDataRef = shift;
    my $prefix     = shift;

    open(FILE, "<$fileName" ) || die "Couldn't open $fileName\n";
    my $currentDocID  = "";
    my $currentSysID  = "";
    my $lastAutoSegID = "";

    while ( my $line = <FILE> ) {
# Example: <DOC docid="AFA20030101.5900" sysid="ahd">
# Note: In Tides 2002 data, different systems had tags with different cases (docid vs
# docID), so in all regexes below, do case insensitive matching (using the "i" flag)
        if ( $line =~ /<DOC/i ) {
            if ( $line =~ /docid=\"([^\"]*)\"/i ) {
                $currentDocID = $1;
		}
            else {
                die "Couldn't read document id from line: $line\n";
		}
            if ( $line =~ /sysid=\"([^\"]*)/i ) {
                $currentSysID = $prefix;
		}
            else {
            	if($fileName =~ /\-src/){
            		$currentSysID = 'src';
		}
		else{
                	die "Couldn't read system id from line: $line\n";
		}
		}

            $lastAutoSegID = 0;
            next;
	}

        if ( $line =~ /<\/DOC>/i ) {
            $currentDocID  = "";
            $currentSysID  = "";
            $lastAutoSegID = 0;    # redundant with re-initialization within the if (/<DOC... block, but what the heck!
            next;
	}

        my $currentSegID = 0;

# Example: <seg id=1> Izzet Ibrahim Meets Saudi Trade Official in Baghdad </seg>
# note that it is possible that the opening and closing seg tags are on separate lines
        if ( $line =~ /<\s*seg\s*id\s*=\s*\"?\s*(.+?)\s*\"?\s*>/i ) {
            $currentSegID = $1;
	}
        elsif ( $line =~ /<\s*seg\s*>/i ) {

# Example: <seg> Izzet Ibrahimi Meets </seg>
# this is the situation when the files don't have seg-ids. In
# this case, assume seg-ids start with 1 for a new document,
# and then increase by 1 until the end of that document
            $currentSegID = $lastAutoSegID + 1;
            $lastAutoSegID++;
	}
        else {
# Ignore everything else
            next;
	}

# Collect all the data of this seg
        while ( ( $line !~ /<\s*\/seg\s*>/i ) && ( my $nextLine = <FILE> ) ) {
            $line .= $nextLine;
	}
        my $string = "";
        my $lineToMatch = $line;
        $lineToMatch =~ s/\s/ /g;

# Extract the text
        if ( $lineToMatch =~ /<\s*seg.*?>(.*)<\s*\/seg\s*>/i ) {
            $string = $1;
	}
        else {
            die "Having trouble reading in the segment information from:\n$line\n";
	}

# if not inside a document, something's gone wrong
        if ( $currentDocID eq "" ) {
            die "The following segment line in file $fileName seems to be outside a DOC block!: [$line]\n";
	}

# otherwise, create the key, and push the data in!
        my $outerHashKey = $currentDocID . "::" . $currentSegID;
        @{ ${$allDataRef}{$outerHashKey} }{$currentSysID} = ()
          unless ( defined @{ ${$allDataRef}{$outerHashKey} }{$currentSysID} );

            my $normalizedString = normalizeText($string);
            push @{ ${ ${$allDataRef}{$outerHashKey} }{$currentSysID} }, $normalizedString;
        next;
	}
    close(FILE);
}

sub normalizeText {
    my ($norm_text) = @_;

# language-independent part:
    $norm_text =~ s/<skipped>//g;    # strip "skipped" tags
    $norm_text =~ s/-\n//g;          # strip end-of-line hyphenation and join lines
    $norm_text =~ s/\n/ /g;          # join lines
    $norm_text =~ s/&quot;/"/g;      # convert SGML tag for quote to "
    $norm_text =~ s/&amp;/&/g;       # convert SGML tag for ampersand to &
#     $norm_text =~ s/&lt;/</g;        # convert SGML tag for less-than to >
#     $norm_text =~ s/&gt;/>/g;        # convert SGML tag for greater-than to <

# language-dependent part (assuming Western languages):
    $norm_text = " $norm_text ";
    $norm_text =~ tr/[A-Z]/[a-z]/;
    
      $norm_text =~ s/([\{-\~\[-\` -\&\(-\+\:-\@\/])/ $1 /g;    # tokenize punctuation
      $norm_text =~ s/([^0-9])([\.,])/$1 $2 /g;                 # tokenize period and comma unless preceded by a digit
      $norm_text =~ s/([\.,])([^0-9])/ $1 $2/g;                 # tokenize period and comma unless followed by a digit
      $norm_text =~ s/([0-9])(-)/$1 $2 /g;                      # tokenize dash when preceded by a digit
      $norm_text =~ s/[_#<]/ /g;                                 # We remove _ & # since wn is not able to handle them.
    $norm_text =~ s/\s+/ /g;                                  # one space only between words
    $norm_text =~ s/^\s+//;                                   # no leading space
    $norm_text =~ s/\s+$//;                                   # no trailing space

    return $norm_text;
}

