#!/usr/bin/perl -w

# Creates PubsParsed and ExptSets and ExptsInSets in the named directory
my $BaseURL = "ftp://genome-ftp.stanford.edu/pub/smd/publications";
my $LinksCMD = "ncftpget -c";

my $dir = shift @ARGV || die "Usage: SMDparsePubsMeta.pl Directory\n";
opendir(DIR,$dir) || die "Cannot open directory $dir\n";
my @pubs = ();
while($filename = readdir(DIR)) {
    if($filename =~ m/^publication_([0-9]+).meta$/) {
	push @pubs, $1;
    }
}
closedir(DIR);


# reads from file handle META. arguments: toplevel and 2nd-level names
# returns: list of contexts and hashes
sub parseTwoLevelXML($$) {
    my ($top, $second, $name) = @_;
    my $context = "";
    my @contexts = ();
    my @hashes = ();
    while(<META>) {
	if (m#<$top>#) {
	    $context = "$top";
	    push(@contexts, $context);
	    push(@hashes, {});
	}
	$context = "" if m#</$top>#;
	if (m#<$second>#) {
	    die "saw $second with context $context in $name" if $context ne "$top";
	    $context = "$second";
	    push(@contexts, $context);
	    push(@hashes, {});
	}
	$context = "$top" if m#</$second>#;
	if (m#^[ \t]*!([A-Za-z]+)=([^\r\n]*)[\r\n]+$#) {
	    my $attr = $1;
	    my $value = $2;
	    die "saw value !$attr=$value in $name without context" if $context eq "";
	    die "Duplicate assignment !$attr=$value in $name" if exists $hashes[$#hashes]->{$attr};
	    $hashes[$#hashes]->{$attr} = $value;
	}
    }
    if (scalar @hashes < 1 || $context ne "") {
	print STDERR "Warning: $name does not terminate correctly - processing will continue anyway\n";
    }
    return(\@contexts, \@hashes);
}

open(PUBS,">","$dir/PubsParsed") || die "Cannot write to $dir/PubsParsed";
open(EXPTSETS,">","$dir/ExptSets") || die "Cannot write to $dir/ExptSets";
open(EXPTSINSETS,">","$dir/ExptsInSets") || die "Cannot write to $dir/ExptsInSets";

my @pubFields = qw/Pub PubMedID Citation Title/;
my @exptsetFields = qw/Pub ExptSetNo Name Description/;
my @exptsInSetsFields = qw/Pub ExptSetNo Exptid Name/;

print PUBS join("\t",@pubFields),"\n";
print EXPTSETS join("\t",@exptsetFields),"\n";
print EXPTSINSETS join("\t",@exptsInSetsFields),"\n";

foreach my $pub (sort {$a <=> $b} @pubs) {
    open(META,"<","$dir/publication_$pub.meta") || die "Cannot open meta file for $pub\n";
    my ($contextsRef,$hashesRef) = &parseTwoLevelXML("publication","experiment_set","pub $pub");
    my @contexts = @{ $contextsRef };
    my @hashes = @{ $hashesRef };
    close(META) || die "Cannot close meta file for $pub\n";

    while(scalar @hashes) {
	my $hash = shift @hashes;
	my $context = shift @contexts;
	die if exists $hash->{"Pub"};
	$hash->{"Pub"} = $pub;
	if ($context eq "publication") {
	    foreach my $field (@pubFields) {
		if (!exists $hash->{$field}) {
		    print STDERR "Warning: no value for $field in pub $pub\n";
		    $hash->{$field} = "";
		}
	    }
	    print PUBS join("\t", map {$hash->{$_}} @pubFields),"\n";
	} elsif ($context eq "experiment_set") {
	    if (!exists $hash->{"ExptSetNo"}) {
		print STDERR "Warning: ExptSet without an ExptSetNo in pub $pub - skipping it\n";
	    } else {
		foreach my $field (@exptsetFields) {
		    if (!exists $hash->{$field}) {
			print STDERR "Warning: no value for $field in pub $pub expset " . $hash->{"ExptSetNo"} . "\n";
			$hash->{$field} = "";
		    }
		}
		print EXPTSETS join("\t", map {$hash->{$_}} @exptsetFields),"\n";
		# and load & parse this experiment set!
		my $exptset = $hash->{"ExptSetNo"};
		open(META,"$LinksCMD $BaseURL/$pub/$exptset/exptset_$exptset.meta |")
		    || die "Cannot open pipe for pub $pub exptset $exptset";
		my ($eCRef,$eHRef) = &parseTwoLevelXML("experiment_set","experiment","pub $pub exptset $exptset");
		my @eC = @{ $eCRef };
		my @eH = @{ $eHRef };
		while(scalar @eH) {
		    my $exptHash = shift @eH;
		    my $eContext = shift @eC;
		    next if $eContext ne "experiment"; # ignore experiment_set - no more top-level data
		    if (!exists $exptHash->{"Exptid"}) {
			print STDERR "Warning: Expt without an Exptid in pub $pub exptset $exptset -- skipping it\n";
		    } else {
			my $exptid = $exptHash->{"Exptid"};
			$exptHash->{"Pub"} = $pub;
			$exptHash->{"ExptSetNo"} = $exptset;
			foreach my $field (@exptsInSetsFields) {
			    if (!exists $exptHash->{$field}) {
				print STDERR "Warning: no value for $field in pub $pub exptset $exptset expt $exptid\n";
				$exptHash->{$field} = "";
			    }
			}
			print EXPTSINSETS join("\t", map {$exptHash->{$_}} @exptsInSetsFields),"\n";
		    }
		}
		close(META) || die "Cannot close pipe for pub $pub exptset $exptset";
	    }
	}
    }
}

close(PUBS) || die "Error closing PUBS";
close(EXPTSETS) || die "Error closing EXPSETS";
close(EXPTSINSETS) || die "Error closing EXPSINSETS";
