postmaster disconnects after heavy load inserts from plperlu -> waht to do?

From: "Christian Maier" <tomtailor(at)freesurf(dot)fr>
To: pgsql-general(at)postgresql(dot)org
Subject: postmaster disconnects after heavy load inserts from plperlu -> waht to do?
Date: 2007-01-15 19:11:04
Message-ID: 1168888264.200333.166440@38g2000cwa.googlegroups.com
Views: Raw Message | Whole Thread | Download mbox | Resend email
Thread:
Lists: pgsql-general

Hello!

I have written a function to load yahoo quote data. abut after parsing
the inserts will overload the server and disconnects. What sould I do
to avound this?

THX
Christian Maier

PS Here the function:

CREATE OR REPLACE FUNCTION get_yahoo(VARCHAR(20), BIGINT, VARCHAR(3))
RETURNS INTEGER AS $$
use LWP::Simple;
use Date::Simple(':all');

my $symbol = $_[0];
my $market_id = $_[1];
my $symbol_whg = $_[2];
my $market_id_whg = $market_id;

my $fi = spi_exec_query( qq { SELECT fi_id
FROM fi.provider
WHERE symbol = '$symbol'
AND provider = 'yahoo'
AND sysstatus = 0;
});
my $fi_id = $fi->{rows}[0]->{fi_id};
my $fi = spi_exec_query( qq { SELECT fi_id
FROM fi.provider
WHERE symbol = '$symbol_whg'
AND provider = 'iso'
AND sysstatus = 0;
});
my $fi_id_whg = $fi->{rows}[0]->{fi_id};

my $query = qq { SELECT max(eod_day) as lastdate
FROM fi.eod
WHERE fi_id = $fi_id
AND market_id = $market_id
AND fi_id_whg = $fi_id_whg;
};

elog(NOTICE, "$query");
my $lastdate = spi_exec_query($query);

my $from;
($from = Date::Simple->new($lastdate->{rows}[0]->{lastdate}) ) ||
($from = Date::Simple->new('1990-01-01') );
my $to = today();
my $fday = $from->day;
my $fmon = sprintf("%02d", $from->month -1);
my $fyea = $from->year;
my $tday = $to->day;
my $tmon = sprintf("%02d", $to->month -1);
my $tyea = $to->year;

elog(NOTICE, "$fyea-$fmon-$fday - $tyea-$tmon-$tday");

my $content =
get("http://ichart.finance.yahoo.com/table.csv?s=$symbol&a=$fmon&b=$fday&c=$fyea&d=$tmon&e=$tday&f=$tyea&g=d&ignore=.csv");
elog(NOTICE, "$content");
my @lines;
if ($contend =~ m/\\r\\n/) {
elog(NOTICE, "Windows");
@lines = split ("\r\n", $content);
} else {
elog(NOTICE, "Unix");
@lines = split ("\n", $content);
}

foreach my $line(@lines) {
my @columns = split(/,/, $line);
#Date,Open,High,Low,Close,Volume,Adj. Close*
#12-Jan-07,30.10,31.17,30.05,30.79,9466200,30.79
if ($columns[0] != "Date") {
$columns[0] =~ s/Jan/01/;
$columns[0] =~ s/Feb/02/;
$columns[0] =~ s/Mar/03/;
$columns[0] =~ s/Apr/04/;
$columns[0] =~ s/May/05/;
$columns[0] =~ s/Jun/06/;
$columns[0] =~ s/Jul/07/;
$columns[0] =~ s/Aug/08/;
$columns[0] =~ s/Sep/09/;
$columns[0] =~ s/Oct/10/;
$columns[0] =~ s/Nov/11/;
$columns[0] =~ s/Dec/12/;

my @date = split(/-/, $columns[0]);
$columns[0] = $date[2] . "-" . $date[1] . "-" .
sprintf("%02d", $date[0]);
$insert = qq{ INSERT INTO
fi.eod(fi_id,fi_id_whg,market_id,market_id_whg,eod_day,o,h,l,c,vo)
VALUES($fi_id, $fi_id_whg,
$market_id, $market_id_whg, $columns[0],
$columns[1], $columns[2],
$columns[3], $columns[4], $columns[5]); };
elog(NOTICE, "$insert");
#this occours the crash
spi_exec_query( $insert );
} #end if
}#end each line
1;
$$ LANGUAGE plperlu;

Responses

Browse pgsql-general by date

  From Date Subject
Next Message Shane Ambler 2007-01-15 20:15:48 Re: Reserve a value in a serial type field
Previous Message Oleg Bartunov 2007-01-15 19:09:52 Re: Avoiding empty queries in tsearch