Re: [INTERFACES] Fastest import?

Matthew Hagerty wrote:
>
> Greetings,
>
> What is the fastest way to import data from an ASCII file? I have ~17,000
> records in a .DBF file that I need to get into my pg6.4.2 database. I have
> written a program that reads the .DBF file, massages the data and writes it
> out to a text file, one record per line.

I do that straight under linux by reading the .dbf file with a small
C-program which writes the data out in tab-separated records. Those
can be read with the copy from command:

/* __________ db2u __________
convert dbase file
Usage:
db2u < dbase-file.dbf > unix_database
or
db2u dbase-file.dbf > unix_database
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#define MAXFIELDS 100 /* maximum nr of fields */

void main(int argc, char *argv[])
{ char *p,*q,*r;
typedef char string11[12];
FILE *in;
int i,j,n,field_pos[MAXFIELDS+1],nfield=0,pos,lrec;
char *record,field_type[MAXFIELDS];
string11 field_name[MAXFIELDS];
struct
{ char version; /* dBase version */
char datum[3]; /* date of last update YYMMDD */
long nrec; /* nr of records in th database file */
short nhead; /* nr of bytes in the header */
short lrec; /* nr of bytes in the record */
char dum1[2];
char transact; /* flag for incomplete transaction */
char encrypt; /* encryption flag */
char lan[12]; /* reserved for network applications */
char mdx; /* flag for presence of mdx-file */
char dum2[3];
} header;

struct
{ char name[11]; /* fieldname zero filled */
char type; /* field type (C, D, F, L, M, N) */
char dum1[4];
unsigned char length; /* field length in binary */
char ndec; /* field decimal count in binary */
char dum2[2];
char workid; /* work area ID */
char dum3[11];
} field;

switch(argc)
{ case 1: in=stdin;
break;
case 2: if ((in = fopen(argv[1],"rb")) == NULL)
{ fprintf(stderr,"Input file %s not found\n",argv[1]);
exit(1);
}
break;
default: fprintf(stderr,"Usage: %s dbase_file > unix_database\n",argv[0]);
fprintf(stderr,"Or: cat dbase_file | %s > unix_database\n",argv[0]);
exit(1);
}
fread(&header,32,1,in);
n=header.lrec;
pos=1;
while (n>1)
{ if (nfield==MAXFIELDS)
{ fprintf(stderr,"Too many fields\n");
exit(1);
}
fread(&field,32,1,in);
strcpy(field_name[nfield],field.name);
field_type[nfield]=field.type;
field_pos[nfield++]=pos;
pos+=(int)field.length;
n-=(int)field.length;
}
for (i=0;i<nfield;i++) printf("!%d-%s%c",i+1,field_name[i],i==nfield-1?'\n':'\t');
for (i=0;i<nfield;i++) printf( "!%s%c", field_name[i],i==nfield-1?'\n':'\t');
field_pos[nfield]=pos; /* one extra for length calculation of last field */
fread(&i,1,1,in); /*read field terminator */
if (i!=0xD)
{ fprintf(stderr,"Illegal field terminator; more than %d fields?\n",MAXFIELDS);
exit(1);
}
lrec=header.lrec;
record=malloc(lrec);
for (i=0;i<header.nrec;i++)
{ if (fread(record,lrec,1,in)!=1)
{ fprintf(stderr,"Error reading record %d\n",i+1);
exit(1);
}
if (*record!='*') /* if not a deleted record */
for (j=0;j<nfield;j++)
{
p=record+field_pos[j]; /* first char of the field */
q=record+field_pos[j+1]-1; /* last char of the field */
while (*p==' ' && p<=q) p++; /* remove leading spaces */
while (*q==' ' && p<=q) *(q--)='\0'; /* remove trailing spaces */
for (r=p;r<=q;r++) putchar(*r);
putchar(j==nfield-1?'\n':'\t');
}
}
}

--
Hartelijke groet, Wybo Dekker
___________________Servalys Analytical Chemistry Services__________________
wybo(at)servalys(dot)hobby(dot)nl | Deilsedijk 60 | tel +31-345-652164
www.hobby.nl/~servalys | 4158 CH Deil, The Netherlands | fax
+31-345-652383