From 4d1502a37c329c28977c84281cdec3a000ba30de Mon Sep 17 00:00:00 2001 From: Antonin Houska Date: Fri, 5 Jul 2019 16:24:02 +0200 Subject: [PATCH 12/17] Add tests for buffile.c. Since the encryption adds complexity to the buffile.c, regression tests seem like a good idea. In order to be able to test corner cases that deal with segment boudaries, this patch makes the segment size configurable - the tests would be too expensive with 1GB segments. The new GUC is marked with DEVELOPER_OPTIONS, DBA is not supposed to use it during normal operation. --- src/backend/storage/file/buffile.c | 73 ++--- src/backend/utils/misc/guc.c | 34 +- src/include/storage/buffile.h | 19 ++ src/test/modules/buffile/Makefile | 21 ++ src/test/modules/buffile/README | 11 + src/test/modules/buffile/buffile--1.0.sql | 54 ++++ src/test/modules/buffile/buffile.c | 429 ++++++++++++++++++++++++++ src/test/modules/buffile/buffile.control | 5 + src/test/modules/buffile/expected/test_01.out | 61 ++++ src/test/modules/buffile/expected/test_02.out | 48 +++ src/test/modules/buffile/expected/test_03.out | 27 ++ src/test/modules/buffile/expected/test_04.out | 84 +++++ src/test/modules/buffile/expected/test_05.out | 33 ++ src/test/modules/buffile/expected/test_06.out | 43 +++ src/test/modules/buffile/expected/test_07.out | 41 +++ src/test/modules/buffile/expected/test_08.out | 41 +++ src/test/modules/buffile/expected/test_09.out | 39 +++ src/test/modules/buffile/expected/test_10.out | 76 +++++ src/test/modules/buffile/expected/test_11.out | 34 ++ src/test/modules/buffile/expected/test_12.out | 16 + src/test/modules/buffile/expected/test_13.out | 86 ++++++ src/test/modules/buffile/sql/test_01.sql | 17 + src/test/modules/buffile/sql/test_02.sql | 13 + src/test/modules/buffile/sql/test_03.sql | 7 + src/test/modules/buffile/sql/test_04.sql | 25 ++ src/test/modules/buffile/sql/test_05.sql | 8 + src/test/modules/buffile/sql/test_06.sql | 15 + src/test/modules/buffile/sql/test_07.sql | 13 + src/test/modules/buffile/sql/test_08.sql | 13 + src/test/modules/buffile/sql/test_09.sql | 9 + src/test/modules/buffile/sql/test_10.sql | 25 ++ src/test/modules/buffile/sql/test_11.sql | 9 + src/test/modules/buffile/sql/test_12.sql | 7 + src/test/modules/buffile/sql/test_13.sql | 25 ++ 34 files changed, 1420 insertions(+), 41 deletions(-) create mode 100644 src/test/modules/buffile/Makefile create mode 100644 src/test/modules/buffile/README create mode 100644 src/test/modules/buffile/buffile--1.0.sql create mode 100644 src/test/modules/buffile/buffile.c create mode 100644 src/test/modules/buffile/buffile.control create mode 100644 src/test/modules/buffile/expected/test_01.out create mode 100644 src/test/modules/buffile/expected/test_02.out create mode 100644 src/test/modules/buffile/expected/test_03.out create mode 100644 src/test/modules/buffile/expected/test_04.out create mode 100644 src/test/modules/buffile/expected/test_05.out create mode 100644 src/test/modules/buffile/expected/test_06.out create mode 100644 src/test/modules/buffile/expected/test_07.out create mode 100644 src/test/modules/buffile/expected/test_08.out create mode 100644 src/test/modules/buffile/expected/test_09.out create mode 100644 src/test/modules/buffile/expected/test_10.out create mode 100644 src/test/modules/buffile/expected/test_11.out create mode 100644 src/test/modules/buffile/expected/test_12.out create mode 100644 src/test/modules/buffile/expected/test_13.out create mode 100644 src/test/modules/buffile/sql/test_01.sql create mode 100644 src/test/modules/buffile/sql/test_02.sql create mode 100644 src/test/modules/buffile/sql/test_03.sql create mode 100644 src/test/modules/buffile/sql/test_04.sql create mode 100644 src/test/modules/buffile/sql/test_05.sql create mode 100644 src/test/modules/buffile/sql/test_06.sql create mode 100644 src/test/modules/buffile/sql/test_07.sql create mode 100644 src/test/modules/buffile/sql/test_08.sql create mode 100644 src/test/modules/buffile/sql/test_09.sql create mode 100644 src/test/modules/buffile/sql/test_10.sql create mode 100644 src/test/modules/buffile/sql/test_11.sql create mode 100644 src/test/modules/buffile/sql/test_12.sql create mode 100644 src/test/modules/buffile/sql/test_13.sql diff --git a/src/backend/storage/file/buffile.c b/src/backend/storage/file/buffile.c index 308ceb01b7..998dabd134 100644 --- a/src/backend/storage/file/buffile.c +++ b/src/backend/storage/file/buffile.c @@ -56,16 +56,11 @@ #include "utils/resowner.h" /* - * We break BufFiles into gigabyte-sized segments, regardless of RELSEG_SIZE. - * The reason is that we'd like large BufFiles to be spread across multiple - * tablespaces when available. - * - * An integer value indicating the number of useful bytes in the segment is - * appended to each segment of if the file is both shared and encrypted, see - * BufFile.useful. + * The functions bellow actually use integer constants so that the size can be + * controlled by GUC. This is useful for development and regression tests. */ -#define MAX_PHYSICAL_FILESIZE 0x40000000 -#define BUFFILE_SEG_SIZE (MAX_PHYSICAL_FILESIZE / BLCKSZ) +int buffile_max_filesize = MAX_PHYSICAL_FILESIZE; +int buffile_seg_blocks = BUFFILE_SEG_BLOCKS(MAX_PHYSICAL_FILESIZE); /* * Fields that both BufFile and TransientBufFile structures need. It must be @@ -123,7 +118,7 @@ struct BufFile BufFileCommon common; /* Common fields, see above. */ int numFiles; /* number of physical files in set */ - /* all files except the last have length exactly MAX_PHYSICAL_FILESIZE */ + /* all files except the last have length exactly buffile_max_filesize */ File *files; /* palloc'd array with numFiles entries */ /* @@ -291,7 +286,7 @@ extendBufFile(BufFile *file) /* * Create a BufFile for a new temporary file (which will expand to become - * multiple temporary files if more than MAX_PHYSICAL_FILESIZE bytes are + * multiple temporary files if more than buffile_max_filesize bytes are * written to it). * * If interXact is true, the temp file will not be automatically deleted @@ -595,7 +590,7 @@ BufFileLoadBuffer(BufFile *file) File thisfile; /* - * Only whole multiple of ENCRYPTION_BLOCK can be encrypted / decrypted, + * Only whole multiple of ENCRYPTION_BLOCK can be encrypted / decrypted. */ Assert((file->common.curOffset % BLCKSZ == 0 && file->common.curOffset % ENCRYPTION_BLOCK == 0) || @@ -604,7 +599,7 @@ BufFileLoadBuffer(BufFile *file) /* * Advance to next component file if necessary and possible. */ - if (file->common.curOffset >= MAX_PHYSICAL_FILESIZE && + if (file->common.curOffset >= buffile_max_filesize && file->common.curFile + 1 < file->numFiles) { file->common.curFile++; @@ -715,7 +710,7 @@ BufFileDumpBuffer(BufFile *file) /* * Advance to next component file if necessary and possible. */ - if (file->common.curOffset >= MAX_PHYSICAL_FILESIZE) + if (file->common.curOffset >= buffile_max_filesize) { while (file->common.curFile + 1 >= file->numFiles) extendBufFile(file); @@ -727,7 +722,7 @@ BufFileDumpBuffer(BufFile *file) * Determine how much we need to write into this file. */ bytestowrite = file->common.nbytes - wpos; - availbytes = MAX_PHYSICAL_FILESIZE - file->common.curOffset; + availbytes = buffile_max_filesize - file->common.curOffset; if ((off_t) bytestowrite > availbytes) bytestowrite = (int) availbytes; @@ -758,7 +753,7 @@ BufFileDumpBuffer(BufFile *file) { file->common.curFile--; Assert(file->common.curFile >= 0); - file->common.curOffset += MAX_PHYSICAL_FILESIZE; + file->common.curOffset += buffile_max_filesize; } /* @@ -796,7 +791,7 @@ BufFileDumpBufferEncrypted(BufFile *file) /* * Advance to next component file if necessary and possible. */ - if (file->common.curOffset >= MAX_PHYSICAL_FILESIZE) + if (file->common.curOffset >= buffile_max_filesize) { while (file->common.curFile + 1 >= file->numFiles) extendBufFile(file); @@ -809,15 +804,14 @@ BufFileDumpBufferEncrypted(BufFile *file) * above) ensure that the encrypted buffer never crosses component file * boundary. */ - StaticAssertStmt((MAX_PHYSICAL_FILESIZE % BLCKSZ) == 0, - "BLCKSZ is not whole multiple of MAX_PHYSICAL_FILESIZE"); + Assert((buffile_max_filesize % BLCKSZ) == 0); /* * Encrypted data is dumped all at once. * * Unlike BufFileDumpBuffer(), we don't have to check here how much bytes * is available in the segment. According to the assertions above, - * currOffset should be lower than MAX_PHYSICAL_FILESIZE by non-zero + * currOffset should be lower than buffile_max_filesize by non-zero * multiple of BLCKSZ. */ bytestowrite = BLCKSZ; @@ -1021,7 +1015,7 @@ BufFileSeek(BufFile *file, int fileno, off_t offset, int whence) { if (--newFile < 0) return EOF; - newOffset += MAX_PHYSICAL_FILESIZE; + newOffset += buffile_max_filesize; } if (newFile == file->common.curFile && newOffset >= file->common.curOffset && @@ -1050,13 +1044,13 @@ BufFileSeek(BufFile *file, int fileno, off_t offset, int whence) if (newFile == file->numFiles && newOffset == 0) { newFile--; - newOffset = MAX_PHYSICAL_FILESIZE; + newOffset = buffile_max_filesize; } - while (newOffset > MAX_PHYSICAL_FILESIZE) + while (newOffset > buffile_max_filesize) { if (++newFile >= file->numFiles) return EOF; - newOffset -= MAX_PHYSICAL_FILESIZE; + newOffset -= buffile_max_filesize; } if (newFile >= file->numFiles) return EOF; @@ -1124,8 +1118,8 @@ int BufFileSeekBlock(BufFile *file, long blknum) { return BufFileSeek(file, - (int) (blknum / BUFFILE_SEG_SIZE), - (off_t) (blknum % BUFFILE_SEG_SIZE) * BLCKSZ, + (int) (blknum / buffile_seg_blocks), + (off_t) (blknum % buffile_seg_blocks) * BLCKSZ, SEEK_SET); } @@ -1166,7 +1160,7 @@ BufFileTweak(char *tweak, BufFileCommon *file, bool is_transient) if (tmpfile->segnos) curFile = tmpfile->segnos[curFile]; - block = curFile * BUFFILE_SEG_SIZE + file->curOffset / BLCKSZ; + block = curFile * buffile_seg_blocks + file->curOffset / BLCKSZ; StaticAssertStmt(sizeof(pid) + sizeof(number) + sizeof(block) <= TWEAK_SIZE, @@ -1258,7 +1252,7 @@ BufFileTellBlock(BufFile *file) long blknum; blknum = (file->common.curOffset + file->common.pos) / BLCKSZ; - blknum += file->common.curFile * BUFFILE_SEG_SIZE; + blknum += file->common.curFile * buffile_seg_blocks; return blknum; } @@ -1304,7 +1298,7 @@ BufFileSize(BufFile *file) lastFileSize = file->common.useful[file->common.nuseful - 1]; } - return ((file->numFiles - 1) * (int64) MAX_PHYSICAL_FILESIZE) + + return ((file->numFiles - 1) * (int64) buffile_max_filesize) + lastFileSize; } @@ -1317,11 +1311,10 @@ BufFileSize(BufFile *file) * called here first. Resource owners for source and target must match, * too. * - * This operation works by manipulating lists of segment files, so the - * file content is always appended at a MAX_PHYSICAL_FILESIZE-aligned - * boundary, typically creating empty holes before the boundary. These - * areas do not contain any interesting data, and cannot be read from by - * caller. + * This operation works by manipulating lists of segment files, so the file + * content is always appended at a buffile_max_filesize-aligned boundary, + * typically creating empty holes before the boundary. These areas do not + * contain any interesting data, and cannot be read from by caller. * * Returns the block number within target where the contents of source * begins. Caller should apply this as an offset when working off block @@ -1330,7 +1323,7 @@ BufFileSize(BufFile *file) long BufFileAppend(BufFile *target, BufFile *source) { - long startBlock = target->numFiles * BUFFILE_SEG_SIZE; + long startBlock = target->numFiles * buffile_seg_blocks; int newNumFiles = target->numFiles + source->numFiles; int i; @@ -2084,14 +2077,14 @@ BufFileUpdateUsefulLength(BufFileCommon *file, bool is_transient) * w/o dumping anything of it. While curFile will be fixed during the * next dump, we need valid fileno now. */ - if (file->curOffset >= MAX_PHYSICAL_FILESIZE) + if (file->curOffset >= buffile_max_filesize) { /* * Even BufFileSeek() should not allow curOffset to become more - * than MAX_PHYSICAL_FILESIZE (if caller passes higher offset, + * than buffile_max_filesize (if caller passes higher offset, * curFile gets increased instead). */ - Assert(file->curOffset == MAX_PHYSICAL_FILESIZE); + Assert(file->curOffset == buffile_max_filesize); fileno++; } @@ -2114,8 +2107,8 @@ BufFileUpdateUsefulLength(BufFileCommon *file, bool is_transient) * new_useful is also relative to the start of that previous * segment. Make sure it's relative to the current (fileno) segment. */ - if (file->curOffset % MAX_PHYSICAL_FILESIZE == 0) - new_useful %= MAX_PHYSICAL_FILESIZE; + if (file->curOffset % buffile_max_filesize == 0) + new_useful %= buffile_max_filesize; /* Finalize the offset. */ new_useful += file->pos; diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c index 440120635e..879c8fd816 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -70,9 +70,10 @@ #include "replication/syncrep.h" #include "replication/walreceiver.h" #include "replication/walsender.h" +#include "storage/buffile.h" #include "storage/bufmgr.h" -#include "storage/encryption.h" #include "storage/dsm_impl.h" +#include "storage/encryption.h" #include "storage/standby.h" #include "storage/fd.h" #include "storage/large_object.h" @@ -189,6 +190,8 @@ static const char *show_tcp_keepalives_idle(void); static const char *show_tcp_keepalives_interval(void); static const char *show_tcp_keepalives_count(void); static const char *show_tcp_user_timeout(void); +static bool check_buffile_max_filesize(int *newval, void **extra, GucSource source); +static void assign_buffile_max_filesize(int newval, void *extra); static bool check_maxconnections(int *newval, void **extra, GucSource source); static bool check_max_worker_processes(int *newval, void **extra, GucSource source); static bool check_autovacuum_max_workers(int *newval, void **extra, GucSource source); @@ -3209,6 +3212,18 @@ static struct config_int ConfigureNamesInt[] = NULL, assign_tcp_user_timeout, show_tcp_user_timeout }, + { + /* Not for general use */ + {"buffile_max_filesize", PGC_SUSET, DEVELOPER_OPTIONS, + gettext_noop("Maximum size of BufFile segment."), + gettext_noop("This makes testing of some corner cases easier, especially when read or write crosses segment boundary."), + GUC_NOT_IN_SAMPLE | GUC_UNIT_BYTE + }, + &buffile_max_filesize, + MAX_PHYSICAL_FILESIZE, BLCKSZ, MAX_PHYSICAL_FILESIZE, + check_buffile_max_filesize, assign_buffile_max_filesize, NULL + }, + /* End-of-list marker */ { {NULL, 0, 0, NULL, NULL}, NULL, 0, 0, 0, NULL, NULL, NULL @@ -11301,6 +11316,23 @@ show_tcp_user_timeout(void) } static bool +check_buffile_max_filesize(int *newval, void **extra, GucSource source) +{ + if (*newval % BLCKSZ != 0) + { + GUC_check_errdetail("The value must be whole multiple of BLCKSZ."); + return false; + } + return true; +} + +static void +assign_buffile_max_filesize(int newval, void *extra) +{ + buffile_seg_blocks = BUFFILE_SEG_BLOCKS(newval); +} + +static bool check_maxconnections(int *newval, void **extra, GucSource source) { if (*newval + autovacuum_max_workers + 1 + diff --git a/src/include/storage/buffile.h b/src/include/storage/buffile.h index 625fa0014c..977942f4b3 100644 --- a/src/include/storage/buffile.h +++ b/src/include/storage/buffile.h @@ -37,6 +37,25 @@ typedef struct BufFile BufFile; typedef struct TransientBufFile TransientBufFile; /* + * We break BufFiles into gigabyte-sized segments, regardless of RELSEG_SIZE. + * The reason is that we'd like large BufFiles to be spread across multiple + * tablespaces when available. + * + * An integer value indicating the number of useful bytes in the segment is + * appended to each segment of if the file is both shared and encrypted, see + * BufFile.useful. + */ +#define MAX_PHYSICAL_FILESIZE 0x40000000 + +/* Express segment size in the number of blocks. */ +#define BUFFILE_SEG_BLOCKS(phys) ((phys) / BLCKSZ) + +/* GUC to control size of the file segment. */ +extern int buffile_max_filesize; +/* Segment size in blocks, derived from the above. */ +extern int buffile_seg_blocks; + +/* * prototypes for functions in buffile.c */ diff --git a/src/test/modules/buffile/Makefile b/src/test/modules/buffile/Makefile new file mode 100644 index 0000000000..f0f163c45e --- /dev/null +++ b/src/test/modules/buffile/Makefile @@ -0,0 +1,21 @@ +PG_CONFIG ?= pg_config +MODULE_big = buffile_test +OBJS = buffile.o $(WIN32RES) +PGFILEDESC = "buffile_test" + +EXTENSION = buffile +DATA = buffile--1.0.sql + +REGRESS = test_01 test_02 test_03 test_04 test_05 test_06 test_07 test_08 \ +test_09 test_10 test_11 test_12 test_13 + +ifdef USE_PGXS +PG_CONFIG = pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) +else +subdir = src/test/modules/buffile +top_builddir = ../../../.. +include $(top_builddir)/src/Makefile.global +include $(top_srcdir)/contrib/contrib-global.mk +endif diff --git a/src/test/modules/buffile/README b/src/test/modules/buffile/README new file mode 100644 index 0000000000..9c80787797 --- /dev/null +++ b/src/test/modules/buffile/README @@ -0,0 +1,11 @@ +This extension was written to check if changes introduced due to cluster +encryption do not break buffile.c. + +Caution: To make the test cheaper, it was decided to adjust the segment sizeq, +see + +#define MAX_PHYSICAL_FILESIZE (4 * BLCKSZ) + +in buffile.c. BLCKSZ is 8192 (the default). All the tests rely on this +value. So if you haven't compiled Postgres with this value, the tests will +create 1 GB files and they will fail. diff --git a/src/test/modules/buffile/buffile--1.0.sql b/src/test/modules/buffile/buffile--1.0.sql new file mode 100644 index 0000000000..7c168f3514 --- /dev/null +++ b/src/test/modules/buffile/buffile--1.0.sql @@ -0,0 +1,54 @@ +CREATE FUNCTION buffile_create() +RETURNS void +AS 'MODULE_PATHNAME', 'buffile_create' +LANGUAGE C; + +CREATE FUNCTION buffile_close() +RETURNS void +AS 'MODULE_PATHNAME', 'buffile_close' +LANGUAGE C; + +CREATE FUNCTION buffile_write(text) +RETURNS bigint +AS 'MODULE_PATHNAME', 'buffile_write' +LANGUAGE C; + +CREATE FUNCTION buffile_read(bigint) +RETURNS bytea +AS 'MODULE_PATHNAME', 'buffile_read' +LANGUAGE C; + +CREATE FUNCTION buffile_seek(int, bigint) +RETURNS int +AS 'MODULE_PATHNAME', 'buffile_seek' +LANGUAGE C; + +CREATE FUNCTION buffile_assert_fileno(int) +RETURNS void +AS 'MODULE_PATHNAME', 'buffile_assert_fileno' +LANGUAGE C; + +CREATE FUNCTION buffile_test_shared() +RETURNS void +AS 'MODULE_PATHNAME', 'buffile_test_shared' +LANGUAGE C; + +CREATE FUNCTION buffile_test_shared_append() +RETURNS void +AS 'MODULE_PATHNAME', 'buffile_test_shared_append' +LANGUAGE C; + +CREATE FUNCTION buffile_open_transient(text, bool, bool) +RETURNS void +AS 'MODULE_PATHNAME', 'buffile_open_transient' +LANGUAGE C; + +CREATE FUNCTION buffile_close_transient() +RETURNS void +AS 'MODULE_PATHNAME', 'buffile_close_transient' +LANGUAGE C; + +CREATE FUNCTION buffile_delete_file(text) +RETURNS void +AS 'MODULE_PATHNAME', 'buffile_delete_file' +LANGUAGE C; diff --git a/src/test/modules/buffile/buffile.c b/src/test/modules/buffile/buffile.c new file mode 100644 index 0000000000..fe0680771a --- /dev/null +++ b/src/test/modules/buffile/buffile.c @@ -0,0 +1,429 @@ +#include + +#include "postgres.h" +#include "fmgr.h" +#include "lib/stringinfo.h" +#include "storage/buffile.h" +#include "utils/array.h" +#include "utils/builtins.h" +#include "utils/lsyscache.h" +#include "utils/memutils.h" +#include "utils/resowner.h" + +PG_MODULE_MAGIC; + +/* + * To cope with files that span multiple segments w/o wasting resources, use + * the smallest possible segment size. The test scripts need to set + * buffile_max_filesize (GUC) accordingly. + */ +#define MAX_PHYSICAL_FILESIZE_TEST (4 * BLCKSZ) + +static BufFile *bf = NULL; +static TransientBufFile *bft = NULL; + +static void check_file(void); + +extern Datum buffile_create_transient(PG_FUNCTION_ARGS); + +PG_FUNCTION_INFO_V1(buffile_create); +Datum +buffile_create(PG_FUNCTION_ARGS) +{ + MemoryContext old_cxt; + ResourceOwner old_ro; + + if (bf != NULL) + elog(ERROR, "file already exists"); + + old_cxt = MemoryContextSwitchTo(TopMemoryContext); + + /* + * Make sure the file is not deleted across function calls. + */ + old_ro = CurrentResourceOwner; + CurrentResourceOwner = TopTransactionResourceOwner; + + bf = BufFileCreateTemp(false); + + CurrentResourceOwner = old_ro; + MemoryContextSwitchTo(old_cxt); + + PG_RETURN_VOID(); +} + +extern Datum buffile_close(PG_FUNCTION_ARGS); + +PG_FUNCTION_INFO_V1(buffile_close); +Datum +buffile_close(PG_FUNCTION_ARGS) +{ + if (bf == NULL) + elog(ERROR, "there's no file to close"); + + BufFileClose(bf); + bf = NULL; + + PG_RETURN_VOID(); +} + +extern Datum buffile_write(PG_FUNCTION_ARGS); + +PG_FUNCTION_INFO_V1(buffile_write); +Datum +buffile_write(PG_FUNCTION_ARGS) +{ + Datum d = PG_GETARG_DATUM(0); + char *s = TextDatumGetCString(d); + size_t res; + + if (bf) + res = BufFileWrite(bf, s, strlen(s)); + else if (bft) + res = BufFileWriteTransient(bft, s, strlen(s)); + else + elog(ERROR, "No file is open"); + + PG_RETURN_INT64(res); +} + +extern Datum buffile_read(PG_FUNCTION_ARGS); + +PG_FUNCTION_INFO_V1(buffile_read); +Datum +buffile_read(PG_FUNCTION_ARGS) +{ + int64 size = PG_GETARG_INT64(0); + StringInfo buf = makeStringInfo(); + size_t res_size; + bytea *result; + + enlargeStringInfo(buf, size); + + if (bf) + res_size = BufFileRead(bf, buf->data, size); + else if (bft) + res_size = BufFileReadTransient(bft, buf->data, size); + else + elog(ERROR, "No file is open"); + + buf->len = res_size; + + result = DatumGetByteaPP(DirectFunctionCall1(bytearecv, + PointerGetDatum(buf))); + PG_RETURN_BYTEA_P(result); +} + +extern Datum buffile_seek(PG_FUNCTION_ARGS); + +PG_FUNCTION_INFO_V1(buffile_seek); +Datum +buffile_seek(PG_FUNCTION_ARGS) +{ + int32 fileno = PG_GETARG_INT32(0); + int64 offset = PG_GETARG_INT64(1); + int32 res; + + check_file(); + res = BufFileSeek(bf, fileno, offset, SEEK_SET); + + PG_RETURN_INT32(res); +} + +extern Datum buffile_assert_fileno(PG_FUNCTION_ARGS); + +PG_FUNCTION_INFO_V1(buffile_assert_fileno); +Datum +buffile_assert_fileno(PG_FUNCTION_ARGS) +{ + int32 fileno_expected = PG_GETARG_INT32(0); + int32 fileno; + off_t offset; + + check_file(); + BufFileTell(bf, &fileno, &offset); + + if (fileno != fileno_expected) + { + /* + * Bring the backend down so that the following tests have no chance + * to create the 1GB files. + */ + elog(FATAL, "file number does not match"); + } + + PG_RETURN_VOID(); +} + +static void +check_file(void) +{ + if (bf == NULL) + elog(ERROR, "the file is not opened"); +} + +/* + * This test is especially important for shared encrypted files, see the + * comments below. + */ +extern Datum buffile_test_shared(PG_FUNCTION_ARGS); + +PG_FUNCTION_INFO_V1(buffile_test_shared); +Datum +buffile_test_shared(PG_FUNCTION_ARGS) +{ + dsm_segment *seg; + SharedFileSet *fileset; + BufFile *bf_1, + *bf_2; + char *data_1, + *data_2, + *data; + Size chunk_size_1, + chunk_size_2; + int fileno, + i; + off_t offset, + res, + total_size; + + /* + * The size is not important, we actually do not need the shared memory. + * The segment is only needed to initialize the fileset. + */ + seg = dsm_create(1024, 0); + + /* + * The fileset must survive error handling, so that dsm_detach works fine. + * (The typical use case is that the fileset is in shared memory.) + */ + fileset = (SharedFileSet *) MemoryContextAlloc(TopTransactionContext, + sizeof(SharedFileSet)); + SharedFileSetInit(fileset, seg); + + bf_1 = BufFileCreateShared(fileset, "file_1"); + + /* + * Write more data than the buffer size, so that we can check that the + * number of "useful bytes" word is only appended at the end of the + * segment, not after each buffer. + */ + chunk_size_1 = BLCKSZ + 256; + data_1 = (char *) palloc(chunk_size_1); + memset(data_1, 1, chunk_size_1); + if (BufFileWrite(bf_1, data_1, chunk_size_1) != chunk_size_1) + elog(ERROR, "Failed to write data"); + pfree(data_1); + + /* + * Enforce buffer flush (The BufFileFlush() function is not exported). + * Thus the "useful bytes" metadata should appear at the current end the + * first file segment. The next write will have to seek back to overwrite + * the metadata. + */ + BufFileTell(bf_1, &fileno, &offset); + if (BufFileSeek(bf_1, 0, 0, SEEK_SET) != 0) + elog(ERROR, "seek failed"); + if (BufFileSeek(bf_1, fileno, offset, SEEK_SET) != 0) + elog(ERROR, "seek failed"); + + /* + * Write another chunk that does not fit into the first segment file. Thus + * the "useful bytes" metadata should appear at the end of both segments. + */ + chunk_size_2 = 3 * BLCKSZ; + data_2 = (char *) palloc(chunk_size_2); + memset(data_2, 1, chunk_size_2); + if (BufFileWrite(bf_1, data_2, chunk_size_2) != chunk_size_2) + elog(ERROR, "Failed to write data"); + pfree(data_2); + BufFileClose(bf_1); + + /* + * The word indicating the number of "useful bytes" (i.e. the actual data + * w/o padding to buffer size) is stored at the end of each segment file. + * Check that this metadata is read correctly. + */ + bf_2 = BufFileOpenShared(fileset, "file_1"); + total_size = BufFileSize(bf_2); + if (total_size != (chunk_size_1 + chunk_size_2)) + elog(ERROR, "Incorrect file size: %zu", total_size); + + data = (char *) palloc(total_size); + res = BufFileRead(bf_2, data, total_size); + if (res != total_size) + elog(ERROR, "Incorrect chunk size read: %zu", res); + for (i = 0; i < total_size; i++) + if (data[i] != 1) + elog(ERROR, "Unexpected data read from the file"); + pfree(data); + BufFileClose(bf_2); + + dsm_detach(seg); + + PG_RETURN_VOID(); +} + + +/* + * Test BufFileAppend(). + */ +extern Datum buffile_test_shared_append(PG_FUNCTION_ARGS); + +PG_FUNCTION_INFO_V1(buffile_test_shared_append); +Datum +buffile_test_shared_append(PG_FUNCTION_ARGS) +{ + dsm_segment *seg; + SharedFileSet *fileset; + BufFile *bf_1, + *bf_2, + *bf_3; + char *data; + Size chunk_size; + int i; + off_t res, + total_size; + + seg = dsm_create(1024, 0); + + fileset = (SharedFileSet *) MemoryContextAlloc(TopTransactionContext, + sizeof(SharedFileSet)); + SharedFileSetInit(fileset, seg); + + /* + * XXX Does the chunk size matter much? + */ + chunk_size = 8; + data = (char *) palloc(chunk_size); + memset(data, 1, chunk_size); + + bf_1 = BufFileCreateShared(fileset, "file_1"); + if (BufFileWrite(bf_1, data, chunk_size) != chunk_size) + elog(ERROR, "Failed to write data"); + + bf_2 = BufFileCreateShared(fileset, "file_2"); + if (BufFileWrite(bf_2, data, chunk_size) != chunk_size) + elog(ERROR, "Failed to write data"); + + /* + * Make sure it's read-only so that BufFileAppend() can accept it as + * source. + */ + BufFileClose(bf_2); + bf_2 = BufFileOpenShared(fileset, "file_2"); + + bf_3 = BufFileCreateShared(fileset, "file_3"); + if (BufFileWrite(bf_3, data, chunk_size) != chunk_size) + elog(ERROR, "Failed to write data"); + BufFileClose(bf_3); + bf_3 = BufFileOpenShared(fileset, "file_3"); + + BufFileAppend(bf_1, bf_2); + BufFileAppend(bf_1, bf_3); + + total_size = BufFileSize(bf_1); + + /* + * The result should contain complete segments of bf_1 and bf_2 and the + * valid part of bf_3. + */ + if (total_size != (2 * MAX_PHYSICAL_FILESIZE_TEST + chunk_size)) + elog(ERROR, "Incorrect total size of the appended data: %zu", + total_size); + + /* + * Check that data of the 2nd segment was decrypted correctly. + */ + if (BufFileSeek(bf_1, 1, 0, SEEK_SET) != 0) + elog(ERROR, "seek failed"); + res = BufFileRead(bf_1, data, chunk_size); + if (res != chunk_size) + elog(ERROR, "Incorrect chunk size read: %zu", res); + for (i = 0; i < chunk_size; i++) + if (data[i] != 1) + elog(ERROR, "Unexpected data read from the file"); + + /* + * And the same for the 3rd segment. + * + * TODO Reuse the code above by putting it into a function. + */ + if (BufFileSeek(bf_1, 2, 0, SEEK_SET) != 0) + elog(ERROR, "seek failed"); + res = BufFileRead(bf_1, data, chunk_size); + if (res != chunk_size) + elog(ERROR, "Incorrect chunk size read: %zu", res); + for (i = 0; i < chunk_size; i++) + if (data[i] != 1) + elog(ERROR, "Unexpected data read from the file"); + + BufFileClose(bf_1); + dsm_detach(seg); + PG_RETURN_VOID(); +} + +extern Datum buffile_open_transient(PG_FUNCTION_ARGS); + +PG_FUNCTION_INFO_V1(buffile_open_transient); +Datum +buffile_open_transient(PG_FUNCTION_ARGS) +{ + MemoryContext old_cxt; + Datum d = PG_GETARG_DATUM(0); + char *path = TextDatumGetCString(d); + bool write_only = PG_GETARG_BOOL(1); + bool append = PG_GETARG_BOOL(2); + int flags = O_CREAT | PG_BINARY; + + if (bft != NULL) + elog(ERROR, "file already exists"); + + if (write_only) + flags |= O_WRONLY; + if (append) + flags |= O_APPEND; + + old_cxt = MemoryContextSwitchTo(TopMemoryContext); + + /* + * Make sure the file is not deleted across function calls. + */ + bft = BufFileOpenTransient(path, flags); + + MemoryContextSwitchTo(old_cxt); + + PG_RETURN_VOID(); +} + +extern Datum buffile_close_transient(PG_FUNCTION_ARGS); + +PG_FUNCTION_INFO_V1(buffile_close_transient); +Datum +buffile_close_transient(PG_FUNCTION_ARGS) +{ + if (bft == NULL) + elog(ERROR, "there's no file to close"); + + BufFileCloseTransient(bft); + bft = NULL; + + PG_RETURN_VOID(); +} + +extern Datum buffile_delete_file(PG_FUNCTION_ARGS); + +PG_FUNCTION_INFO_V1(buffile_delete_file); +Datum +buffile_delete_file(PG_FUNCTION_ARGS) +{ + Datum d = PG_GETARG_DATUM(0); + char *path = TextDatumGetCString(d); + + if (bft != NULL) + elog(ERROR, "the file is still open"); + + PathNameDeleteTemporaryFile(path, true); + + PG_RETURN_VOID(); +} diff --git a/src/test/modules/buffile/buffile.control b/src/test/modules/buffile/buffile.control new file mode 100644 index 0000000000..8472c5a348 --- /dev/null +++ b/src/test/modules/buffile/buffile.control @@ -0,0 +1,5 @@ +# buffile_test extension +comment = 'buffile_test' +default_version = '1.0' +module_pathname = '$libdir/buffile_test' +relocatable = true diff --git a/src/test/modules/buffile/expected/test_01.out b/src/test/modules/buffile/expected/test_01.out new file mode 100644 index 0000000000..c1a4f5ac7b --- /dev/null +++ b/src/test/modules/buffile/expected/test_01.out @@ -0,0 +1,61 @@ +-- This is the first test, so make sure the test extension is there. +CREATE EXTENSION IF NOT EXISTS buffile; +BEGIN; +SELECT buffile_create(); + buffile_create +---------------- + +(1 row) + +SELECT buffile_seek(0, 1); + buffile_seek +-------------- + 0 +(1 row) + +SELECT buffile_write('abc'); + buffile_write +--------------- + 3 +(1 row) + +SELECT buffile_seek(0, 0); + buffile_seek +-------------- + 0 +(1 row) + +-- Check that the trailing zeroes are not fetched. +SELECT buffile_read(16); + buffile_read +-------------- + \x00616263 +(1 row) + +-- Adjust the number of useful bytes. +SELECT buffile_write('abc'); + buffile_write +--------------- + 3 +(1 row) + +-- ... and check again. +SELECT buffile_seek(0, 0); + buffile_seek +-------------- + 0 +(1 row) + +SELECT buffile_read(16); + buffile_read +------------------ + \x00616263616263 +(1 row) + +SELECT buffile_close(); + buffile_close +--------------- + +(1 row) + +COMMIT; diff --git a/src/test/modules/buffile/expected/test_02.out b/src/test/modules/buffile/expected/test_02.out new file mode 100644 index 0000000000..f783d0cb24 --- /dev/null +++ b/src/test/modules/buffile/expected/test_02.out @@ -0,0 +1,48 @@ +BEGIN; +SELECT buffile_create(); + buffile_create +---------------- + +(1 row) + +SELECT buffile_seek(0, 8189); + buffile_seek +-------------- + 0 +(1 row) + +-- Initialize the last 3 positions of the first buffer and the initial 3 +-- positions of the 2nd buffer. +SELECT buffile_write('abcdef'); + buffile_write +--------------- + 6 +(1 row) + +SELECT buffile_seek(0, 0); + buffile_seek +-------------- + 0 +(1 row) + +-- Read the first buffer. +SELECT length(buffile_read(8192)); + length +-------- + 8192 +(1 row) + +-- Only 3 bytes of the 2nd buffer should be fetched. +SELECT length(buffile_read(8192)); + length +-------- + 3 +(1 row) + +SELECT buffile_close(); + buffile_close +--------------- + +(1 row) + +COMMIT; diff --git a/src/test/modules/buffile/expected/test_03.out b/src/test/modules/buffile/expected/test_03.out new file mode 100644 index 0000000000..e899fa3b38 --- /dev/null +++ b/src/test/modules/buffile/expected/test_03.out @@ -0,0 +1,27 @@ +BEGIN; +SELECT buffile_create(); + buffile_create +---------------- + +(1 row) + +-- Read from an empty file. +SELECT buffile_seek(0, 8); + buffile_seek +-------------- + 0 +(1 row) + +SELECT buffile_read(16); + buffile_read +-------------- + \x +(1 row) + +SELECT buffile_close(); + buffile_close +--------------- + +(1 row) + +COMMIT; diff --git a/src/test/modules/buffile/expected/test_04.out b/src/test/modules/buffile/expected/test_04.out new file mode 100644 index 0000000000..1f8eeabe48 --- /dev/null +++ b/src/test/modules/buffile/expected/test_04.out @@ -0,0 +1,84 @@ +BEGIN; +SELECT buffile_create(); + buffile_create +---------------- + +(1 row) + +-- Write something near the end of the first buffer, but leave some trailing +-- space. +SELECT buffile_seek(0, 8184); + buffile_seek +-------------- + 0 +(1 row) + +SELECT buffile_write('abcd'); + buffile_write +--------------- + 4 +(1 row) + +-- Leave the 2nd buffer empty, as well as a few leading bytes. Thus we should +-- get a hole that spans the whole 2nd buffer as well as a few adjacent bytes +-- on each side. +SELECT buffile_seek(0, 2 * 8192 + 4); + buffile_seek +-------------- + 0 +(1 row) + +SELECT buffile_write('efgh'); + buffile_write +--------------- + 4 +(1 row) + +-- Check the initial part of the hole, which crosses the boundary of the 1st +-- and the 2nd buffer. +SELECT buffile_seek(0, 8184); + buffile_seek +-------------- + 0 +(1 row) + +SELECT buffile_read(16); + buffile_read +------------------------------------ + \x61626364000000000000000000000000 +(1 row) + +-- Check the trailing part of the whole, which crosses the boundary of the 2nd +-- and the 3rd buffer. +SELECT buffile_seek(0, 2 * 8192 - 8); + buffile_seek +-------------- + 0 +(1 row) + +SELECT buffile_read(16); + buffile_read +------------------------------------ + \x00000000000000000000000065666768 +(1 row) + +-- Check that the hole contains nothing but zeroes. +SELECT buffile_seek(0, 8192 - 4); + buffile_seek +-------------- + 0 +(1 row) + +SELECT btrim(buffile_read(8192 + 8), '\x00'); + btrim +------- + \x +(1 row) + +SELECT buffile_close(); + buffile_close +--------------- + +(1 row) + +COMMIT; diff --git a/src/test/modules/buffile/expected/test_05.out b/src/test/modules/buffile/expected/test_05.out new file mode 100644 index 0000000000..6a73147711 --- /dev/null +++ b/src/test/modules/buffile/expected/test_05.out @@ -0,0 +1,33 @@ +BEGIN; +SELECT buffile_create(); + buffile_create +---------------- + +(1 row) + +-- Seek does not extend the file if it's not followed by write. +SELECT buffile_seek(0, 1); + buffile_seek +-------------- + 0 +(1 row) + +SELECT buffile_seek(0, 0); + buffile_seek +-------------- + 0 +(1 row) + +SELECT buffile_read(2); + buffile_read +-------------- + \x +(1 row) + +SELECT buffile_close(); + buffile_close +--------------- + +(1 row) + +COMMIT; diff --git a/src/test/modules/buffile/expected/test_06.out b/src/test/modules/buffile/expected/test_06.out new file mode 100644 index 0000000000..d406bda9e6 --- /dev/null +++ b/src/test/modules/buffile/expected/test_06.out @@ -0,0 +1,43 @@ +-- This test shows that the if first component file (segment) stays empty, +-- read stops prematurely even if it starts within that segment, although it'd +-- otherwise receive some data from the following one. +BEGIN; +-- Neither disk space nor time needs to be wasted. +SET buffile_max_filesize TO 8192; +SELECT buffile_create(); + buffile_create +---------------- + +(1 row) + +SELECT buffile_seek(0, 8192); + buffile_seek +-------------- + 0 +(1 row) + +SELECT buffile_write('a'); + buffile_write +--------------- + 1 +(1 row) + +SELECT buffile_seek(0, 8191); + buffile_seek +-------------- + 0 +(1 row) + +SELECT buffile_read(2); + buffile_read +-------------- + \x +(1 row) + +SELECT buffile_close(); + buffile_close +--------------- + +(1 row) + +COMMIT; diff --git a/src/test/modules/buffile/expected/test_07.out b/src/test/modules/buffile/expected/test_07.out new file mode 100644 index 0000000000..6f2ee3d2d9 --- /dev/null +++ b/src/test/modules/buffile/expected/test_07.out @@ -0,0 +1,41 @@ +BEGIN; +-- Use a small segment, not to waste disk space and time. +SET buffile_max_filesize TO 8192; +SELECT buffile_create(); + buffile_create +---------------- + +(1 row) + +-- Write data at component file boundary and try to read it. +SELECT buffile_seek(0, 8192); + buffile_seek +-------------- + 0 +(1 row) + +SELECT buffile_write('abcd'); + buffile_write +--------------- + 4 +(1 row) + +SELECT buffile_seek(0, 8192); + buffile_seek +-------------- + 0 +(1 row) + +SELECT buffile_read(8); + buffile_read +-------------- + \x61626364 +(1 row) + +SELECT buffile_close(); + buffile_close +--------------- + +(1 row) + +COMMIT; diff --git a/src/test/modules/buffile/expected/test_08.out b/src/test/modules/buffile/expected/test_08.out new file mode 100644 index 0000000000..6ffcb9dcd6 --- /dev/null +++ b/src/test/modules/buffile/expected/test_08.out @@ -0,0 +1,41 @@ +BEGIN; +SELECT buffile_create(); + buffile_create +---------------- + +(1 row) + +-- Neither disk space nor time needs to be wasted. +SET buffile_max_filesize TO 8192; +-- Write data across component file boundary and try to read it. +SELECT buffile_seek(0, 8190); + buffile_seek +-------------- + 0 +(1 row) + +SELECT buffile_write('abcd'); + buffile_write +--------------- + 4 +(1 row) + +SELECT buffile_seek(0, 8190); + buffile_seek +-------------- + 0 +(1 row) + +SELECT buffile_read(8); + buffile_read +-------------- + \x61626364 +(1 row) + +SELECT buffile_close(); + buffile_close +--------------- + +(1 row) + +COMMIT; diff --git a/src/test/modules/buffile/expected/test_09.out b/src/test/modules/buffile/expected/test_09.out new file mode 100644 index 0000000000..b9d325b676 --- /dev/null +++ b/src/test/modules/buffile/expected/test_09.out @@ -0,0 +1,39 @@ +BEGIN; +SELECT buffile_create(); + buffile_create +---------------- + +(1 row) + +-- Write data across buffer boundary and try to read it. +SELECT buffile_seek(0, 8190); + buffile_seek +-------------- + 0 +(1 row) + +SELECT buffile_write('abcd'); + buffile_write +--------------- + 4 +(1 row) + +SELECT buffile_seek(0, 8190); + buffile_seek +-------------- + 0 +(1 row) + +SELECT buffile_read(8); + buffile_read +-------------- + \x61626364 +(1 row) + +SELECT buffile_close(); + buffile_close +--------------- + +(1 row) + +COMMIT; diff --git a/src/test/modules/buffile/expected/test_10.out b/src/test/modules/buffile/expected/test_10.out new file mode 100644 index 0000000000..8e457fdcda --- /dev/null +++ b/src/test/modules/buffile/expected/test_10.out @@ -0,0 +1,76 @@ +BEGIN; +SELECT buffile_create(); + buffile_create +---------------- + +(1 row) + +-- Write some data at the end of the buffer. +SELECT buffile_seek(0, 8188); + buffile_seek +-------------- + 0 +(1 row) + +SELECT buffile_write('abcd'); + buffile_write +--------------- + 4 +(1 row) + +SELECT buffile_seek(0, 8189); + buffile_seek +-------------- + 0 +(1 row) + +-- Enforce flush with the write position not at the end of the buffer. This is +-- special by not moving curOffset to the next buffer. +SELECT buffile_read(1); + buffile_read +-------------- + \x62 +(1 row) + +-- Therefore the next writes should eventually affect the original data. (Here +-- we also test going directly from read to write and vice versa.) +SELECT buffile_write('x'); + buffile_write +--------------- + 1 +(1 row) + +SELECT buffile_read(1); + buffile_read +-------------- + \x64 +(1 row) + +-- Start a new buffer, i.e. force flushing of the previous one. +SELECT buffile_write('z'); + buffile_write +--------------- + 1 +(1 row) + +-- Check that the 'x' and 'y' letters are in the first buffer, not in the +-- 2nd. (We read enough data to find any non-zero bytes in the 2nd buffer.) +SELECT buffile_seek(0, 8188); + buffile_seek +-------------- + 0 +(1 row) + +SELECT buffile_read(4 + 8192); + buffile_read +-------------- + \x616278647a +(1 row) + +SELECT buffile_close(); + buffile_close +--------------- + +(1 row) + +COMMIT; diff --git a/src/test/modules/buffile/expected/test_11.out b/src/test/modules/buffile/expected/test_11.out new file mode 100644 index 0000000000..c6804d08af --- /dev/null +++ b/src/test/modules/buffile/expected/test_11.out @@ -0,0 +1,34 @@ +BEGIN; +SELECT buffile_create(); + buffile_create +---------------- + +(1 row) + +SELECT buffile_write('abcd'); + buffile_write +--------------- + 4 +(1 row) + +-- Seek beyond EOF not followed by write. +SELECT buffile_seek(0, 5); + buffile_seek +-------------- + 0 +(1 row) + +-- Nothing should be fetched. +SELECT buffile_read(8); + buffile_read +-------------- + \x +(1 row) + +SELECT buffile_close(); + buffile_close +--------------- + +(1 row) + +COMMIT; diff --git a/src/test/modules/buffile/expected/test_12.out b/src/test/modules/buffile/expected/test_12.out new file mode 100644 index 0000000000..fe507a8e58 --- /dev/null +++ b/src/test/modules/buffile/expected/test_12.out @@ -0,0 +1,16 @@ +-- Neither disk space nor time needs to be wasted. +-- +-- Here the tests are designed for segments of multiple buffers. +SET buffile_max_filesize TO 32768; +SELECT buffile_test_shared(); + buffile_test_shared +--------------------- + +(1 row) + +SELECT buffile_test_shared_append(); + buffile_test_shared_append +---------------------------- + +(1 row) + diff --git a/src/test/modules/buffile/expected/test_13.out b/src/test/modules/buffile/expected/test_13.out new file mode 100644 index 0000000000..7ba50a9c9d --- /dev/null +++ b/src/test/modules/buffile/expected/test_13.out @@ -0,0 +1,86 @@ +-- Use transaction block so that the file does not closed automatically at +-- command boundary. +BEGIN; +SELECT buffile_open_transient('trans1', true, false); + buffile_open_transient +------------------------ + +(1 row) + +SELECT buffile_write('01234567'); + buffile_write +--------------- + 8 +(1 row) + +SELECT buffile_close_transient(); + buffile_close_transient +------------------------- + +(1 row) + +-- Open for reading. +SELECT buffile_open_transient('trans1', false, false); + buffile_open_transient +------------------------ + +(1 row) + +SELECT length(buffile_read(65536)); + length +-------- + 8 +(1 row) + +SELECT buffile_close_transient(); + buffile_close_transient +------------------------- + +(1 row) + +-- Open for writing in append mode. +SELECT buffile_open_transient('trans1', true, true); + buffile_open_transient +------------------------ + +(1 row) + +-- Add BLCKSZ bytes, so that buffer boundary is crossed. +SELECT buffile_write(repeat('x', 8192)); + buffile_write +--------------- + 8192 +(1 row) + +SELECT buffile_close_transient(); + buffile_close_transient +------------------------- + +(1 row) + +-- Open for reading and verify the valid part. +SELECT buffile_open_transient('trans1', false, false); + buffile_open_transient +------------------------ + +(1 row) + +SELECT length(buffile_read(65536)); + length +-------- + 8200 +(1 row) + +SELECT buffile_close_transient(); + buffile_close_transient +------------------------- + +(1 row) + +SELECT buffile_delete_file('trans1'); + buffile_delete_file +--------------------- + +(1 row) + +COMMIT; diff --git a/src/test/modules/buffile/sql/test_01.sql b/src/test/modules/buffile/sql/test_01.sql new file mode 100644 index 0000000000..92331dd0de --- /dev/null +++ b/src/test/modules/buffile/sql/test_01.sql @@ -0,0 +1,17 @@ +-- This is the first test, so make sure the test extension is there. +CREATE EXTENSION IF NOT EXISTS buffile; + +BEGIN; +SELECT buffile_create(); +SELECT buffile_seek(0, 1); +SELECT buffile_write('abc'); +SELECT buffile_seek(0, 0); +-- Check that the trailing zeroes are not fetched. +SELECT buffile_read(16); +-- Adjust the number of useful bytes. +SELECT buffile_write('abc'); +-- ... and check again. +SELECT buffile_seek(0, 0); +SELECT buffile_read(16); +SELECT buffile_close(); +COMMIT; diff --git a/src/test/modules/buffile/sql/test_02.sql b/src/test/modules/buffile/sql/test_02.sql new file mode 100644 index 0000000000..35d34722d7 --- /dev/null +++ b/src/test/modules/buffile/sql/test_02.sql @@ -0,0 +1,13 @@ +BEGIN; +SELECT buffile_create(); +SELECT buffile_seek(0, 8189); +-- Initialize the last 3 positions of the first buffer and the initial 3 +-- positions of the 2nd buffer. +SELECT buffile_write('abcdef'); +SELECT buffile_seek(0, 0); +-- Read the first buffer. +SELECT length(buffile_read(8192)); +-- Only 3 bytes of the 2nd buffer should be fetched. +SELECT length(buffile_read(8192)); +SELECT buffile_close(); +COMMIT; diff --git a/src/test/modules/buffile/sql/test_03.sql b/src/test/modules/buffile/sql/test_03.sql new file mode 100644 index 0000000000..a95391f7c3 --- /dev/null +++ b/src/test/modules/buffile/sql/test_03.sql @@ -0,0 +1,7 @@ +BEGIN; +SELECT buffile_create(); +-- Read from an empty file. +SELECT buffile_seek(0, 8); +SELECT buffile_read(16); +SELECT buffile_close(); +COMMIT; diff --git a/src/test/modules/buffile/sql/test_04.sql b/src/test/modules/buffile/sql/test_04.sql new file mode 100644 index 0000000000..64e8d39f94 --- /dev/null +++ b/src/test/modules/buffile/sql/test_04.sql @@ -0,0 +1,25 @@ +BEGIN; +SELECT buffile_create(); +-- Write something near the end of the first buffer, but leave some trailing +-- space. +SELECT buffile_seek(0, 8184); +SELECT buffile_write('abcd'); +-- Leave the 2nd buffer empty, as well as a few leading bytes. Thus we should +-- get a hole that spans the whole 2nd buffer as well as a few adjacent bytes +-- on each side. +SELECT buffile_seek(0, 2 * 8192 + 4); +SELECT buffile_write('efgh'); +-- Check the initial part of the hole, which crosses the boundary of the 1st +-- and the 2nd buffer. +SELECT buffile_seek(0, 8184); +SELECT buffile_read(16); +-- Check the trailing part of the whole, which crosses the boundary of the 2nd +-- and the 3rd buffer. +SELECT buffile_seek(0, 2 * 8192 - 8); +SELECT buffile_read(16); +-- Check that the hole contains nothing but zeroes. +SELECT buffile_seek(0, 8192 - 4); +SELECT btrim(buffile_read(8192 + 8), '\x00'); + +SELECT buffile_close(); +COMMIT; diff --git a/src/test/modules/buffile/sql/test_05.sql b/src/test/modules/buffile/sql/test_05.sql new file mode 100644 index 0000000000..5fd642e558 --- /dev/null +++ b/src/test/modules/buffile/sql/test_05.sql @@ -0,0 +1,8 @@ +BEGIN; +SELECT buffile_create(); +-- Seek does not extend the file if it's not followed by write. +SELECT buffile_seek(0, 1); +SELECT buffile_seek(0, 0); +SELECT buffile_read(2); +SELECT buffile_close(); +COMMIT; diff --git a/src/test/modules/buffile/sql/test_06.sql b/src/test/modules/buffile/sql/test_06.sql new file mode 100644 index 0000000000..35aa48857c --- /dev/null +++ b/src/test/modules/buffile/sql/test_06.sql @@ -0,0 +1,15 @@ +-- This test shows that the if first component file (segment) stays empty, +-- read stops prematurely even if it starts within that segment, although it'd +-- otherwise receive some data from the following one. +BEGIN; + +-- Neither disk space nor time needs to be wasted. +SET buffile_max_filesize TO 8192; + +SELECT buffile_create(); +SELECT buffile_seek(0, 8192); +SELECT buffile_write('a'); +SELECT buffile_seek(0, 8191); +SELECT buffile_read(2); +SELECT buffile_close(); +COMMIT; diff --git a/src/test/modules/buffile/sql/test_07.sql b/src/test/modules/buffile/sql/test_07.sql new file mode 100644 index 0000000000..06057c5a1b --- /dev/null +++ b/src/test/modules/buffile/sql/test_07.sql @@ -0,0 +1,13 @@ +BEGIN; + +-- Use a small segment, not to waste disk space and time. +SET buffile_max_filesize TO 8192; + +SELECT buffile_create(); +-- Write data at component file boundary and try to read it. +SELECT buffile_seek(0, 8192); +SELECT buffile_write('abcd'); +SELECT buffile_seek(0, 8192); +SELECT buffile_read(8); +SELECT buffile_close(); +COMMIT; diff --git a/src/test/modules/buffile/sql/test_08.sql b/src/test/modules/buffile/sql/test_08.sql new file mode 100644 index 0000000000..2736dc22df --- /dev/null +++ b/src/test/modules/buffile/sql/test_08.sql @@ -0,0 +1,13 @@ +BEGIN; +SELECT buffile_create(); + +-- Neither disk space nor time needs to be wasted. +SET buffile_max_filesize TO 8192; + +-- Write data across component file boundary and try to read it. +SELECT buffile_seek(0, 8190); +SELECT buffile_write('abcd'); +SELECT buffile_seek(0, 8190); +SELECT buffile_read(8); +SELECT buffile_close(); +COMMIT; diff --git a/src/test/modules/buffile/sql/test_09.sql b/src/test/modules/buffile/sql/test_09.sql new file mode 100644 index 0000000000..cc7060932e --- /dev/null +++ b/src/test/modules/buffile/sql/test_09.sql @@ -0,0 +1,9 @@ +BEGIN; +SELECT buffile_create(); +-- Write data across buffer boundary and try to read it. +SELECT buffile_seek(0, 8190); +SELECT buffile_write('abcd'); +SELECT buffile_seek(0, 8190); +SELECT buffile_read(8); +SELECT buffile_close(); +COMMIT; diff --git a/src/test/modules/buffile/sql/test_10.sql b/src/test/modules/buffile/sql/test_10.sql new file mode 100644 index 0000000000..63af760d9f --- /dev/null +++ b/src/test/modules/buffile/sql/test_10.sql @@ -0,0 +1,25 @@ +BEGIN; +SELECT buffile_create(); +-- Write some data at the end of the buffer. +SELECT buffile_seek(0, 8188); +SELECT buffile_write('abcd'); +SELECT buffile_seek(0, 8189); +-- Enforce flush with the write position not at the end of the buffer. This is +-- special by not moving curOffset to the next buffer. +SELECT buffile_read(1); + +-- Therefore the next writes should eventually affect the original data. (Here +-- we also test going directly from read to write and vice versa.) +SELECT buffile_write('x'); +SELECT buffile_read(1); + +-- Start a new buffer, i.e. force flushing of the previous one. +SELECT buffile_write('z'); + +-- Check that the 'x' and 'y' letters are in the first buffer, not in the +-- 2nd. (We read enough data to find any non-zero bytes in the 2nd buffer.) +SELECT buffile_seek(0, 8188); +SELECT buffile_read(4 + 8192); + +SELECT buffile_close(); +COMMIT; diff --git a/src/test/modules/buffile/sql/test_11.sql b/src/test/modules/buffile/sql/test_11.sql new file mode 100644 index 0000000000..94300d253f --- /dev/null +++ b/src/test/modules/buffile/sql/test_11.sql @@ -0,0 +1,9 @@ +BEGIN; +SELECT buffile_create(); +SELECT buffile_write('abcd'); +-- Seek beyond EOF not followed by write. +SELECT buffile_seek(0, 5); +-- Nothing should be fetched. +SELECT buffile_read(8); +SELECT buffile_close(); +COMMIT; diff --git a/src/test/modules/buffile/sql/test_12.sql b/src/test/modules/buffile/sql/test_12.sql new file mode 100644 index 0000000000..017050ef5b --- /dev/null +++ b/src/test/modules/buffile/sql/test_12.sql @@ -0,0 +1,7 @@ +-- Neither disk space nor time needs to be wasted. +-- +-- Here the tests are designed for segments of multiple buffers. +SET buffile_max_filesize TO 32768; + +SELECT buffile_test_shared(); +SELECT buffile_test_shared_append(); diff --git a/src/test/modules/buffile/sql/test_13.sql b/src/test/modules/buffile/sql/test_13.sql new file mode 100644 index 0000000000..bcabf7bfe6 --- /dev/null +++ b/src/test/modules/buffile/sql/test_13.sql @@ -0,0 +1,25 @@ +-- Use transaction block so that the file does not closed automatically at +-- command boundary. +BEGIN; +SELECT buffile_open_transient('trans1', true, false); +SELECT buffile_write('01234567'); +SELECT buffile_close_transient(); + +-- Open for reading. +SELECT buffile_open_transient('trans1', false, false); +SELECT length(buffile_read(65536)); +SELECT buffile_close_transient(); + +-- Open for writing in append mode. +SELECT buffile_open_transient('trans1', true, true); +-- Add BLCKSZ bytes, so that buffer boundary is crossed. +SELECT buffile_write(repeat('x', 8192)); +SELECT buffile_close_transient(); + +-- Open for reading and verify the valid part. +SELECT buffile_open_transient('trans1', false, false); +SELECT length(buffile_read(65536)); +SELECT buffile_close_transient(); + +SELECT buffile_delete_file('trans1'); +COMMIT; -- 2.13.7