From 9ffed6aecfd6d6aa672d9cfaf9329dd68b8b10f5 Mon Sep 17 00:00:00 2001 From: Alvaro Herrera Date: Fri, 28 Aug 2015 12:53:36 -0300 Subject: [PATCH 14/24] Add a generic API for column stores to implement This patch introduces struct ColumnStoreRoutine, similar to FdwRoutine, which contains function pointers that core code uses to operate on column stores. The actual API for column stores is defined in src/include/colstore/colstoreapi.h, and includes functions to insert and fetch (in retail and batch modes), as well as "discard" and "prune" elements from the store. Implementation notes: - There's a new execnode, ColumnStoreInfo, which is used by the executor to access the provided routines. --- src/backend/catalog/colstore.c | 156 +++++++++++++++++++++++++++++++++++++ src/backend/utils/cache/relcache.c | 3 + src/include/catalog/colstore.h | 3 + src/include/colstore/colstoreapi.h | 85 ++++++++++++++++++++ src/include/nodes/execnodes.h | 19 +++++ src/include/nodes/nodes.h | 4 +- src/include/utils/rel.h | 3 + 7 files changed, 272 insertions(+), 1 deletion(-) create mode 100644 src/include/colstore/colstoreapi.h diff --git a/src/backend/catalog/colstore.c b/src/backend/catalog/colstore.c index 553ab68..3954e1d 100644 --- a/src/backend/catalog/colstore.c +++ b/src/backend/catalog/colstore.c @@ -21,8 +21,10 @@ #include "catalog/heap.h" #include "catalog/indexing.h" #include "catalog/pg_cstore.h" +#include "catalog/pg_cstore_am.h" #include "catalog/pg_namespace.h" #include "catalog/pg_tablespace.h" /* GLOBALTABLESPACE_OID */ +#include "colstore/colstoreapi.h" #include "commands/tablespace.h" #include "miscadmin.h" #include "nodes/bitmapset.h" @@ -30,6 +32,7 @@ #include "utils/builtins.h" #include "utils/fmgroids.h" #include "utils/lsyscache.h" +#include "utils/memutils.h" #include "utils/syscache.h" #include "utils/rel.h" @@ -542,3 +545,156 @@ GetColumnStoreAMByName(char *cstamname, bool missing_ok) return oid; } + +/* ---------------- + * BuildColumnStoreInfo + * Construct an ColumnStoreInfo record for an open column store + * + * ColumnStoreInfo stores the information about the column store that's needed + * by FormColumnStoreDatum, which is used for insertion of individual column + * store tuples. Normally we build a ColumnStoreInfo for a column store just + * once per command, and then use it for (potentially) many tuples. + * ---------------- + */ +ColumnStoreInfo * +BuildColumnStoreInfo(Relation cstore) +{ + ColumnStoreInfo *csi = makeNode(ColumnStoreInfo); + Form_pg_cstore cstoreStruct = cstore->rd_cstore; + int i; + int numKeys; + + /* check the number of keys, and copy attr numbers into the IndexInfo */ + numKeys = cstoreStruct->cstnatts; + if (numKeys < 1 || numKeys > INDEX_MAX_KEYS) + elog(ERROR, "invalid cstnatts %d for column store %u", + numKeys, RelationGetRelid(cstore)); + csi->csi_NumColumnStoreAttrs = numKeys; + for (i = 0; i < numKeys; i++) + csi->csi_KeyAttrNumbers[i] = cstoreStruct->cstatts.values[i]; + + csi->csi_ColumnStoreRoutine = + GetColumnStoreRoutineForRelation(cstore, true); + + return csi; +} + +/* + * GetColumnStoreRoutine - call the specified column store handler routine + * to get its ColumnStoreRoutine struct. + */ +ColumnStoreRoutine * +GetColumnStoreRoutine(Oid cstamhandler) +{ + Datum datum; + ColumnStoreRoutine *routine; + + datum = OidFunctionCall0(cstamhandler); + routine = (ColumnStoreRoutine *) DatumGetPointer(datum); + + if (routine == NULL || !IsA(routine, ColumnStoreRoutine)) + elog(ERROR, "column store handler function %u did not return an ColumnStoreRoutine struct", + cstamhandler); + + return routine; +} + +/* + * GetColumnStoreHandlerByRelId - look up the handler of the column store handler + * for the given column store relation + */ +Oid +GetColumnStoreHandlerByRelId(Oid relid) +{ + HeapTuple tp; + Form_pg_cstore_am cstform; + Oid cstamhandler; + Relation rel; + Oid amoid; + + rel = relation_open(relid, AccessShareLock); + amoid = rel->rd_rel->relam; + + Assert(amoid != InvalidOid); + Assert(rel->rd_rel->relkind == RELKIND_COLUMN_STORE); + + relation_close(rel, AccessShareLock); + + /* Get server OID for the foreign table. */ + tp = SearchSysCache1(CSTOREAMOID, ObjectIdGetDatum(amoid)); + if (!HeapTupleIsValid(tp)) + elog(ERROR, "cache lookup failed for foreign table %u", amoid); + cstform = (Form_pg_cstore_am) GETSTRUCT(tp); + cstamhandler = cstform->cstamhandler; + + /* Complain if column store has been set to NO HANDLER. */ + if (!OidIsValid(cstamhandler)) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("column store \"%s\" has no handler", + NameStr(cstform->cstamname)))); + + ReleaseSysCache(tp); + + return cstamhandler; +} + +/* + * GetColumnStoreRoutineByRelId - look up the handler of the column store, and + * retrieve its ColumnStoreRoutine struct. + */ +ColumnStoreRoutine * +GetColumnStoreRoutineByRelId(Oid relid) +{ + Oid cstamhandler = GetColumnStoreHandlerByRelId(relid); + + return GetColumnStoreRoutine(cstamhandler); +} + +/* + * GetColumnStoreRoutineForRelation - look up the handler of the given column + * store, and retrieve its ColumnStoreRoutine struct. + * + * This function is preferred over GetColumnStoreRoutineByRelId because it + * caches the data in the relcache entry, saving a number of catalog lookups. + * + * If makecopy is true then the returned data is freshly palloc'd in the + * caller's memory context. Otherwise, it's a pointer to the relcache data, + * which will be lost in any relcache reset --- so don't rely on it long. + */ +ColumnStoreRoutine * +GetColumnStoreRoutineForRelation(Relation relation, bool makecopy) +{ + ColumnStoreRoutine *cstroutine; + ColumnStoreRoutine *ccstroutine; + + Assert(relation->rd_rel->relkind == RELKIND_COLUMN_STORE); + Assert(relation->rd_rel->relam != 0); + + if (relation->rd_colstoreroutine == NULL) + { + /* Get the info by consulting the catalogs */ + cstroutine = GetColumnStoreRoutineByRelId(RelationGetRelid(relation)); + + /* Save the data for later reuse in CacheMemoryContext */ + ccstroutine = + (ColumnStoreRoutine *) MemoryContextAlloc(CacheMemoryContext, + sizeof(ColumnStoreRoutine)); + memcpy(ccstroutine, cstroutine, sizeof(ColumnStoreRoutine)); + relation->rd_colstoreroutine = ccstroutine; + + /* Give back the locally palloc'd copy regardless of makecopy */ + return cstroutine; + } + + /* We have valid cached data --- does the caller want a copy? */ + if (makecopy) + { + ccstroutine = (ColumnStoreRoutine *) palloc(sizeof(ColumnStoreRoutine)); + memcpy(ccstroutine, relation->rd_colstoreroutine, sizeof(ColumnStoreRoutine)); + return ccstroutine; + } + + /* Only a short-lived reference is needed, so just hand back cached copy */ + return relation->rd_colstoreroutine; +} diff --git a/src/backend/utils/cache/relcache.c b/src/backend/utils/cache/relcache.c index 8dcf42a..3e82223 100644 --- a/src/backend/utils/cache/relcache.c +++ b/src/backend/utils/cache/relcache.c @@ -2054,6 +2054,8 @@ RelationDestroyRelation(Relation relation, bool remember_tupdesc) MemoryContextDelete(relation->rd_rsdesc->rscxt); if (relation->rd_fdwroutine) pfree(relation->rd_fdwroutine); + if (relation->rd_colstoreroutine) + pfree(relation->rd_colstoreroutine); pfree(relation); } @@ -4975,6 +4977,7 @@ load_relcache_init_file(bool shared) rel->rd_exclprocs = NULL; rel->rd_exclstrats = NULL; rel->rd_fdwroutine = NULL; + rel->rd_colstoreroutine = NULL; /* * Reset transient-state fields in the relcache entry diff --git a/src/include/catalog/colstore.h b/src/include/catalog/colstore.h index 5dcc91a..638c0e5 100644 --- a/src/include/catalog/colstore.h +++ b/src/include/catalog/colstore.h @@ -1,6 +1,7 @@ #ifndef COLSTORE_H #define COLSTORE_H +#include "nodes/execnodes.h" #include "nodes/nodes.h" #include "nodes/parsenodes.h" #include "nodes/pg_list.h" @@ -38,4 +39,6 @@ extern Oid get_relation_cstore_oid(Oid relid, const char *cstore_name, extern void RemoveColstoreById(Oid cstoreOid); extern Oid GetColumnStoreAMByName(char *cstamname, bool missing_ok); +extern ColumnStoreInfo *BuildColumnStoreInfo(Relation cstore); + #endif /* COLSTORE_H */ diff --git a/src/include/colstore/colstoreapi.h b/src/include/colstore/colstoreapi.h new file mode 100644 index 0000000..c3625dc --- /dev/null +++ b/src/include/colstore/colstoreapi.h @@ -0,0 +1,85 @@ +/*------------------------------------------------------------------------- + * + * colstoreapi.h + * API for column store implementations + * + * Copyright (c) 2010-2015, PostgreSQL Global Development Group + * + * src/include/colstore/colstoreapi.h + * + *------------------------------------------------------------------------- + */ +#ifndef COLSTOREAPI_H +#define COLSTOREAPI_H + +#include "nodes/execnodes.h" +#include "nodes/relation.h" + +typedef void (*ExecColumnStoreInsert_function) (Relation rel, + Relation colstorerel, ColumnStoreInfo *info, + int natts, Datum *values, bool *nulls, + ItemPointer tupleid); + +typedef void (*ExecColumnStoreBatchInsert_function) (Relation rel, + Relation colstorerel, ColumnStoreInfo *info, + int nrows, int natts, Datum **values, bool **nulls, + ItemPointer *tupleids); + +typedef void (*ExecColumnStoreFetch_function) (Relation rel, + Relation colstorerel, ColumnStoreInfo *info, ItemPointer tid); + +typedef void (*ExecColumnStoreBatchFetch_function) (Relation rel, + Relation colstorerel, ColumnStoreInfo *info, + int ntids, ItemPointer *tids); + +typedef void (*ExecColumnStoreDiscard_function) (Relation rel, + Relation colstorerel, ColumnStoreInfo *info, + int ntids, ItemPointer *tids); + +typedef void (*ExecColumnStorePrune_function) (Relation rel, + Relation colstorerel, ColumnStoreInfo *info, + int ntids, ItemPointer *tids); + +/* + * ColumnStoreRoutine is the struct returned by a column store's handler + * function. It provides pointers to the callback functions needed by the + * planner and executor. + * + * More function pointers are likely to be added in the future. Therefore + * it's recommended that the handler initialize the struct with + * makeNode(ColumnStoreRoutine) so that all fields are set to NULL. This will + * ensure that no fields are accidentally left undefined. + * + * XXX Mostly a copy of FdwRoutine. + */ +typedef struct ColumnStoreRoutine +{ + NodeTag type; + + /* insert a single row into the column store */ + ExecColumnStoreInsert_function ExecColumnStoreInsert; + + /* insert a batch of rows into the column store */ + ExecColumnStoreBatchInsert_function ExecColumnStoreBatchInsert; + + /* fetch values for a single row */ + ExecColumnStoreFetch_function ExecColumnStoreFetch; + + /* fetch a batch of values for a single row */ + ExecColumnStoreBatchFetch_function ExecColumnStoreBatchFetch; + + /* discard a batch of deleted rows from the column store */ + ExecColumnStoreDiscard_function ExecColumnStoreDiscard; + + /* prune the store - keep only the valid rows */ + ExecColumnStorePrune_function ExecColumnStorePrune; + +} ColumnStoreRoutine; + +extern Oid GetColumnStoreHandlerByRelId(Oid relid); +extern ColumnStoreRoutine *GetColumnStoreRoutine(Oid csthandler); +extern ColumnStoreRoutine *GetColumnStoreRoutineByRelId(Oid relid); +extern ColumnStoreRoutine *GetColumnStoreRoutineForRelation(Relation relation, + bool makecopy); + +#endif /* COLSTOREAPI_H */ diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index 5796de8..5816a37 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -77,6 +77,25 @@ typedef struct IndexInfo } IndexInfo; /* ---------------- + * ColumnStoreInfo information + * + * this struct holds the information needed to construct new column store + * entries for a particular column store. + * + * NumColumnStoreAttrs number of columns in this column store + * KeyAttrNumbers underlying-rel attribute numbers used as keys + * ColumnStoreRoutine ColumnStore callback functions + * ---------------- + */ +typedef struct ColumnStoreInfo +{ + NodeTag type; + int csi_NumColumnStoreAttrs; + AttrNumber csi_KeyAttrNumbers[INDEX_MAX_KEYS]; + struct ColumnStoreRoutine *csi_ColumnStoreRoutine; +} ColumnStoreInfo; + +/* ---------------- * ExprContext_CB * * List of callbacks to be called at ExprContext shutdown. diff --git a/src/include/nodes/nodes.h b/src/include/nodes/nodes.h index 53e0a0b..6950e87 100644 --- a/src/include/nodes/nodes.h +++ b/src/include/nodes/nodes.h @@ -37,6 +37,7 @@ typedef enum NodeTag T_ResultRelInfo, T_EState, T_TupleTableSlot, + T_ColumnStoreInfo, /* * TAGS FOR PLAN NODES (plannodes.h) @@ -455,7 +456,8 @@ typedef enum NodeTag T_TIDBitmap, /* in nodes/tidbitmap.h */ T_InlineCodeBlock, /* in nodes/parsenodes.h */ T_FdwRoutine, /* in foreign/fdwapi.h */ - T_TsmRoutine /* in access/tsmapi.h */ + T_TsmRoutine, /* in access/tsmapi.h */ + T_ColumnStoreRoutine /* in colstore/colstoreapi.h */ } NodeTag; /* diff --git a/src/include/utils/rel.h b/src/include/utils/rel.h index 99c7ef6..29a24b0 100644 --- a/src/include/utils/rel.h +++ b/src/include/utils/rel.h @@ -181,6 +181,9 @@ typedef struct RelationData /* use "struct" here to avoid needing to include htup.h: */ struct HeapTupleData *rd_cstoretuple; /* all of pg_cstore tuple */ + /* use "struct" here to avoid needing to include colstoreapi.h: */ + struct ColumnStoreRoutine *rd_colstoreroutine; /* cached function pointers, or NULL */ + /* * Hack for CLUSTER, rewriting ALTER TABLE, etc: when writing a new * version of a table, we need to make any toast pointers inserted into it -- 2.1.4