From 993eea51a2b73f7ee02454b290e0e40288e86a03 Mon Sep 17 00:00:00 2001 From: Alvaro Herrera Date: Fri, 24 Apr 2015 19:15:02 -0300 Subject: [PATCH 11/24] Infrastructure to create column stores This introduces enough infrastructure to actually create the catalog entries when the grammar has received COLUMN STORE clauses. This creates pg_class entries (and relfilenodes) for the colstores, with the corresponding pg_attribute entries; pg_cstore entries are also created. Add pg_depend entries too, so that everything is dropped when the table is. We create one relfilenode for each column store, which is referenced in pg_class.relfilenode. The column store AM is free to use this or not, as well as create other relfilenodes by itself. If we get a pg_relfilenode catalog as discussed, this may be improved. Implementation notes: - Column store creation has two steps: DetermineColumnStores (called before the heap itself is created) examines table-level and column-level clauses, and merges both with column stores coming from inherited relations. These are transformed into a list of ColumnStoreElem struct. The second step occurs after the heap itself is created, and records everything into pg_class, pg_cstore and pg_attribute entries, one relfilenodes for each column store, assigns OIDs to everything, etc. - This patch creates new ObjectClass and ObjectType enum members for column stores, and associated fallout. - Need to get rid of OIDs in pg_cstore, which requires some changes in the pg_depend entries and the deletion code introduced here. - FIXME CloneColumnStores is a stub. This function is used to recreate column stores when a table's heap is rewritten (cluster, vacuum full, etc). - FIXME The event trigger system hasn't been updated to cope with the new object types. --- src/backend/access/common/tupdesc.c | 20 +- src/backend/bootstrap/bootparse.y | 1 + src/backend/catalog/Makefile | 6 +- src/backend/catalog/colstore.c | 544 +++++++++++++++++++++++++++++++++++ src/backend/catalog/dependency.c | 9 + src/backend/catalog/heap.c | 22 +- src/backend/catalog/objectaddress.c | 80 ++++++ src/backend/catalog/toasting.c | 1 + src/backend/commands/cluster.c | 2 + src/backend/commands/event_trigger.c | 4 + src/backend/commands/tablecmds.c | 79 ++++- src/backend/commands/view.c | 2 +- src/include/access/tupdesc.h | 6 +- src/include/catalog/colstore.h | 41 +++ src/include/catalog/dependency.h | 1 + src/include/catalog/heap.h | 7 + src/include/nodes/parsenodes.h | 1 + 17 files changed, 811 insertions(+), 15 deletions(-) create mode 100644 src/backend/catalog/colstore.c create mode 100644 src/include/catalog/colstore.h diff --git a/src/backend/access/common/tupdesc.c b/src/backend/access/common/tupdesc.c index 41d71c8..cb622a7 100644 --- a/src/backend/access/common/tupdesc.c +++ b/src/backend/access/common/tupdesc.c @@ -20,6 +20,7 @@ #include "postgres.h" #include "access/htup_details.h" +#include "catalog/colstore.h" #include "catalog/pg_type.h" #include "miscadmin.h" #include "parser/parse_type.h" @@ -580,12 +581,17 @@ TupleDescInitEntryCollation(TupleDesc desc, * * Given a relation schema (list of ColumnDef nodes), build a TupleDesc. * + * Also append ColumnStoreClauseInfo structs in *colstores, with one element for + * each attribute that contains a COLUMN STORE clause. colstores may be NULL if + * the caller is certain that there are no colstore clauses (e.g. when defining a + * view.) + * * Note: the default assumption is no OIDs; caller may modify the returned * TupleDesc if it wants OIDs. Also, tdtypeid will need to be filled in * later on. */ TupleDesc -BuildDescForRelation(List *schema) +BuildDescForRelation(List *schema, List **colstores) { int natts; AttrNumber attnum; @@ -648,6 +654,18 @@ BuildDescForRelation(List *schema) has_not_null |= entry->is_not_null; desc->attrs[attnum - 1]->attislocal = entry->is_local; desc->attrs[attnum - 1]->attinhcount = entry->inhcount; + + /* Fill in the column store info, if this column requires it */ + if (entry->cstoreClause) + { + ColumnStoreClauseInfo *store = palloc(sizeof(ColumnStoreClauseInfo)); + + store->attnum = attnum; + store->cstoreClause = entry->cstoreClause; + store->attnums = NIL; + store->cstoreOid = InvalidOid; + *colstores = lappend(*colstores, store); + } } if (has_not_null) diff --git a/src/backend/bootstrap/bootparse.y b/src/backend/bootstrap/bootparse.y index d8d1b06..10d38d2 100644 --- a/src/backend/bootstrap/bootparse.y +++ b/src/backend/bootstrap/bootparse.y @@ -240,6 +240,7 @@ Boot_CreateStmt: BOOTSTRAP_SUPERUSERID, tupdesc, NIL, + NIL, RELKIND_RELATION, RELPERSISTENCE_PERMANENT, shared_relation, diff --git a/src/backend/catalog/Makefile b/src/backend/catalog/Makefile index bdf4e35..f28dd8c 100644 --- a/src/backend/catalog/Makefile +++ b/src/backend/catalog/Makefile @@ -10,9 +10,9 @@ subdir = src/backend/catalog top_builddir = ../../.. include $(top_builddir)/src/Makefile.global -OBJS = catalog.o dependency.o heap.o index.o indexing.o namespace.o aclchk.o \ - objectaccess.o objectaddress.o pg_aggregate.o pg_collation.o \ - pg_constraint.o pg_conversion.o \ +OBJS = aclchk.o catalog.o colstore.o dependency.o heap.o index.o indexing.o \ + namespace.o objectaccess.o objectaddress.o \ + pg_aggregate.o pg_collation.o pg_constraint.o pg_conversion.o \ pg_depend.o pg_enum.o pg_inherits.o pg_largeobject.o pg_namespace.o \ pg_operator.o pg_proc.o pg_range.o pg_db_role_setting.o pg_shdepend.o \ pg_type.o storage.o toasting.o diff --git a/src/backend/catalog/colstore.c b/src/backend/catalog/colstore.c new file mode 100644 index 0000000..553ab68 --- /dev/null +++ b/src/backend/catalog/colstore.c @@ -0,0 +1,544 @@ +/*------------------------------------------------------------------------- + * + * colstore.c + * POSTGRES column store support code + * + * Portions Copyright (c) 1996-2015, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/catalog/colstore.c + */ +#include "postgres.h" + +#include "access/genam.h" +#include "access/heapam.h" +#include "access/htup_details.h" +#include "access/sysattr.h" +#include "catalog/catalog.h" +#include "catalog/colstore.h" +#include "catalog/dependency.h" +#include "catalog/heap.h" +#include "catalog/indexing.h" +#include "catalog/pg_cstore.h" +#include "catalog/pg_namespace.h" +#include "catalog/pg_tablespace.h" /* GLOBALTABLESPACE_OID */ +#include "commands/tablespace.h" +#include "miscadmin.h" +#include "nodes/bitmapset.h" +#include "nodes/parsenodes.h" +#include "utils/builtins.h" +#include "utils/fmgroids.h" +#include "utils/lsyscache.h" +#include "utils/syscache.h" +#include "utils/rel.h" + +typedef struct ColumnStoreElem +{ + char *name; + Oid cst_am_oid; + Oid tablespaceId; + List *columns; /* of AttrNumber */ +} ColumnStoreElem; + +static TupleDesc ColumnStoreBuildDesc(ColumnStoreElem *elem, Relation parent); +static void AddNewColstoreTuple(Relation pg_cstore, ColumnStoreElem *entry, + Oid storeId, Oid relid); + +/* + * Figure out the set of column stores to create. + * + * decl_cstores is a list of column stores directly declared for the relation. + * inh_cstores is a list of column stores inherited from parent relations. + * We need this distinction because multiple uses of a column in declared ones + * is an error, but we ignore duplicates for the inherited ones. + * + * tablespaceId is the tablespace of the owning relation; it is used as the + * default tablespace for column stores that do not have a tablespace + * specification. + * + * Return value is a list of ColumnStoreElem. + */ +List * +DetermineColumnStores(TupleDesc tupdesc, List *decl_cstores, + List *inh_cstores, Oid tablespaceId) +{ + List *newstores = NIL; + Bitmapset *used; + ListCell *cell, + *cell2; + + /* + * There is no provision (not here, nor in the grammar) for a column that's + * part of a column store in the parent, but in the heap for the child + * table. Do we need a fix for that? + */ + + /* + * We use this bitmapset to keep track of columns which have already been + * assigned to a store. + */ + used = NULL; + + /* + * First scan the list of declared column stores. Using columns here more + * than once causes an error to be raised. + */ + foreach(cell, decl_cstores) + { + ColumnStoreClauseInfo *info = (ColumnStoreClauseInfo *) lfirst(cell); + ColumnStoreClause *clause = info->cstoreClause; + ColumnStoreElem *newstore; + + Assert(clause != NULL); + Assert((info->attnum != InvalidAttrNumber) || + (clause->columns != NIL)); + Assert(info->attnums == NIL && info->cstoreOid == InvalidOid); + + /* + * Verify that the name has not already been taken + */ + foreach(cell2, newstores) + { + ColumnStoreElem *elem = (ColumnStoreElem *) lfirst(cell2); + + if (strcmp(elem->name, clause->name) == 0) + /* XXX ereport */ + elog(ERROR, "duplicate column store name \"%s\"", clause->name); + } + + newstore = (ColumnStoreElem *) palloc(sizeof(ColumnStoreElem)); + newstore->name = clause->name; + newstore->cst_am_oid = GetColumnStoreAMByName(clause->storetype, false); + + /* + * Select tablespace to use. If not specified, use the tablespace + * of the parent relation. + * + * These are effectively the same checks as in DefineRelation. + */ + if (clause->tablespacename) + { + newstore->tablespaceId = + get_tablespace_oid(clause->tablespacename, false); + } + else + /* use tablespace of the relation */ + newstore->tablespaceId = tablespaceId; + + /* Check permissions except when using database's default */ + if (OidIsValid(newstore->tablespaceId) && + newstore->tablespaceId != MyDatabaseTableSpace) + { + AclResult aclresult; + + aclresult = pg_tablespace_aclcheck(newstore->tablespaceId, + GetUserId(), ACL_CREATE); + if (aclresult != ACLCHECK_OK) + aclcheck_error(aclresult, ACL_KIND_TABLESPACE, + get_tablespace_name(newstore->tablespaceId)); + } + + /* In all cases disallow placing user objects in pg_global */ + if (newstore->tablespaceId == GLOBALTABLESPACE_OID) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("only shared relations can be placed in pg_global tablespace"))); + + /* + * Fill in the attnum list: if it's a column store declared as column + * constraint, the only attnum was already determined by + * BuildDescForRelation. Otherwise we need to resolve column names to + * attnums using the tuple descriptor. + */ + if (info->attnum != InvalidAttrNumber) + newstore->columns = list_make1_int(info->attnum); + else + { + newstore->columns = NIL; + + foreach(cell2, clause->columns) + { + char *colname = strVal(lfirst(cell2)); + AttrNumber attnum, + i; + + attnum = InvalidAttrNumber; + for (i = 1; i <= tupdesc->natts; i++) + { + if (namestrcmp(&(tupdesc->attrs[i - 1]->attname), colname) == 0) + attnum = i; + } + /* XXX ereport */ + if (attnum == InvalidAttrNumber) + elog(ERROR, "no column \"%s\" in the table", colname); + + newstore->columns = lappend_int(newstore->columns, attnum); + } + } + + /* Make sure there are no columns specified in multiple stores */ + foreach(cell2, newstore->columns) + { + AttrNumber attno = lfirst_int(cell2); + + /* XXX ereport */ + if (bms_is_member(attno, used)) + elog(ERROR, "column already in a store"); + + used = bms_add_member(used, attno); + } + + newstores = lappend(newstores, newstore); + } + + /* + * Now process the list of column stores coming from parent relations. + * Columns that are already used by previous stores are silently ignored. + * (In particular, this means that some parent stores might not exist at + * all in the child). + */ + foreach (cell, inh_cstores) + { + ColumnStoreClauseInfo *info = (ColumnStoreClauseInfo *) lfirst(cell); + Relation parentstore; + List *attnums = NIL; + + Assert((info->attnum == InvalidAttrNumber) && + (info->cstoreClause == NULL)); + Assert((info->attnums != NIL) && + (info->cstoreOid != InvalidOid)); + + parentstore = relation_open(info->cstoreOid, AccessShareLock); + + /* + * Examine the column list first. If all columns are used in + * previously defined column stores, we can ignore this one. + */ + foreach(cell2, info->attnums) + { + AttrNumber attnum = lfirst_int(cell2); + + if (bms_is_member(attnum, used)) + continue; + attnums = lappend_int(attnums, attnum); + used = bms_add_member(used, attnum); + } + + /* If we ended up with a nonempty list, add this store to the list */ + if (attnums != NIL) + { + ColumnStoreElem *newstore; + newstore = (ColumnStoreElem *) palloc(sizeof(ColumnStoreElem)); + + newstore->name = pstrdup(NameStr(parentstore->rd_cstore->cstname)); + newstore->cst_am_oid = parentstore->rd_cstore->cststoreid; + newstore->tablespaceId = parentstore->rd_rel->reltablespace; + newstore->columns = attnums; + + newstores = lappend(newstores, newstore); + } + + relation_close(parentstore, AccessShareLock); + } + + /* + * Check that the names of the column stores are unique, so that we can + * print a nice error message instead of a confusing unique violation + * later. This is O(N^2), but that should not be a problem. + * + * XXX We don't have relname here, so we can't put it to the message. + */ + foreach (cell, newstores) + { + ListCell *cell2; + ColumnStoreElem *elem1 = (ColumnStoreElem *) lfirst(cell); + + foreach (cell2, newstores) + { + ColumnStoreElem *elem2 = (ColumnStoreElem *) lfirst(cell2); + + if (elem1 == elem2) /* skip the same element */ + continue; + + /* same names */ + if (strcmp(elem1->name, elem2->name) == 0) + ereport(ERROR, + (errcode(ERRCODE_DUPLICATE_OBJECT), + errmsg("column store \"%s\" already exists", elem1->name))); + + } + } + + /* + * Return the info we collected. + */ + return newstores; +} + +/* + * Create the column stores for the given table. This creates the files + * assigned as relfilenode for each column store; also, the pg_class and + * pg_cstore catalog entries are created. + */ +void +CreateColumnStores(Relation rel, List *colstores) +{ + Relation pg_class; + Relation pg_cstore; + ListCell *cell; + int storenum = 1; + ObjectAddress parentrel; + + if (colstores == NIL) + return; + + pg_class = heap_open(RelationRelationId, RowExclusiveLock); + pg_cstore = heap_open(CStoreRelationId, RowExclusiveLock); + + ObjectAddressSet(parentrel, RelationRelationId, + RelationGetRelid(rel)); + + foreach(cell, colstores) + { + ColumnStoreElem *elem = (ColumnStoreElem *) lfirst(cell); + Relation store; + Oid newStoreId; + TupleDesc storedesc = ColumnStoreBuildDesc(elem, rel); + char *classname; + ObjectAddress myself; + + /* + * Get the OID for the new column store. + */ + newStoreId = GetNewRelFileNode(elem->tablespaceId, pg_class, + rel->rd_rel->relpersistence); + + classname = psprintf("pg_colstore_%u_%d", + RelationGetRelid(rel), storenum++); + + /* + * Create the relcache entry for the store. This also creates the + * underlying storage; it's smgr's responsibility to remove the file if + * we fail later on. + */ + store = + heap_create(classname, + PG_COLSTORE_NAMESPACE, + elem->tablespaceId, + newStoreId, + InvalidOid, + storedesc, + RELKIND_COLUMN_STORE, + rel->rd_rel->relpersistence, + false, + false, + allowSystemTableMods); + + /* insert pg_attribute tuples */ + AddNewAttributeTuples(newStoreId, + storedesc, + RELKIND_COLUMN_STORE, + false, + 0); + + /* + * Insert the pg_class tuple + */ + store->rd_rel->relam = elem->cst_am_oid; + InsertPgClassTuple(pg_class, store, newStoreId, + (Datum) 0, (Datum) 0); + + /* And finally insert the pg_cstore tuple */ + AddNewColstoreTuple(pg_cstore, elem, newStoreId, + RelationGetRelid(rel)); + + /* + * This is a good place to record dependencies. We choose to have all the + * subsidiary entries (both pg_class and pg_cstore entries) depend on the + * pg_class entry for the main relation. + */ + ObjectAddressSet(myself, CStoreRelationId, newStoreId); + recordDependencyOn(&myself, &parentrel, DEPENDENCY_INTERNAL); + ObjectAddressSet(myself, RelationRelationId, newStoreId); + recordDependencyOn(&myself, &parentrel, DEPENDENCY_INTERNAL); + + heap_close(store, NoLock); + } + + heap_close(pg_class, RowExclusiveLock); + heap_close(pg_cstore, RowExclusiveLock); +} + +/* + * Build a tuple descriptor for a not-yet-catalogued column store. + */ +static TupleDesc +ColumnStoreBuildDesc(ColumnStoreElem *elem, Relation parent) +{ + TupleDesc tupdesc; + TupleDesc parentdesc; + ListCell *cell; + AttrNumber attnum = 1; + + parentdesc = RelationGetDescr(parent); + + tupdesc = CreateTemplateTupleDesc(list_length(elem->columns), false); + + foreach(cell, elem->columns) + { + AttrNumber parentattnum = lfirst_int(cell); + + TupleDescInitEntry(tupdesc, + attnum++, + NameStr(parentdesc->attrs[parentattnum - 1]->attname), + parentdesc->attrs[parentattnum - 1]->atttypid, + parentdesc->attrs[parentattnum - 1]->atttypmod, + parentdesc->attrs[parentattnum - 1]->attndims); + } + + return tupdesc; +} + +/* + * Return a list of column store definitions for an existing table + */ +List * +CloneColumnStores(Relation rel) +{ + /* FIXME fill this in */ + return NIL; +} + +/* + * Add a new pg_cstore tuple for a column store + */ +static void +AddNewColstoreTuple(Relation pg_cstore, ColumnStoreElem *entry, + Oid storeId, Oid relid) +{ + HeapTuple newtup; + ListCell *cell; + Datum values[Natts_pg_cstore]; + bool nulls[Natts_pg_cstore]; + NameData cstname; + int natts; + int16 *attrarr; + int2vector *attrs; + int i = 0; + + /* build the int2vector of attribute numbers */ + natts = list_length(entry->columns); + Assert(natts > 0); + attrarr = palloc(sizeof(int16 *) * natts); + foreach(cell, entry->columns) + attrarr[i++] = (AttrNumber) lfirst_int(cell); + attrs = buildint2vector(attrarr, natts); + + /* build the pg_cstore tuple */ + namestrcpy(&cstname, entry->name); + values[Anum_pg_cstore_cstname - 1] = NameGetDatum(&cstname); + values[Anum_pg_cstore_cstrelid - 1] = ObjectIdGetDatum(relid); + values[Anum_pg_cstore_cststoreid - 1] = ObjectIdGetDatum(storeId); + values[Anum_pg_cstore_cstnatts - 1] = Int32GetDatum(list_length(entry->columns)); + values[Anum_pg_cstore_cstatts - 1] = PointerGetDatum(attrs); + memset(nulls, 0, sizeof(nulls)); + newtup = heap_form_tuple(RelationGetDescr(pg_cstore), values, nulls); + + HeapTupleSetOid(newtup, storeId); + + /* insert it into pg_cstore */ + simple_heap_insert(pg_cstore, newtup); + + /* keep indexes current */ + CatalogUpdateIndexes(pg_cstore, newtup); + + heap_freetuple(newtup); +} + +Oid +get_relation_cstore_oid(Oid relid, const char *cstore_name, bool missing_ok) +{ + Relation pg_cstore_rel; + ScanKeyData skey[2]; + SysScanDesc sscan; + HeapTuple cstore_tuple; + Oid cstore_oid; + + pg_cstore_rel = heap_open(CStoreRelationId, AccessShareLock); + + ScanKeyInit(&skey[0], + Anum_pg_cstore_cstrelid, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(relid)); + ScanKeyInit(&skey[1], + Anum_pg_cstore_cstname, + BTEqualStrategyNumber, F_NAMEEQ, + CStringGetDatum(cstore_name)); + + sscan = systable_beginscan(pg_cstore_rel, + CStoreCstRelidCstnameIndexId, true, NULL, 2, + skey); + + cstore_tuple = systable_getnext(sscan); + + if (!HeapTupleIsValid(cstore_tuple)) + { + if (!missing_ok) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_OBJECT), + errmsg("column store \"%s\" for table \"%s\" does not exist", + cstore_name, get_rel_name(relid)))); + + cstore_oid = InvalidOid; + } + else + cstore_oid = HeapTupleGetOid(cstore_tuple); + + /* Clean up. */ + systable_endscan(sscan); + heap_close(pg_cstore_rel, AccessShareLock); + + return cstore_oid; +} + +void +RemoveColstoreById(Oid cstoreOid) +{ + Relation pg_cstore; + HeapTuple tuple; + + pg_cstore = heap_open(CStoreRelationId, RowExclusiveLock); + + tuple = SearchSysCache1(CSTOREOID, ObjectIdGetDatum(cstoreOid)); + + if (!HeapTupleIsValid(tuple)) + elog(ERROR, "cache lookup failed for column store %u", cstoreOid); + + simple_heap_delete(pg_cstore, &tuple->t_self); + + ReleaseSysCache(tuple); + + heap_close(pg_cstore, RowExclusiveLock); +} + +Oid +GetColumnStoreAMByName(char *cstamname, bool missing_ok) +{ + Oid oid; + + oid = GetSysCacheOid1(CSTOREAMNAME, + CStringGetDatum(cstamname)); + if (!OidIsValid(oid)) + { + if (missing_ok) + return InvalidOid; + + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_OBJECT), + errmsg("column store access method \"%s\" does not exist", + cstamname))); + } + + return oid; +} diff --git a/src/backend/catalog/dependency.c b/src/backend/catalog/dependency.c index 90b1cd8..8584890 100644 --- a/src/backend/catalog/dependency.c +++ b/src/backend/catalog/dependency.c @@ -16,6 +16,7 @@ #include "access/htup_details.h" #include "access/xact.h" +#include "catalog/colstore.h" #include "catalog/dependency.h" #include "catalog/heap.h" #include "catalog/index.h" @@ -144,6 +145,7 @@ static const Oid object_classes[] = { AccessMethodProcedureRelationId, /* OCLASS_AMPROC */ RewriteRelationId, /* OCLASS_REWRITE */ TriggerRelationId, /* OCLASS_TRIGGER */ + CStoreRelationId, /* OCLASS_COLSTORE */ NamespaceRelationId, /* OCLASS_SCHEMA */ TSParserRelationId, /* OCLASS_TSPARSER */ TSDictionaryRelationId, /* OCLASS_TSDICT */ @@ -1214,6 +1216,10 @@ doDeletion(const ObjectAddress *object, int flags) RemoveTriggerById(object->objectId); break; + case OCLASS_COLSTORE: + RemoveColstoreById(object->objectId); + break; + case OCLASS_SCHEMA: RemoveSchemaById(object->objectId); break; @@ -2412,6 +2418,9 @@ getObjectClass(const ObjectAddress *object) case PolicyRelationId: return OCLASS_POLICY; + case CStoreRelationId: + return OCLASS_COLSTORE; + case TransformRelationId: return OCLASS_TRANSFORM; } diff --git a/src/backend/catalog/heap.c b/src/backend/catalog/heap.c index 2a6c576..a9f34d5 100644 --- a/src/backend/catalog/heap.c +++ b/src/backend/catalog/heap.c @@ -37,6 +37,7 @@ #include "access/xlog.h" #include "catalog/binary_upgrade.h" #include "catalog/catalog.h" +#include "catalog/colstore.h" #include "catalog/dependency.h" #include "catalog/heap.h" #include "catalog/index.h" @@ -653,7 +654,7 @@ InsertPgAttributeTuple(Relation pg_attribute_rel, * tuples to pg_attribute. * -------------------------------- */ -static void +void AddNewAttributeTuples(Oid new_rel_oid, TupleDesc tupdesc, char relkind, @@ -712,10 +713,11 @@ AddNewAttributeTuples(Oid new_rel_oid, /* * Next we add the system attributes. Skip OID if rel has no OIDs. Skip - * all for a view or type relation. We don't bother with making datatype - * dependencies here, since presumably all these types are pinned. + * all for a colstore, view or type relation. We don't bother with making + * datatype dependencies here, since presumably all these types are pinned. */ - if (relkind != RELKIND_VIEW && relkind != RELKIND_COMPOSITE_TYPE) + if (relkind != RELKIND_VIEW && relkind != RELKIND_COMPOSITE_TYPE && + relkind != RELKIND_COLUMN_STORE) { for (i = 0; i < (int) lengthof(SysAtt); i++) { @@ -994,6 +996,7 @@ AddNewRelationType(const char *typeName, * ownerid: OID of new rel's owner * tupdesc: tuple descriptor (source of column definitions) * cooked_constraints: list of precooked check constraints and defaults + * colstores: list (of ColumnStoreElem) of column stores for this rel * relkind: relkind for new rel * relpersistence: rel's persistence status (permanent, temp, or unlogged) * shared_relation: TRUE if it's to be a shared relation @@ -1023,6 +1026,7 @@ heap_create_with_catalog(const char *relname, Oid ownerid, TupleDesc tupdesc, List *cooked_constraints, + List *colstores, char relkind, char relpersistence, bool shared_relation, @@ -1248,6 +1252,9 @@ heap_create_with_catalog(const char *relname, pfree(relarrayname); } + /* Set relhascstore correctly */ + new_rel_desc->rd_rel->relhascstore = colstores != NIL; + /* * now create an entry in pg_class for the relation. * @@ -1266,6 +1273,13 @@ heap_create_with_catalog(const char *relname, reloptions); /* + * If the new relation has any column stores, create them now. This + * assigns their OIDs and creates the files on disk (it's smgr's + * responsibility to remove these files if we fail below.) + */ + CreateColumnStores(new_rel_desc, colstores); + + /* * now add tuples to pg_attribute for the attributes in our new relation. */ AddNewAttributeTuples(relid, new_rel_desc->rd_att, relkind, diff --git a/src/backend/catalog/objectaddress.c b/src/backend/catalog/objectaddress.c index 052aab1..51b5b6f 100644 --- a/src/backend/catalog/objectaddress.c +++ b/src/backend/catalog/objectaddress.c @@ -18,6 +18,7 @@ #include "access/htup_details.h" #include "access/sysattr.h" #include "catalog/catalog.h" +#include "catalog/colstore.h" #include "catalog/indexing.h" #include "catalog/objectaddress.h" #include "catalog/pg_amop.h" @@ -132,6 +133,18 @@ static const ObjectPropertyType ObjectProperty[] = true }, { + CStoreRelationId, + CStoreOidIndexId, + CSTOREOID, + -1, + Anum_pg_cstore_cstname, + InvalidAttrNumber, + InvalidAttrNumber, + InvalidAttrNumber, + -1, + false + }, + { ConstraintRelationId, ConstraintOidIndexId, CONSTROID, @@ -575,6 +588,10 @@ static const struct object_type_map { "trigger", OBJECT_TRIGGER }, + /* OCLASS_COLSTORE */ + { + "column store", OBJECT_COLSTORE + }, /* OCLASS_SCHEMA */ { "schema", OBJECT_SCHEMA @@ -765,6 +782,7 @@ get_object_address(ObjectType objtype, List *objname, List *objargs, case OBJECT_TRIGGER: case OBJECT_TABCONSTRAINT: case OBJECT_POLICY: + case OBJECT_COLSTORE: address = get_object_address_relobject(objtype, objname, &relation, missing_ok); break; @@ -1278,6 +1296,13 @@ get_object_address_relobject(ObjectType objtype, List *objname, InvalidOid; address.objectSubId = 0; break; + case OBJECT_COLSTORE: + address.classId = CStoreRelationId; + address.objectId = relation ? + get_relation_cstore_oid(reloid, depname, missing_ok) : + InvalidOid; + address.objectSubId = 0; + break; default: elog(ERROR, "unrecognized objtype: %d", (int) objtype); /* placate compiler, which doesn't know elog won't return */ @@ -2028,6 +2053,7 @@ check_object_ownership(Oid roleid, ObjectType objtype, ObjectAddress address, case OBJECT_TRIGGER: case OBJECT_POLICY: case OBJECT_TABCONSTRAINT: + case OBJECT_COLSTORE: if (!pg_class_ownercheck(RelationGetRelid(relation), roleid)) aclcheck_error(ACLCHECK_NOT_OWNER, ACL_KIND_CLASS, RelationGetRelationName(relation)); @@ -2791,6 +2817,30 @@ getObjectDescription(const ObjectAddress *object) break; } + case OCLASS_COLSTORE: + { + Relation cstDesc; + HeapTuple tup; + Form_pg_cstore store; + + cstDesc = heap_open(CStoreRelationId, AccessShareLock); + + tup = get_catalog_object_by_oid(cstDesc, object->objectId); + + if (!HeapTupleIsValid(tup)) + elog(ERROR, "could not find tuple for column store %u", + object->objectId); + + store = (Form_pg_cstore) GETSTRUCT(tup); + + appendStringInfo(&buffer, _("column store %s on "), + NameStr(store->cstname)); + getRelationDescription(&buffer, store->cstrelid); + + heap_close(cstDesc, AccessShareLock); + break; + } + case OCLASS_TRANSFORM: { HeapTuple trfTup; @@ -3544,6 +3594,10 @@ getObjectTypeDescription(const ObjectAddress *object) appendStringInfoString(&buffer, "trigger"); break; + case OCLASS_COLSTORE: + appendStringInfoString(&buffer, "column store"); + break; + case OCLASS_SCHEMA: appendStringInfoString(&buffer, "schema"); break; @@ -4185,6 +4239,32 @@ getObjectIdentityParts(const ObjectAddress *object, break; } + case OCLASS_COLSTORE: + { + Relation cstDesc; + HeapTuple tup; + Form_pg_cstore store; + + cstDesc = heap_open(CStoreRelationId, AccessShareLock); + + tup = get_catalog_object_by_oid(cstDesc, object->objectId); + + if (!HeapTupleIsValid(tup)) + elog(ERROR, "could not find tuple for column store %u", + object->objectId); + + store = (Form_pg_cstore) GETSTRUCT(tup); + + appendStringInfo(&buffer, "%s on ", + quote_identifier(NameStr(store->cstname))); + getRelationIdentity(&buffer, store->cstrelid, objname); + if (objname) + *objname = lappend(*objname, pstrdup(NameStr(store->cstname))); + + heap_close(cstDesc, AccessShareLock); + break; + } + case OCLASS_POLICY: { Relation polDesc; diff --git a/src/backend/catalog/toasting.c b/src/backend/catalog/toasting.c index 3652d7b..33ba582 100644 --- a/src/backend/catalog/toasting.c +++ b/src/backend/catalog/toasting.c @@ -279,6 +279,7 @@ create_toast_table(Relation rel, Oid toastOid, Oid toastIndexOid, rel->rd_rel->relowner, tupdesc, NIL, + NIL, RELKIND_TOASTVALUE, rel->rd_rel->relpersistence, shared_relation, diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index 7ab4874..c40a575 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -25,6 +25,7 @@ #include "access/xact.h" #include "access/xlog.h" #include "catalog/catalog.h" +#include "catalog/colstore.h" #include "catalog/dependency.h" #include "catalog/heap.h" #include "catalog/index.h" @@ -667,6 +668,7 @@ make_new_heap(Oid OIDOldHeap, Oid NewTableSpace, char relpersistence, OldHeap->rd_rel->relowner, OldHeapDesc, NIL, + CloneColumnStores(OldHeap), RELKIND_RELATION, relpersistence, false, diff --git a/src/backend/commands/event_trigger.c b/src/backend/commands/event_trigger.c index 3d1cb0b..d323932 100644 --- a/src/backend/commands/event_trigger.c +++ b/src/backend/commands/event_trigger.c @@ -1118,6 +1118,8 @@ EventTriggerSupportsObjectType(ObjectType obtype) case OBJECT_USER_MAPPING: case OBJECT_VIEW: return true; + case OBJECT_COLSTORE: + elog(ERROR, "unsupported --- XXX fill this in"); } return true; } @@ -1168,6 +1170,8 @@ EventTriggerSupportsObjectClass(ObjectClass objclass) case OCLASS_EXTENSION: case OCLASS_POLICY: return true; + case OCLASS_COLSTORE: + elog(ERROR, "unsupported --- XXX fill this in"); } return true; diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index 126b119..854177b 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -23,6 +23,7 @@ #include "access/xact.h" #include "access/xlog.h" #include "catalog/catalog.h" +#include "catalog/colstore.h" #include "catalog/dependency.h" #include "catalog/heap.h" #include "catalog/index.h" @@ -269,7 +270,8 @@ struct DropRelationCallbackState static void truncate_check_rel(Relation rel); static List *MergeAttributes(List *schema, List *supers, char relpersistence, - List **supOids, List **supconstr, int *supOidCount); + List **supOids, List **supconstr, int *supOidCount, + List **colstores); static bool MergeCheckConstraint(List *constraints, char *name, Node *expr); static void MergeAttributesIntoExisting(Relation child_rel, Relation parent_rel); static void MergeConstraintsIntoExisting(Relation child_rel, Relation parent_rel); @@ -459,6 +461,9 @@ DefineRelation(CreateStmt *stmt, char relkind, Oid ownerId, Oid tablespaceId; Relation rel; TupleDesc descriptor; + List *decl_cstores = NIL, + *inh_cstores = NIL, + *colstores; List *inheritOids; List *old_constraints; bool localHasOids; @@ -568,19 +573,43 @@ DefineRelation(CreateStmt *stmt, char relkind, Oid ownerId, ofTypeId = InvalidOid; /* + * Initialize the list of column stores with the ones provided in + * table constraint form. + */ + foreach(listptr, stmt->colstores) + { + ColumnStoreClause *clause = (ColumnStoreClause *) lfirst(listptr); + ColumnStoreClauseInfo *store = palloc(sizeof(ColumnStoreClauseInfo)); + + store->cstoreClause = clause; + store->attnum = InvalidAttrNumber; + store->attnums = NIL; + store->cstoreOid = InvalidOid; + + decl_cstores = lappend(decl_cstores, store); + } + + /* * Look up inheritance ancestors and generate relation schema, including - * inherited attributes. + * inherited attributes. Add column stores coming from parent rels. */ schema = MergeAttributes(schema, stmt->inhRelations, stmt->relation->relpersistence, - &inheritOids, &old_constraints, &parentOidCount); + &inheritOids, &old_constraints, &parentOidCount, + &inh_cstores); /* * Create a tuple descriptor from the relation schema. Note that this * deals with column names, types, and NOT NULL constraints, but not * default values or CHECK constraints; we handle those below. */ - descriptor = BuildDescForRelation(schema); + descriptor = BuildDescForRelation(schema, &decl_cstores); + + /* + * Determine the column stores we need. + */ + colstores = DetermineColumnStores(descriptor, decl_cstores, inh_cstores, + tablespaceId); /* * Notice that we allow OIDs here only for plain tables, even though some @@ -662,6 +691,7 @@ DefineRelation(CreateStmt *stmt, char relkind, Oid ownerId, descriptor, list_concat(cookedDefaults, old_constraints), + colstores, relkind, stmt->relation->relpersistence, false, @@ -1360,6 +1390,7 @@ storage_name(char c) * 'supconstr' receives a list of constraints belonging to the parents, * updated as necessary to be valid for the child. * 'supOidCount' is set to the number of parents that have OID columns. + * 'colstores' is appended ColumnStoreClauseInfo structs from parent rels. * * Return value: * Completed schema list. @@ -1405,7 +1436,8 @@ storage_name(char c) */ static List * MergeAttributes(List *schema, List *supers, char relpersistence, - List **supOids, List **supconstr, int *supOidCount) + List **supOids, List **supconstr, int *supOidCount, + List **colstores) { ListCell *entry; List *inhSchema = NIL; @@ -1503,6 +1535,8 @@ MergeAttributes(List *schema, List *supers, char relpersistence, TupleConstr *constr; AttrNumber *newattno; AttrNumber parent_attno; + List *pstores; + ListCell *cell; /* * A self-exclusive lock is needed here. If two backends attempt to @@ -1659,6 +1693,7 @@ MergeAttributes(List *schema, List *supers, char relpersistence, def->collClause = NULL; def->collOid = attribute->attcollation; def->constraints = NIL; + def->cstoreClause = NULL; def->location = -1; inhSchema = lappend(inhSchema, def); newattno[parent_attno - 1] = ++child_attno; @@ -1767,6 +1802,38 @@ MergeAttributes(List *schema, List *supers, char relpersistence, } } + /* + * Process column stores in the parent, using the completed + * newattno map. + */ + pstores = RelationGetColStoreList(relation); + foreach(cell, pstores) + { + Oid cstoreid = lfirst_oid(cell); + Relation storerel; + Form_pg_cstore cst; + ColumnStoreClauseInfo *cstinfo; + int i; + + /* AccessShare should be sufficient, since we hold lock on rel */ + storerel = relation_open(cstoreid, AccessShareLock); + cst = storerel->rd_cstore; + + cstinfo = palloc(sizeof(ColumnStoreClauseInfo)); + cstinfo->attnum = InvalidAttrNumber; + cstinfo->cstoreClause = NULL; + cstinfo->cstoreOid = RelationGetRelid(storerel); + cstinfo->attnums = NIL; + for (i = 0; i < cst->cstnatts; i++) + cstinfo->attnums = lappend_int(cstinfo->attnums, + newattno[cst->cstatts.values[i]-1]); + + relation_close(storerel, AccessShareLock); + + *colstores = lappend(*colstores, cstinfo); + } + list_free(pstores); + pfree(newattno); /* @@ -1854,6 +1921,8 @@ MergeAttributes(List *schema, List *supers, char relpersistence, storage_name(def->storage), storage_name(newdef->storage)))); + /* FIXME see about merging cstore decl here */ + /* Mark the column as locally defined */ def->is_local = true; /* Merge of NOT NULL constraints = OR 'em together */ diff --git a/src/backend/commands/view.c b/src/backend/commands/view.c index efa4be1..d7b702b 100644 --- a/src/backend/commands/view.c +++ b/src/backend/commands/view.c @@ -170,7 +170,7 @@ DefineVirtualRelation(RangeVar *relation, List *tlist, bool replace, * verify that the old column list is an initial prefix of the new * column list. */ - descriptor = BuildDescForRelation(attrList); + descriptor = BuildDescForRelation(attrList, NULL); checkViewTupleDesc(descriptor, rel->rd_att); /* diff --git a/src/include/access/tupdesc.h b/src/include/access/tupdesc.h index 91b0034..e1e96bc 100644 --- a/src/include/access/tupdesc.h +++ b/src/include/access/tupdesc.h @@ -123,7 +123,11 @@ extern void TupleDescInitEntryCollation(TupleDesc desc, AttrNumber attributeNumber, Oid collationid); -extern TupleDesc BuildDescForRelation(List *schema); +extern void TupleDescInitEntryColStore(TupleDesc desc, + AttrNumber attributeNumber, + Oid colstore); + +extern TupleDesc BuildDescForRelation(List *schema, List **colstores); extern TupleDesc BuildDescFromLists(List *names, List *types, List *typmods, List *collations); diff --git a/src/include/catalog/colstore.h b/src/include/catalog/colstore.h new file mode 100644 index 0000000..5dcc91a --- /dev/null +++ b/src/include/catalog/colstore.h @@ -0,0 +1,41 @@ +#ifndef COLSTORE_H +#define COLSTORE_H + +#include "nodes/nodes.h" +#include "nodes/parsenodes.h" +#include "nodes/pg_list.h" +#include "utils/relcache.h" + +/* + * When creating a table with column store declarations, this struct + * carries the necessary info. + * + * We're catering for several different cases here: (a) column store + * declarations as column constraints (attnum and cstoreClause are both set, + * attnums and cstoreOid are invalid), (b) declarations as table constraint + * (attnum is invalid but cstoreClause is set; attnums and cstoreOid are + * invalid), and (c) store definitions inherited from parent relations (attnum + * and cstoreClause are both invalid, attnums list the attribute numbers and + * cstoreOid is the OID of the pg_cstore entry of the column store for the + * parent relation. Note that the cstatts data from the parent's entry must be + * ignored in favor of the attnum list given here.) + */ +typedef struct ColumnStoreClauseInfo +{ + AttrNumber attnum; + ColumnStoreClause *cstoreClause; + List *attnums; + Oid cstoreOid; +} ColumnStoreClauseInfo; + + +extern List *DetermineColumnStores(TupleDesc tupdesc, List *decl_cstores, + List *inh_cstores, Oid tablespaceId); +extern void CreateColumnStores(Relation rel, List *colstores); +extern List *CloneColumnStores(Relation rel); +extern Oid get_relation_cstore_oid(Oid relid, const char *cstore_name, + bool missing_ok); +extern void RemoveColstoreById(Oid cstoreOid); +extern Oid GetColumnStoreAMByName(char *cstamname, bool missing_ok); + +#endif /* COLSTORE_H */ diff --git a/src/include/catalog/dependency.h b/src/include/catalog/dependency.h index fbcf904..71a28c3 100644 --- a/src/include/catalog/dependency.h +++ b/src/include/catalog/dependency.h @@ -138,6 +138,7 @@ typedef enum ObjectClass OCLASS_AMPROC, /* pg_amproc */ OCLASS_REWRITE, /* pg_rewrite */ OCLASS_TRIGGER, /* pg_trigger */ + OCLASS_COLSTORE, /* pg_cstore */ OCLASS_SCHEMA, /* pg_namespace */ OCLASS_TSPARSER, /* pg_ts_parser */ OCLASS_TSDICT, /* pg_ts_dict */ diff --git a/src/include/catalog/heap.h b/src/include/catalog/heap.h index e6ac394..27a065c 100644 --- a/src/include/catalog/heap.h +++ b/src/include/catalog/heap.h @@ -60,6 +60,7 @@ extern Oid heap_create_with_catalog(const char *relname, Oid ownerid, TupleDesc tupdesc, List *cooked_constraints, + List *colstores, char relkind, char relpersistence, bool shared_relation, @@ -95,6 +96,12 @@ extern void InsertPgClassTuple(Relation pg_class_desc, Datum relacl, Datum reloptions); +extern void AddNewAttributeTuples(Oid new_rel_oid, + TupleDesc tupdesc, + char relkind, + bool oidislocal, + int oidinhcount); + extern List *AddRelationNewConstraints(Relation rel, List *newColDefaults, List *newConstraints, diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index 302ba1f..7c790bf 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -1401,6 +1401,7 @@ typedef enum ObjectType OBJECT_CAST, OBJECT_COLUMN, OBJECT_COLLATION, + OBJECT_COLSTORE, OBJECT_CONVERSION, OBJECT_DATABASE, OBJECT_DEFAULT, -- 2.1.4