mirror/postgresql
2
0
Fork 0
mirror of https://git.postgresql.org/git/postgresql.git synced 2025-02-23 19:39:53 +08:00
Code Issues Packages Projects Releases Wiki Activity

Allow batching of inserts during cross-partition updates.

Browse Source 
Commit 927f453a9 disallowed batching added by commit b663a4136 to be
used for the inserts performed as part of cross-partition updates of
partitioned tables, mainly because the previous code in
nodeModifyTable.c couldn't handle pending inserts into foreign-table
partitions that are also UPDATE target partitions.  But we don't have
such a limitation anymore (cf. commit ffbb7e65a), so let's allow for
this by removing from execPartition.c the restriction added by commit
927f453a9 that batching is only allowed if the query command type is
CMD_INSERT.

In postgres_fdw, since commit 86dc90056 changed it to effectively
disable cross-partition updates in the case where a foreign-table
partition chosen to insert rows into is also an UPDATE target partition,
allow batching in the case where a foreign-table partition chosen to
do so is *not* also an UPDATE target partition.  This is enabled by the
"batch_size" option added by commit b663a4136, which is disabled by
default.

This patch also adjusts the test case added by commit 927f453a9 to
confirm that the inserts performed as part of a cross-partition update
of a partitioned table indeed uses batching.

Amit Langote, reviewed and/or tested by Georgios Kokolatos, Zhihong Yu,
Bharath Rupireddy, Hou Zhijie, Vignesh C, and me.

Discussion: http://postgr.es/m/CA%2BHiwqH1Lz1yJmPs%3DaD-pzd_HLLynLHvq5iYeT9mB0bBV7oJ6w%40mail.gmail.com
This commit is contained in:
Etsuro Fujita 2022-12-20 19:05:00 +09:00
parent 3226f47282
commit 594f8d3776
4 changed files with 106 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

64
contrib/postgres_fdw/expected/postgres_fdw.out
View File

@ -10347,8 +10347,12 @@ SELECT COUNT(*) FROM batch_table;
66
(1 row)
-- Check that enabling batched inserts doesn't interfere with cross-partition
-- updates
-- Clean up
DROP TABLE batch_table;
DROP TABLE batch_table_p0;
DROP TABLE batch_table_p1;
-- Check that batched mode also works for some inserts made during
-- cross-partition updates
CREATE TABLE batch_cp_upd_test (a int) PARTITION BY LIST (a);
CREATE TABLE batch_cp_upd_test1 (LIKE batch_cp_upd_test);
CREATE FOREIGN TABLE batch_cp_upd_test1_f
@ -10356,21 +10360,59 @@ CREATE FOREIGN TABLE batch_cp_upd_test1_f
FOR VALUES IN (1)
SERVER loopback
OPTIONS (table_name 'batch_cp_upd_test1', batch_size '10');
CREATE TABLE batch_cp_up_test1 PARTITION OF batch_cp_upd_test
CREATE TABLE batch_cp_upd_test2 PARTITION OF batch_cp_upd_test
FOR VALUES IN (2);
INSERT INTO batch_cp_upd_test VALUES (1), (2);
-- The following moves a row from the local partition to the foreign one
UPDATE batch_cp_upd_test t SET a = 1 FROM (VALUES (1), (2)) s(a) WHERE t.a = s.a;
ERROR: cannot route tuples into foreign table to be updated "batch_cp_upd_test1_f"
SELECT tableoid::regclass, * FROM batch_cp_upd_test;
CREATE TABLE batch_cp_upd_test3 (LIKE batch_cp_upd_test);
CREATE FOREIGN TABLE batch_cp_upd_test3_f
PARTITION OF batch_cp_upd_test
FOR VALUES IN (3)
SERVER loopback
OPTIONS (table_name 'batch_cp_upd_test3', batch_size '1');
-- Create statement triggers on remote tables that "log" any INSERTs
-- performed on them.
CREATE TABLE cmdlog (cmd text);
CREATE FUNCTION log_stmt() RETURNS TRIGGER LANGUAGE plpgsql AS $$
BEGIN INSERT INTO public.cmdlog VALUES (TG_OP || ' on ' || TG_RELNAME); RETURN NULL; END;
$$;
CREATE TRIGGER stmt_trig AFTER INSERT ON batch_cp_upd_test1
FOR EACH STATEMENT EXECUTE FUNCTION log_stmt();
CREATE TRIGGER stmt_trig AFTER INSERT ON batch_cp_upd_test3
FOR EACH STATEMENT EXECUTE FUNCTION log_stmt();
-- This update moves rows from the local partition 'batch_cp_upd_test2' to the
-- foreign partition 'batch_cp_upd_test1', one that has insert batching
-- enabled, so a single INSERT for both rows.
INSERT INTO batch_cp_upd_test VALUES (2), (2);
UPDATE batch_cp_upd_test t SET a = 1 FROM (VALUES (1), (2)) s(a) WHERE t.a = s.a AND s.a = 2;
-- This one moves rows from the local partition 'batch_cp_upd_test2' to the
-- foreign partition 'batch_cp_upd_test2', one that has insert batching
-- disabled, so separate INSERTs for the two rows.
INSERT INTO batch_cp_upd_test VALUES (2), (2);
UPDATE batch_cp_upd_test t SET a = 3 FROM (VALUES (1), (2)) s(a) WHERE t.a = s.a AND s.a = 2;
SELECT tableoid::regclass, * FROM batch_cp_upd_test ORDER BY 1;
tableoid | a
----------------------+---
batch_cp_upd_test1_f | 1
batch_cp_up_test1 | 2
(2 rows)
batch_cp_upd_test1_f | 1
batch_cp_upd_test3_f | 3
batch_cp_upd_test3_f | 3
(4 rows)
-- Should see 1 INSERT on batch_cp_upd_test1 and 2 on batch_cp_upd_test3 as
-- described above.
SELECT * FROM cmdlog ORDER BY 1;
cmd
------------------------------
INSERT on batch_cp_upd_test1
INSERT on batch_cp_upd_test3
INSERT on batch_cp_upd_test3
(3 rows)
-- Clean up
DROP TABLE batch_table, batch_cp_upd_test, batch_table_p0, batch_table_p1 CASCADE;
DROP TABLE batch_cp_upd_test;
DROP TABLE batch_cp_upd_test1;
DROP TABLE batch_cp_upd_test3;
DROP TABLE cmdlog;
DROP FUNCTION log_stmt();
-- Use partitioning
ALTER SERVER loopback OPTIONS (ADD batch_size '10');
CREATE TABLE batch_table ( x int, field1 text, field2 text) PARTITION BY HASH (x);

10
contrib/postgres_fdw/postgres_fdw.c
View File

@ -2017,16 +2017,16 @@ static int
postgresGetForeignModifyBatchSize(ResultRelInfo *resultRelInfo)
{
int batch_size;
PgFdwModifyState *fmstate = resultRelInfo->ri_FdwState ?
(PgFdwModifyState *) resultRelInfo->ri_FdwState :
NULL;
PgFdwModifyState *fmstate = (PgFdwModifyState *) resultRelInfo->ri_FdwState;
/* should be called only once */
Assert(resultRelInfo->ri_BatchSize == 0);
/*
* Should never get called when the insert is being performed as part of a
* row movement operation.
* Should never get called when the insert is being performed on a table
* that is also among the target relations of an UPDATE operation,
* because postgresBeginForeignInsert() currently rejects such insert
* attempts.
*/
Assert(fmstate == NULL || fmstate->aux_fmstate == NULL);

55
contrib/postgres_fdw/sql/postgres_fdw.sql
View File

@ -3329,8 +3329,13 @@ CREATE TABLE batch_table_p2
INSERT INTO batch_table SELECT * FROM generate_series(1, 66) i;
SELECT COUNT(*) FROM batch_table;
-- Check that enabling batched inserts doesn't interfere with cross-partition
-- updates
-- Clean up
DROP TABLE batch_table;
DROP TABLE batch_table_p0;
DROP TABLE batch_table_p1;
-- Check that batched mode also works for some inserts made during
-- cross-partition updates
CREATE TABLE batch_cp_upd_test (a int) PARTITION BY LIST (a);
CREATE TABLE batch_cp_upd_test1 (LIKE batch_cp_upd_test);
CREATE FOREIGN TABLE batch_cp_upd_test1_f
@ -3338,16 +3343,50 @@ CREATE FOREIGN TABLE batch_cp_upd_test1_f
FOR VALUES IN (1)
SERVER loopback
OPTIONS (table_name 'batch_cp_upd_test1', batch_size '10');
CREATE TABLE batch_cp_up_test1 PARTITION OF batch_cp_upd_test
CREATE TABLE batch_cp_upd_test2 PARTITION OF batch_cp_upd_test
FOR VALUES IN (2);
INSERT INTO batch_cp_upd_test VALUES (1), (2);
CREATE TABLE batch_cp_upd_test3 (LIKE batch_cp_upd_test);
CREATE FOREIGN TABLE batch_cp_upd_test3_f
PARTITION OF batch_cp_upd_test
FOR VALUES IN (3)
SERVER loopback
OPTIONS (table_name 'batch_cp_upd_test3', batch_size '1');
-- The following moves a row from the local partition to the foreign one
UPDATE batch_cp_upd_test t SET a = 1 FROM (VALUES (1), (2)) s(a) WHERE t.a = s.a;
SELECT tableoid::regclass, * FROM batch_cp_upd_test;
-- Create statement triggers on remote tables that "log" any INSERTs
-- performed on them.
CREATE TABLE cmdlog (cmd text);
CREATE FUNCTION log_stmt() RETURNS TRIGGER LANGUAGE plpgsql AS $$
BEGIN INSERT INTO public.cmdlog VALUES (TG_OP || ' on ' || TG_RELNAME); RETURN NULL; END;
$$;
CREATE TRIGGER stmt_trig AFTER INSERT ON batch_cp_upd_test1
FOR EACH STATEMENT EXECUTE FUNCTION log_stmt();
CREATE TRIGGER stmt_trig AFTER INSERT ON batch_cp_upd_test3
FOR EACH STATEMENT EXECUTE FUNCTION log_stmt();
-- This update moves rows from the local partition 'batch_cp_upd_test2' to the
-- foreign partition 'batch_cp_upd_test1', one that has insert batching
-- enabled, so a single INSERT for both rows.
INSERT INTO batch_cp_upd_test VALUES (2), (2);
UPDATE batch_cp_upd_test t SET a = 1 FROM (VALUES (1), (2)) s(a) WHERE t.a = s.a AND s.a = 2;
-- This one moves rows from the local partition 'batch_cp_upd_test2' to the
-- foreign partition 'batch_cp_upd_test2', one that has insert batching
-- disabled, so separate INSERTs for the two rows.
INSERT INTO batch_cp_upd_test VALUES (2), (2);
UPDATE batch_cp_upd_test t SET a = 3 FROM (VALUES (1), (2)) s(a) WHERE t.a = s.a AND s.a = 2;
SELECT tableoid::regclass, * FROM batch_cp_upd_test ORDER BY 1;
-- Should see 1 INSERT on batch_cp_upd_test1 and 2 on batch_cp_upd_test3 as
-- described above.
SELECT * FROM cmdlog ORDER BY 1;
-- Clean up
DROP TABLE batch_table, batch_cp_upd_test, batch_table_p0, batch_table_p1 CASCADE;
DROP TABLE batch_cp_upd_test;
DROP TABLE batch_cp_upd_test1;
DROP TABLE batch_cp_upd_test3;
DROP TABLE cmdlog;
DROP FUNCTION log_stmt();
-- Use partitioning
ALTER SERVER loopback OPTIONS (ADD batch_size '10');

3
src/backend/executor/execPartition.c
View File

@ -1018,8 +1018,7 @@ ExecInitRoutingInfo(ModifyTableState *mtstate,
*
* If the FDW does not support batching, we set the batch size to 1.
*/
if (mtstate->operation == CMD_INSERT &&
partRelInfo->ri_FdwRoutine != NULL &&
if (partRelInfo->ri_FdwRoutine != NULL &&
partRelInfo->ri_FdwRoutine->GetForeignModifyBatchSize &&
partRelInfo->ri_FdwRoutine->ExecForeignBatchInsert)
partRelInfo->ri_BatchSize =