postgresql/contrib/pg_upgrade/pg_upgrade

#!/bin/sh
#
# pg_upgrade: update a database without needing a full dump/reload cycle.
# CAUTION: Read the manual page before trying to use this!

# $Header: /cvsroot/pgsql/contrib/pg_upgrade/Attic/pg_upgrade,v 1.15 2002/08/29 22:19:03 petere Exp $
#
# To migrate this to newer versions of PostgreSQL:  
#	1)  Update the version numbers at the top of the file
#	2)  Search for specific version mentions in the script and update
#	    accordingly.
#	3)  Add changes for next version.

#set -x

# UPGRADE_VERSION is the expected old database version
UPGRADE_VERSION="7.3"
CUR_VERSION="7.3"

# Set this to "Y" to enable this program
ENABLE="Y"

if [ "$ENABLE" != "Y" ]
then
	echo "Sorry, $0 cannot upgrade database version $SRC_VERSION to $DST_VERSION." 1>&2
	echo "The on-disk structure of tables has changed." 1>&2
	echo "You will need to dump and restore using pg_dumpall." 1>&2
	exit 1
fi


trap "rm -f /tmp/$$.*" 0 1 2 3 15

BASENAME=`basename "$0"`
PHASE=""

while [ "$#" -ne 0 ]
do
	if [ "X$1" = "X-1" ]
	then	PHASE="1"
		shift
	elif [ "X$1" = "X-2" ]
	then	PHASE="2"
		shift
	elif [ "X$1" = "X-D" ]
	then	PGDATA="$2"
		shift 2
	fi
done

if [ "$PHASE" = "" ]
then	echo "You must run $BASENAME in either mode 1 or mode 2." 1>&2
	echo "Usage:  $BASENAME [-D datadir] -1 | -2" 1>&2
	exit 1
fi

if [ "$PGDATA" = "" ]
then	echo "You must set the PGDATA environment variable or specify it with -D." 1>&2
	echo "Usage:  $BASENAME [-D datadir] -1 | -2" 1>&2
	exit 1
fi

if [ ! -d "$PGDATA" ]
then	echo "$PGDATA does not exist.  Exiting." 1>&2
	if [ "$PHASE" -eq 2 ]
	then	echo "Perhaps you didn't run initdb." 1>&2
	fi
	exit 1
fi

if [ "$USER" = "root" -o ! -r "$PGDATA"/PG_VERSION ]
then	echo "You must run this as the PostgreSQL superuser.  Exiting." 1>&2
	exit 1
fi

# Strip off the trailing directory name and store our data there
# in the hope we are in the same filesystem so 'mv 'works.

INFODIR=`dirname "$PGDATA"`/pg_upgrade_info
SAVEDATA="$INFODIR"/data

make_dbobjoidmap()
{
	psql -d template1 -At -c "SELECT datname FROM pg_database" |
	grep -v '^template0$' |
	while read DB
	do	
		QUERY="SELECT 	relname, oid
			FROM 	pg_class
			WHERE 	relkind = 'r' OR
				relkind = 'i' OR
				relkind = 'S' OR
				relkind = 't';"

		psql -d "$DB" -At -F'	' -c "$QUERY" |
		while read RELNAME_OID
		do	
			echo "$DB	$RELNAME_OID"
		done
	done
}


make_dboidmap()
{
	psql -d template1 -At -F'	' -c \
		'SELECT datname, oid FROM pg_database;' |
	grep -v '^template0$'
}


move_objfiles()
{
	# Test to make sure there is a matching file in each place

	if [ ! -f "$SAVEDATA"/base/"$SRC_DBOID"/"$SRC_OID" -a \
	     ! -h "$SAVEDATA"/base/"$SRC_DBOID"/"$SRC_OID" ]
	then	echo "Moving of database $DB, OID $SRC_OID, object $OBJ failed." 1>&2
		echo "File not found.  Exiting." 1>&2
		return 1
	fi

	if [ ! -f "$PGDATA"/base/"$DST_DBOID"/"$DST_OID" -a \
	     ! -h "$PGDATA"/base/"$DST_DBOID"/"$DST_OID" ]
	then	echo "Moving of database $DB, OID $DST_OID, object $OBJ failed." 1>&2
		echo "File not found.  Exiting." 1>&2
		return 1
	fi

	# Move files

	mv -f "$SAVEDATA"/base/"$SRC_DBOID"/"$SRC_OID" "$PGDATA"/base/"$DST_DBOID"/"$DST_OID"
	if [ "$?" -ne 0 ]
	then	echo "Moving of database $DB, OID $SRC_OID, object $OBJ" 1>&2
		echo "to $DST_OID failed.  Exiting" 1>&2
		return 1
	fi

	# handle table extents

	ls "$SAVEDATA"/base/"$SRC_DBOID"/"$SRC_OID".* 2> /dev/null | while read FILE
	do
		EXT=`basename "$FILE" | sed 's/^.*\.\(.*\)$/\1/'`
		mv -f "$FILE" "$PGDATA"/base/"$DST_DBOID"/"$DST_OID"."$EXT"
		if [ "$?" -ne 0 ]
		then	echo "Moving of database $DB, OID $SRC_OID, object $OBJ" 1>&2
			echo "to $DST_OID failed.  Exiting." 1>&2
			return 1
		fi
	done
}

if [ "$PHASE" -eq 1 ]
then

	##########################
	#  Phase 1 starts here   #
	##########################


	if [ ! -d "$PGDATA"/base/1 ]
	then	echo "There is no database template1 in $PGDATA/base." 1>&2
		exit 1
	fi

	# get version
	SRC_VERSION=`cat "$PGDATA"/PG_VERSION`
	if [ "$SRC_VERSION" = "" ]
	then	echo "$BASENAME can not find the PostgreSQL version file" 1>&2
		echo "'$PGDATA/PG_VERSION'.  Exiting." 1>&2
		exit 1
	fi

	if [ 	"$SRC_VERSION" != "$CUR_VERSION" -a \
		"$SRC_VERSION" != "$UPGRADE_VERSION" ]
	then	echo "$BASENAME supports versions $UPGRADE_VERSION and $CUR_VERSION only." 1>&2
		echo "However, your database is version $SRC_VERSION." 1>&2
		echo "You will need to dump and restore using pg_dumpall.  Exiting." 1>&2
		exit 1
	fi

	# If server is down, start it so we can do some work.
	if ! pg_ctl status | sed -n '1p' | grep "is running" > /dev/null 2>&1
	then	pg_ctl -w start
		if [ $? -ne 0 ]
		then	echo "Can not start server.  Exiting." 1>&2
			exit 1
		fi
	fi

	# create directory for our data
	if ! rm -rf "$INFODIR" 
	then	echo "Deletion of old pg_upgrade_info directory $INFODIR failed." 1>&2
		echo "Exiting." 1>&2
		exit 1
	fi
	if ! mkdir "$INFODIR"
	then	echo "Creation of new pg_upgrade_info directory $INFODIR failed." 1>&2
		echo "Exiting." 1>&2
		exit 1
	fi

	if ! chmod og-rwx "$INFODIR"
 	then	echo "Permission change on new pg_upgrade_info directory $INFODIR failed." 1>&2
		echo "Exiting." 1>&2
		exit 1
	fi


	# Dump schema
	pg_dumpall -s > "$INFODIR"/schema
	if [ $? -ne 0 ]
	then	echo "Can not dump schema.  Exiting." 1>&2
		exit 1
	fi
	
	# Generate mappings for database
	make_dboidmap > "$INFODIR"/dboidmap || exit "$?"
	make_dbobjoidmap > "$INFODIR"/dbobjoidmap || exit "$?"

	# Vacuum all databases to remove exipired rows.
	# We will lose our transaction log file during the upgrade so we
	# have to do this.

	vacuumdb -a
	if [ $? -ne 0 ]
	then	echo "Can not vacuum server.  Exiting." 1>&2
		exit 1
	fi

	# Stop server so we can move the directory.
	pg_ctl -w stop
	if [ $? -ne 0 ]
	then	echo "Can not stop server.  Exiting." 1>&2
		exit 1
	fi
	
	# No matter what the directory name, call it data
	mv "$PGDATA" "$INFODIR"/data
	if [ $? -ne 0 ]
	then	echo "Can not move old $PGDATA out of the way.  Exiting." 1>&2
		exit 1
	fi
	echo
	echo
	echo "$BASENAME phase 1 completed."
	echo "Continue with the steps outlined in the $BASENAME manual page."
	exit 0
fi


	##########################
	#  Phase 2 starts here   #
	##########################

# check things

if [ ! -d "$INFODIR" ]
then	echo "There is no '$INFODIR' directory from the phase 1 run of $BASENAME." 1>&2
	exit 1	
fi

if [ ! -d "$SAVEDATA" ]
then	echo "There is no '$SAVEDATA' directory from the phase 1 run of $BASENAME." 1>&2
	exit 1	
fi

if [ ! -f "$SAVEDATA/PG_VERSION" ]
then	echo "Cannot read '$SAVEDATA/PG_VERSION' --- something is wrong." 1>&2
	exit 1
fi

if [ ! -f "$PGDATA/PG_VERSION" ]
then	echo "Cannot read '$PGDATA/PG_VERSION' --- something is wrong." 1>&2
	exit 1
fi

if [ ! -d "$PGDATA/base/1" ]
then	echo "Cannot find database template1 in '$PGDATA/base'." 1>&2
	echo "Are you running $BASENAME as the postgres superuser?" 1>&2
	exit 1
fi

# Get the actual versions seen in the data dirs.

SRC_VERSION=`cat "$SAVEDATA"/PG_VERSION`
DST_VERSION=`cat "$PGDATA"/PG_VERSION`

# Check for version compatibility.
# This code will need to be updated/reviewed for each new PostgreSQL release.

if [ "$DST_VERSION" != "$CUR_VERSION" ]
then	echo "$BASENAME is for PostgreSQL version $CUR_VERSION" 1>&2
	echo "but $PGDATA/PG_VERSION contains $DST_VERSION." 1>&2
	echo "Did you run initdb for version $UPGRADE_VERSION by mistake?" 1>&2
	exit 1
fi

# Stop server for pg_resetxlog use

if pg_ctl status | sed -n '1p' | grep "is running" > /dev/null 2>&1
then	pg_ctl -w stop
	if [ $? -ne 0 ]
	then	echo "Can not start server.  Exiting." 1>&2
		exit 1
	fi
fi

# check for proper pg_resetxlog version

pg_resetxlog 2> /dev/null
# file not found status is normally 127, not 1
if [ "$?" -ne 1 ]
then	echo "Unable to find pg_resetxlog in your path." 1>&2
	echo "Install it from pgsql/contrib/pg_resetxlog and continue.  Exiting." 1>&2
	exit 1
fi

if ! pg_resetxlog -x 2>&1 | grep 'xid' > /dev/null 2>&1
then	echo "Old version of pg_resetxlog found in path." 1>&2
	echo "Install a newer version of pg_resetxlog from pgsql/contrib/pg_resetxlog." 1>&2
	echo "Exiting." 1>&2
	exit 1
fi

SRC_XID=`pg_resetxlog -n "$SAVEDATA" | grep "NextXID" | awk -F'  *' '{print $4}'`
DST_XID=`pg_resetxlog -n "$PGDATA" | grep "NextXID" | awk -F'  *' '{print $4}'`

# compare locales to make sure they match

pg_resetxlog -n "$SAVEDATA" | grep "^LC_" > /tmp/$$.0
pg_resetxlog -n "$PGDATA" | grep "^LC_" > /tmp/$$.1
if ! diff /tmp/$$.0 /tmp/$$.1 > /dev/null
then	echo "Locales do not match between the two versions.  Exiting." 1>&2
	exit 1
fi

# Restart postmaster

pg_ctl -w start
if [ $? -ne 0 ]
then	echo "Can not start server.  Exiting." 1>&2
	exit 1
fi


###################################
# Checking done.  Ready to proceed.
###################################


# Execute the schema script to create everything

psql template1 < "$INFODIR"/schema
if [ $? -ne 0 ]
then	echo "There were errors in the input script.  Exiting." 1>&2
	exit 1
fi

echo "Input script completed, fixing row commit statuses..."

# Generate mappings for new database
make_dboidmap > /tmp/$$.dboidmap || exit "$?"
make_dbobjoidmap > /tmp/$$.dbobjoidmap || exit "$?"

# we are done with SQL database access
# shutdown forces buffers to disk

pg_ctl -w stop
if [ "$?" -ne 0 ]
then	echo "Unable to stop database server.  Exiting." 1>&2
	exit 1
fi

echo "Commit fixes complete, moving data files..."

# Move table/index/sequence files

cat "$INFODIR"/dbobjoidmap | while read LINE
do
	DB=`echo "$LINE" | awk '{print $1}'`
	OBJ=`echo "$LINE" | awk '{print $2}'`

	# Skip system tables, except for pg_largeobject
	# pg_toast tables are handled later as part of the
	# base table move
	if [ `expr X"$OBJ" : X'pg_'` -eq 4 -a \
	     `expr X"$OBJ" : X'pg_largeobject'` -ne 15 ]
	then	continue
	fi

	SRC_OID=`echo "$LINE" | awk '{print $3}'`
	SRC_DBOID=`grep "^$DB	" "$INFODIR"/dboidmap | awk '{print $2}'`
	DST_DBOID=`grep "^$DB	" /tmp/$$.dboidmap | awk '{print $2}'`
	DST_OID=`grep "^$DB	$OBJ	" /tmp/$$.dbobjoidmap | awk '{print $3}'`

	move_objfiles

	# Handle TOAST files if they exist
	if grep "^$DB	pg_toast_$SRC_OID	" "$INFODIR"/dbobjoidmap \
		> /dev/null 2>&1
	then	# toast heap
		SAVE_SRC_OID="$SRC_OID"
		SAVE_DST_OID="$DST_OID"
		SRC_OID=`grep "^$DB	pg_toast_$SAVE_SRC_OID	" \
			"$INFODIR"/dbobjoidmap | awk '{print $3}'`
		DST_OID=`grep "^$DB	pg_toast_$SAVE_DST_OID	" \
			/tmp/$$.dbobjoidmap | awk '{print $3}'`
		move_objfiles
		# toast index
		SRC_OID=`grep "^$DB	pg_toast_${SAVE_SRC_OID}_idx	" \
			"$INFODIR"/dbobjoidmap | awk '{print $3}'`
		DST_OID=`grep "^$DB	pg_toast_${SAVE_DST_OID}_idx	" \
			/tmp/$$.dbobjoidmap | awk '{print $3}'`
		move_objfiles
	fi
done
								   

# Set this so future backends don't think these tuples are their own
# because it matches their own XID.
# Commit status already updated by vacuum above
# Set to maximum XID just in case SRC wrapped around recently and
# is lower than DST's database

if [ "$SRC_XID" -gt "$DST_XID" ]
then	MAX_XID="$SRC_XID"
else	MAX_XID="$DST_XID"
fi

pg_resetxlog -x "$MAX_XID" "$PGDATA"
if [ "$?" -ne 0 ]
then	echo "Unable to set new XID.  Exiting." 1>&2
	exit 1
fi

# Move over old WAL

rm -r "$PGDATA"/pg_xlog
mv -f "$SAVEDATA"/pg_xlog "$PGDATA"

# Move over old clog

rm -r "$PGDATA"/pg_clog
mv -f "$SAVEDATA"/pg_clog "$PGDATA"

# Set last log file id and segment from old database

LOG_ID=`pg_resetxlog -n "$SAVEDATA" | grep "Current log file id:" |
	awk -F'  *' '{print $5}'`
if [ "$LOG_ID" = "" ]
then	echo "Unable to get old log file id.  Exiting." 1>&2
	exit 1
fi
SEG_ID=`pg_resetxlog -n "$SAVEDATA" | grep "Next log file segment:" |
	awk -F'  *' '{print $5}'`
if [ "$SEG_ID" = "" ]
then	echo "Unable to get old log segment id.  Exiting." 1>&2
	exit 1
fi

# Set checkpoint location of new database

pg_resetxlog -l "$LOG_ID","$SEG_ID" "$PGDATA"
if [ "$?" -ne 0 ]
then	echo "Unable to set new log file/segment id.  Exiting." 1>&2
	exit 1
fi

# Restart server with moved data

pg_ctl -w start
if [ "$?" -ne 0 ]
then	echo "Unable to restart database server.  Exiting." 1>&2
	exit 1
fi

# Now that we have moved the WAL/transaction log files, vacuum again to
# mark install rows with fixed transaction ids to prevent problems on xid
# wraparound.

vacuumdb -a
if [ $? -ne 0 ]
then	echo "There were errors during VACUUM.  Exiting." 1>&2
	exit 1
fi

echo
echo
echo "$BASENAME phase 2 completed."
echo "You may remove the old database files with 'rm -r $INFODIR'."
exit 0