hdf5/test/test_swmr.pwsh.in

# Copyright by The HDF Group.
# All rights reserved.
#
# This file is part of HDF5.  The full HDF5 copyright notice, including
# terms governing use, modification, and redistribution, is contained in
# the COPYING file, which can be found at the root of the source code
# distribution tree, or in https://www.hdfgroup.org/licenses.
# If you do not have access to either file, you may request a copy from
# help@hdfgroup.org.
#
# Tests for the swmr feature.

$srcdir  = '@srcdir@'
$utils_testdir=@H5_UTILS_TEST_BUILDDIR@
$testdir=@H5_TEST_BUILDDIR@

###############################################################################
## test parameters
###############################################################################

$Nreaders     = 5          # number of readers to launch
$Nrdrs_spa    = 3          # number of sparse readers to launch
$Nrecords     = 200000     # number of records to write
$Nrecs_rem    = 40000      # number of times to shrink
$Nrecs_spa    = 20000      # number of records to write in the sparse test
$Nsecs_add    = 5          # number of seconds per read interval
$Nsecs_rem    = 3          # number of seconds per read interval
$Nsecs_addrem = 8          # number of seconds per read interval
$nerrors      = 0

###############################################################################
## definitions for message file to coordinate test runs
###############################################################################
$WRITER_MESSAGE = 'SWMR_WRITER_MESSAGE'    # The message file created by writer that the open is complete
                                           # This should be the same as the define in "test/swmr_common.h"
$MESSAGE_TIMEOUT = 300                     # Message timeout length in secs
                                           # This should be the same as the define in "test/h5test.h"

# Print a line-line message left justified in a field of 70 characters
# beginning with the word "Testing".
#
function Test-WithSpaces {
   $SPACES='                                                               '
   #Write-Output  "Testing $* $SPACES" | cut -c1-70 | tr -d '\012'
}

# To wait for the writer message file or till the maximum # of seconds is reached
# $Message is the message file to wait for
# This performs similar function as the routine h5_wait_message() in test/h5test.c
function Wait-Message {
    [CmdletBinding()]
    param (
        [Parameter(Mandatory, ValueFromPipeline)]
        [string]$TestPath,                             # Get the path of the message file to wait for
        [Parameter(Mandatory, ValueFromPipeline)]
        [string]$Message                              # Get the name of the message file to wait for
    )

    PROCESS {
        $message = Join-Path -Path $TestPath -ChildPath $Message
        $t0 = Get-Date                                # Get current time
        $t1 = $t0
        $difft = New-Timespan -Start $t0 -End $t1     # Initialize the time difference
        $mexist = 0                                   # Indicate whether the message file is found
        while ($difft.TotalSeconds -lt $MESSAGE_TIMEOUT) {    # Loop till message times out
            $t1 = Get-Date                                    # Get current time in seconds
            $difft = New-Timespan -Start $t0 -End $t1         # Calculate the time difference
            #Write-Output "Check for $message : time=$difft"
            if ([System.IO.File]::Exists($message)) {         # If message file is found:
                $mexist = 1                                   #       indicate the message file is found
                Remove-Item $message                          #       remove the message file
                break                                         #       get out of the while loop
            }
        }
        if ($mexist -eq 0) {
            # Issue warning that the writer message file is not found, continue with launching the reader(s)
            Write-Warning -Message "$WRITER_MESSAGE is not found after waiting $MESSAGE_TIMEOUT seconds"
        }
        else {
            Write-Output "$WRITER_MESSAGE is found"
        }
    }
}

###############################################################################
## Main
##
## Modifications:
##   Vailin Choi; July 2013
##     Add waiting of message file before launching the reader(s).
##     Due to the implementation of file locking, coordination
##     is needed in file opening for the writer/reader tests
##     to proceed as expected.
##
###############################################################################

# Check to see if the VFD specified by the HDF5_DRIVER environment variable
# supports SWMR.
$testprog = Join-Path -Path $utils_testdir -ChildPath swmr_check_compat_vfd.exe
$rp = Start-Process -FilePath $testprog -PassThru -Wait -NoNewWindow
if ($rp.ExitCode -ne 0) {
    Write-Output ""
    Write-Output "The VFD specified by the HDF5_DRIVER environment variable"
    Write-Output "does not support SWMR."
    Write-Output ""
    Write-Output "SWMR acceptance tests skipped"
    Write-Output ""
    exit 0
}

Set-Location -Path swmr_test
$testdir = Join-Path -Path $testdir -ChildPath swmr_test

# Loop over index types
$IndexTypeLoop = '-i ea', '-i b2'
foreach ($index_type in $IndexTypeLoop) {
    # Try with and without compression
    $CompressLoop = '', '-c 5'
    foreach ($compress in $CompressLoop) {
        Write-Output ""
        Write-Output "*******************************************************************************"
        Write-Output "** Loop testing parameters: $index_type $compress"
        Write-Output "*******************************************************************************"
        Write-Output ""
        Write-Output ""
        Write-Output "###############################################################################"
        Write-Output "## Generator test"
        Write-Output "###############################################################################"
        # Launch the Generator without SWMR_WRITE
        Write-Output "launch the swmr_generator"
        $rp = Start-Process -FilePath $testdir/swmr_generator.exe -NoNewWindow -ArgumentList "$compress $index_type" -PassThru -Wait
        if ($rp.ExitCode -ne 0) {
            Write-Warning "generator had error"
            $nerrors += 1
        }

        # Launch the Generator with SWMR_WRITE
        Write-Output "launch the swmr_generator with SWMR_WRITE"
        $rp = Start-Process -FilePath $testdir/swmr_generator.exe -NoNewWindow -ArgumentList "-s $compress $index_type" -PassThru -Wait
        if ($rp.ExitCode -ne 0) {
            Write-Warning "generator had error"
            $nerrors += 1
        }

        # Check for error and exit if one occurred
        #Write-Output "nerrors=$nerrors"
        if ($nerrors -ne 0) {
            Write-Warning "SWMR tests failed with $nerrors errors."
            exit 1
        }

        Write-Output ""
        Write-Output "###############################################################################"
        Write-Output "## Use H5Fstart_swmr_write() to enable SWMR writing mode"
        Write-Output "###############################################################################"

        # Remove any possible writer message file before launching writer
        if ([System.IO.File]::Exists($WRITER_MESSAGE)) {                # If message file is found
            Remove-Item $WRITER_MESSAGE
        }
        #
        # Launch the Writer
        Write-Output "launch the swmr_start_writer"
        $seed = "" # Put -r <random seed> command here
        $rp = Start-Process -FilePath $testdir/swmr_start_write.exe -NoNewWindow -PassThru -ArgumentList "$compress $index_type $Nrecords $seed" 2>&1 |tee swmr_writer.out
        $pid_writer = $rp.id
        #Write-Output "pid_writer=$pid_writer"

        # Wait for message from writer process before starting reader(s)
        Wait-Message $testdir $WRITER_MESSAGE

        #
        # Launch the Readers
        #declare -a seeds = (<seed1> <seed2> <seed3> ... )
        Write-Output "launch $Nreaders swmr_readers"
        $pid_readers = @()
        $n = 0
        while ($n -lt $Nreaders) {
            #seed = "-r ${seeds[$n]}"
            $seed = ""
            $rp = Start-Process -FilePath $testdir/swmr_reader.exe -NoNewWindow -PassThru -ArgumentList "$Nsecs_add $seed" 2>&1 | tee swmr_reader.out.$n
            $pid_readers +=  $rp.id
            $n += 1
        }
        #Write-Output "pid_readers=$pid_readers"

        # Collect exit code of the readers first because they usually finish
        # before the writer.
        foreach ($xpid in $pid_readers) {
            #Write-Output "checked reader $xpid"
            $result = Wait-Process -Id  $xpid
            if ($result.ExitCode -ne 0) {
                Write-Warning "reader had error"
                $nerrors += 1
            }
        }

        # Collect exit code of the writer
        #Write-Output "checked writer $pid_writer"
        $result = Wait-Process -Id  $pid_writer
        if ($result.ExitCode -ne 0) {
            Write-Warning "writer had error"
            $nerrors += 1
        }

        # Check for error and exit if one occurred
        #Write-Output "nerrors=$nerrors"
        if ($nerrors -ne 0) {
            Write-Warning "SWMR tests failed with $nerrors errors."
            Write-Warning "(Writer and reader output preserved)"
            exit 1
        }

        # Clean up output files
        Remove-Item swmr_writer.out
        Remove-Item swmr_reader.out.*

        Write-Output ""
        Write-Output "###############################################################################"
        Write-Output "## Writer test - test expanding the dataset"
        Write-Output "###############################################################################"

        # Launch the Generator
        Write-Output "launch the swmr_generator"
        $rp = Start-Process -FilePath $testdir/swmr_generator.exe -NoNewWindow -PassThru -Wait -ArgumentList "-s $compress $index_type"
        if ($rp.ExitCode -ne 0) {
            Write-Warning "generator had error"
            $nerrors += 1
        }

        # Remove any possible writer message file before launching writer
        Remove-Item $WRITER_MESSAGE
        #
        # Launch the Writer
        Write-Output "launch the swmr_writer"
        $seed = "" # Put -r <random seed> command here
        $rp = Start-Process -FilePath $testdir/swmr_writer.exe -NoNewWindow -PassThru -ArgumentList "-o $Nrecords $seed" 2>&1 |tee swmr_writer.out
        $pid_writer = $rp.id
        #Write-Output "pid_writer=$pid_writer"

        # Wait for message from writer process before starting reader(s)
        Wait-Message $testdir $WRITER_MESSAGE
        #
        # Launch the Readers
        #declare -a seeds = (<seed1> <seed2> <seed3> ... )
        Write-Output "launch $Nreaders swmr_readers"
        $pid_readers = @()
        $n = 0
        while ($n -lt $Nreaders) {
            #seed = "-r ${seeds[$n]}"
            $seed = ""
            $rp = Start-Process -FilePath $testdir/swmr_reader.exe -NoNewWindow -PassThru -ArgumentList "$Nsecs_add $seed" 2>&1 |tee swmr_reader.out.$n
            $pid_readers += $rp.id
            $n += 1
        }
        #Write-Output "pid_readers=$pid_readers"

        # Collect exit code of the readers first because they usually finish
        # before the writer.
        foreach ($xpid in $pid_readers) {
            #Write-Output "checked reader $xpid"
            $result = Wait-Process -Id  $xpid
            if ($result.ExitCode -ne 0) {
                Write-Warning "reader had error"
                $nerrors += 1
            }
        }

        # Collect exit code of the writer
        #Write-Output "checked writer $pid_writer"
        $result = Wait-Process -Id  $pid_writer
        if ($result.ExitCode -ne 0) {
            Write-Warning "writer had error"
            $nerrors += 1
        }

        # Check for error and exit if one occurred
        #Write-Output "nerrors=$nerrors"
        if ($nerrors -ne 0) {
            Write-Warning "SWMR tests failed with $nerrors errors."
            Write-Warning "(Writer and reader output preserved)"
            exit 1
        }

        # Clean up output files
        Remove-Item swmr_writer.out
        Remove-Item swmr_reader.out.*

        Write-Output ""
        Write-Output "###############################################################################"
        Write-Output "## Remove test - test shrinking the dataset"
        Write-Output "###############################################################################"

        # Remove any possible writer message file before launching writer
        Remove-Item $WRITER_MESSAGE
        # Launch the Remove Writer
        Write-Output "launch the swmr_remove_writer"
        $seed = "" # Put -r <random seed> command here
        $rp = Start-Process -FilePath $testdir/swmr_remove_writer.exe -NoNewWindow -PassThru -ArgumentList "-o $Nrecs_rem $seed" 2>&1 |tee swmr_writer.out
        $pid_writer = $rp.id
        #Write-Output "pid_writer=$pid_writer"

        # Wait for message from writer process before starting reader(s)
        Wait-Message $testdir $WRITER_MESSAGE
        #
        # Launch the Remove Readers
        #declare -a seeds = (<seed1> <seed2> <seed3> ... )
        $n = 0
        $pid_readers = @()
        Write-Output "launch $Nreaders swmr_remove_readers"
        while ($n -lt $Nreaders) {
            #seed = "-r ${seeds[$n]}"
            $seed = ""
            $rp = Start-Process -FilePath $testdir/swmr_remove_reader.exe -NoNewWindow -PassThru -ArgumentList "$Nsecs_rem $seed" 2>&1 |tee swmr_reader.out.$n
            $pid_readers += $rp.id
            $n += 1
        }
        #Write-Output "pid_readers=$pid_readers"

        # Collect exit code of the readers first because they usually finish
        # before the writer.
        foreach ($xpid in $pid_readers) {
            #Write-Output "checked reader $xpid"
            $result = Wait-Process -Id  $xpid
            if ($result.ExitCode -ne 0) {
                Write-Warning "reader had error"
                $nerrors += 1
            }
        }

        # Collect exit code of the writer
        #Write-Output "checked writer $pid_writer"
        $result = Wait-Process -Id  $pid_writer
        if ($result.ExitCode -ne 0) {
            Write-Warning "writer had error"
            $nerrors += 1
        }

        # Check for error and exit if one occurred
        #Write-Output "nerrors=$nerrors"
        if ($nerrors -ne 0) {
            Write-Warning "SWMR tests failed with $nerrors errors."
            Write-Warning "(Writer and reader output preserved)"
            exit 1
        }

        # Clean up output files
        Remove-Item swmr_writer.out
        Remove-Item swmr_reader.out.*

        Write-Output ""
        Write-Output "###############################################################################"
        Write-Output "## Add/remove test - randomly grow or shrink the dataset"
        Write-Output "###############################################################################"

        # Launch the Generator
        Write-Output "launch the swmr_generator"
        $rp = Start-Process -FilePath $testdir/swmr_generator.exe -NoNewWindow-PassThru -Wait -ArgumentList "$compress $index_type"
        if ($rp.ExitCode -ne 0) {
            Write-Warning "generator had error"
            $nerrors += 1
        }

        # Launch the Writer (not in parallel - just to rebuild the datasets)
        Write-Output "launch the swmr_writer"
        $seed = "" # Put -r <random seed> command here
        $rp = Start-Process -FilePath $testdir/swmr_writer.exe -NoNewWindow -PassThru -Wait -ArgumentList "$Nrecords $seed"
        if ($rp.ExitCode -ne 0) {
            Write-Warning "writer had error"
            $nerrors += 1
        }

        # Remove any possible writer message file before launching writer
        Remove-Item $WRITER_MESSAGE
        #
        # Launch the Add/Remove Writer
        Write-Output "launch the swmr_addrem_writer"
        $seed = "" # Put -r <random seed> command here
        $rp = Start-Process -FilePath $testdir/swmr_addrem_writer.exe -NoNewWindow -PassThru -ArgumentList "$Nrecords $seed" 2>&1 |tee swmr_writer.out
        $pid_writer = $rp.id
        #Write-Output "pid_writer=$pid_writer"

        # Wait for message from writer process before starting reader(s)
        Wait-Message $testdir $WRITER_MESSAGE
        #
        # Launch the Add/Remove Readers
        #declare -a seeds = (<seed1> <seed2> <seed3> ... )
        $n = 0
        $pid_readers = @()
        Write-Output "launch $Nreaders swmr_remove_readers"
        while ($n -lt $Nreaders) {
            #seed = "-r ${seeds[$n]}"
            $seed = ""
            $rp = Start-Process -FilePath $testdir/swmr_remove_reader.exe -NoNewWindow -PassThru -ArgumentList "$Nsecs_addrem $seed" 2>&1 |tee swmr_reader.out.$n
            $pid_readers += $rp.id
            $n += 1
        }
        #Write-Output "pid_readers=$pid_readers"

        # Collect exit code of the readers first because they usually finish
        # before the writer.
        foreach ($xpid in $pid_readers) {
            #Write-Output "checked reader $xpid"
            $result = Wait-Process -Id  $xpid
            if ($result.ExitCode -ne 0) {
                Write-Warning "reader had error"
                $nerrors += 1
            }
        }

        # Collect exit code of the writer
        #Write-Output "checked writer $pid_writer"
        $result = Wait-Process -Id  $pid_writer
        if ($result.ExitCode -ne 0) {
            Write-Warning "writer had error"
            $nerrors += 1
        }

        # Check for error and exit if one occurred
        #Write-Output "nerrors=$nerrors"
        if ($nerrors -ne 0) {
            Write-Warning "SWMR tests failed with $nerrors errors."
            Write-Warning "(Writer and reader output preserved)"
            exit 1
        }

        # Clean up output files
        Remove-Item swmr_writer.out
        Remove-Item swmr_reader.out.*

        Write-Output ""
        Write-Output "###############################################################################"
        Write-Output "## Sparse writer test - test writing to random locations in the dataset"
        Write-Output "###############################################################################"

        # Launch the Generator
        # NOTE: Random seed is shared between readers and writers and is
        #       created by the generator.
        Write-Output "launch the swmr_generator"
        $seed = "" # Put -r <random seed> command here
        $rp = Start-Process -FilePath $testdir/swmr_generator.exe -NoNewWindow -PassThru -Wait -ArgumentList "$compress $index_type $seed"
        if ($rp.ExitCode -ne 0) {
            Write-Warning "generator had error"
            $nerrors += 1
        }

        # Remove any possible writer message file before launching writer
        Remove-Item $WRITER_MESSAGE
        # Launch the Sparse writer
        Write-Output "launch the swmr_sparse_writer"
        $rp = Start-Process -FilePath $testdir/swmr_sparse_writer.exe -NoNewWindow -PassThru -ArgumentList "$Nrecs_spa" 2>&1 |tee swmr_writer.out
        $pid_writer = $rp.Id
        #Write-Output "pid_writer=$pid_writer"

        # Wait for message from writer process before starting reader(s)
        Wait-Message $testdir $WRITER_MESSAGE
        #
        # Launch the Sparse readers
        $n = 0
        $pid_readers = @()
        Write-Output "launch $Nrdrs_spa swmr_sparse_readers"
        while ($n -lt $Nrdrs_spa) {
            # The sparse reader spits out a LOT of data so it's set to 'quiet'
            $rp = Start-Process -FilePath $testdir/swmr_sparse_reader.exe -NoNewWindow -PassThru -ArgumentList "-q $Nrecs_spa" 2>&1 |tee swmr_reader.out.$n
            $pid_readers += $rp.id
            $n += 1
        }
        #Write-Output "pid_readers=$pid_readers"

        # Collect exit code of the writer
        #Write-Output "checked writer $pid_writer"
        $result = Wait-Process -Id  $pid_writer
        if ($result.ExitCode -ne 0) {
            Write-Warning "writer had error"
            $nerrors += 1
        }

        # Collect exit code of the readers
        foreach ($xpid in $pid_readers) {
            #Write-Output "checked reader $xpid"
            $result = Wait-Process -Id  $xpid
            if ($result.ExitCode -ne 0) {
                Write-Warning "reader had error"
                $nerrors += 1
            }
        }

        # Check for error and exit if one occurred
        #Write-Output "nerrors=$nerrors"
        if ($nerrors -ne 0) {
            Write-Warning "SWMR tests failed with $nerrors errors."
            Write-Warning "(Writer and reader output preserved)"
            exit 1
        }

        # Clean up output files
        Remove-Item swmr_writer.out
        Remove-Item swmr_reader.out.*
    }
}

###############################################################################
## Report and exit
###############################################################################
cd ..
#Write-Output "nerrors=$nerrors"
if ($nerrors -eq 0) {
    Write-Output "SWMR tests passed."
#    if test -z "$HDF5_NOCLEANUP"; then
#        # delete the test directory
#        Remove-Item swmr_test -Recurse
#    fi
    exit 0
}
else {
    Write-Warning "SWMR tests failed with $nerrors errors."
    exit 1
}