tests/*server.py: close log file after each log line

Make sure the log file is not locked once a test has
finished and align with the behavior of our logmsg.

Rename curl_test_data.py to be a general util.py.
Format and sort Python imports with isort/VSCode.

Bug: #6058
Closes #6206

tests/Makefile.am

@@ -30,7 +30,7 @@ EXTRA_DIST = ftpserver.pl httpserver.pl secureserver.pl runtests.pl           \
  serverhelp.pm tftpserver.pl rtspserver.pl directories.pm symbol-scan.pl      \
  CMakeLists.txt mem-include-scan.pl valgrind.supp extern-scan.pl              \
  manpage-scan.pl nroff-scan.pl http2-server.pl dictserver.py                  \
- negtelnetserver.py smbserver.py curl_test_data.py                            \
+ negtelnetserver.py smbserver.py util.py                                      \
  disable-scan.pl manpage-syntax.pl error-codes.pl badsymbols.pl               \
  azure.pm appveyor.pm version-scan.pl options-scan.pl
 

tests/dictserver.py

@@ -26,16 +26,19 @@
 
 from __future__ import (absolute_import, division, print_function,
                         unicode_literals)
+
 import argparse
+import logging
 import os
 import sys
-import logging
+
+from util import ClosingFileHandler
+
 try:  # Python 2
     import SocketServer as socketserver
 except ImportError:  # Python 3
     import socketserver
 
-
 log = logging.getLogger(__name__)
 HOST = "localhost"
 
@@ -138,7 +141,7 @@ def setup_logging(options):
 
     # Write out to a logfile
     if options.logfile:
-        handler = logging.FileHandler(options.logfile, mode="w")
+        handler = ClosingFileHandler(options.logfile)
         handler.setFormatter(formatter)
         handler.setLevel(logging.DEBUG)
         root_logger.addHandler(handler)

tests/negtelnetserver.py

@@ -23,10 +23,14 @@
 
 from __future__ import (absolute_import, division, print_function,
                         unicode_literals)
+
 import argparse
+import logging
 import os
 import sys
-import logging
+
+from util import ClosingFileHandler
+
 if sys.version_info.major >= 3:
     import socketserver
 else:
@@ -313,7 +317,7 @@ def setup_logging(options):
 
     # Write out to a logfile
     if options.logfile:
-        handler = logging.FileHandler(options.logfile, mode="w")
+        handler = ClosingFileHandler(options.logfile)
         handler.setFormatter(formatter)
         handler.setLevel(logging.DEBUG)
         root_logger.addHandler(handler)

tests/smbserver.py

@@ -21,21 +21,23 @@
 #
 """Server for testing SMB"""
 
-from __future__ import (absolute_import, division, print_function)
+from __future__ import absolute_import, division, print_function
 # NOTE: the impacket configuration is not unicode_literals compatible!
+
 import argparse
+import logging
 import os
 import sys
-import logging
 import tempfile
+
+# Import our curl test data helper
+from util import ClosingFileHandler, TestData
+
 if sys.version_info.major >= 3:
     import configparser
 else:
     import ConfigParser as configparser
 
-# Import our curl test data helper
-import curl_test_data
-
 # impacket needs to be installed in the Python environment
 try:
     import impacket
@@ -43,10 +45,10 @@ except ImportError:
     sys.stderr.write('Python package impacket needs to be installed!\n')
     sys.stderr.write('Use pip or your package manager to install it.\n')
     sys.exit(1)
-from impacket import smbserver as imp_smbserver
 from impacket import smb as imp_smb
-from impacket.nt_errors import (STATUS_ACCESS_DENIED, STATUS_SUCCESS,
-                                STATUS_NO_SUCH_FILE)
+from impacket import smbserver as imp_smbserver
+from impacket.nt_errors import (STATUS_ACCESS_DENIED, STATUS_NO_SUCH_FILE,
+                                STATUS_SUCCESS)
 
 log = logging.getLogger(__name__)
 SERVER_MAGIC = "SERVER_MAGIC"
@@ -120,7 +122,7 @@ class TestSmbServer(imp_smbserver.SMBSERVER):
                                          config_parser=config_parser)
 
         # Set up a test data object so we can get test data later.
-        self.ctd = curl_test_data.TestData(test_data_directory)
+        self.ctd = TestData(test_data_directory)
 
         # Override smbComNtCreateAndX so we can pretend to have files which
         # don't exist.
@@ -353,7 +355,7 @@ def setup_logging(options):
 
     # Write out to a logfile
     if options.logfile:
-        handler = logging.FileHandler(options.logfile, mode="w")
+        handler = ClosingFileHandler(options.logfile)
         handler.setFormatter(formatter)
         handler.setLevel(logging.DEBUG)
         root_logger.addHandler(handler)

tests/util.py

@@ -19,13 +19,14 @@
 # This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
 # KIND, either express or implied.
 #
-"""Module for extracting test data from the test data folder"""
+"""Module for extracting test data from the test data folder and other utils"""
 
 from __future__ import (absolute_import, division, print_function,
                         unicode_literals)
+
+import logging
 import os
 import re
-import logging
 
 log = logging.getLogger(__name__)
 
@@ -33,6 +34,19 @@ log = logging.getLogger(__name__)
 REPLY_DATA = re.compile("<reply>[ \t\n\r]*<data[^<]*>(.*?)</data>", re.MULTILINE | re.DOTALL)
 
 
+class ClosingFileHandler(logging.StreamHandler):
+    def __init__(self, filename):
+        super(ClosingFileHandler, self).__init__()
+        self.filename = os.path.abspath(filename)
+        self.setStream(None)
+
+    def emit(self, record):
+        with open(self.filename, "a") as fp:
+            self.setStream(fp)
+            super(ClosingFileHandler, self).emit(record)
+            self.setStream(None)
+
+
 class TestData(object):
     def __init__(self, data_folder):
         self.data_folder = data_folder