# -*- coding: utf-8 -*-
#
# This file is subject to the terms and conditions defined in
# file 'LICENSE.txt', which is part of this source code package.
#
"""Testing infrastructure for Muscat and extra modules
Developer documentation:
all the import statements are placed after the treatment of the
user option "-y" (dont_write_bytecode). So no pyc are created prior
the default behavior
"""
from typing import Callable, List, Tuple, Dict, Optional
import sys
import traceback
import time
import getopt
import os
Test_Help_String = """
python Muscat.Helper.Check.py -c -f -s -e <extraModules> -m <moduleFilter>
python -m Muscat.Helper.Check -c -f -s -e <extraModules> -m <moduleFilter>
options :
-f Full output for all the test
-v Activate maximal level of verbosity (implied -f)
-s Stop at first error
-e To test extra Modules (-e can be repeated)
-m To filter modules to test by this string (-m can be repeated) (for example -m .IO. )
-k Skip test base on string
-d Dry run do not execute anything, only show what will be executed
-c To activate coverage and generate a html report
-cb (coverage with browser and local file index.html generated)
for correct platform dependent coverage you can use the following
comment to disconnect coverage in some platform:
# only nt coverage
# only posix coverage
-l Generation of the coverage (html) report locally (current path)
-b Launch browser after the coverage report generation
-p Activate profiling
-t use mypy to check the typing of every module
-y Generate .pyc when importing modules (default False)
-L Output Locally, Use the current path for all the outputs
-P <dir> Set temporary output directory to
-g to use the Froze decorator during testing
"""
def __RunAndCheck(lis, bp, stopAtFirstError, dryRun, profiling, typing) -> Dict: # pragma: no cover
res: dict = {}
from Muscat.Helpers.TextFormatHelper import TFormat
for name in lis:
bp.Print(TFormat.InBlue(TFormat.Center("Running Test " + name, width=80)))
if lis[name] is None:
bp.Print(TFormat().InRed(TFormat().GetIndent() + 'Sub Module "' + name + '" does not have the CheckIntegrity function'))
r = 'Not OK'
if stopAtFirstError:
raise (Exception(TFormat().GetIndent() + 'Sub Module "' + name + '" does not have the CheckIntegrity function'))
try:
start_time: float = time.time()
stop_time: float = time.time()
if dryRun:
r = "Dry Run "
else:
testStr = """
from {} import CheckIntegrity
res = CheckIntegrity()""".format(name)
local = {}
if profiling:
import cProfile
import pstats
from io import StringIO
pr = cProfile.Profile()
pr.enable()
exec(testStr, {}, local)
r = local["res"]
pr.disable()
s = StringIO()
sortby = 'cumulative'
pstats.Stats(pr, stream=s).sort_stats(pstats.SortKey.CUMULATIVE).print_stats(50)
print(s.getvalue())
else:
exec(testStr, {}, local)
r = local["res"]
sys.stdout.flush()
sys.stderr.flush()
stop_time = time.time()
res[name] = r
if not isinstance(r, str):
bp.Print(TFormat.InRed(TFormat().GetIndent() + "Please add a correct return statement in the CheckIntegrity of the module" + name))
#raise Exception()
r = 'Not OK'
if typing:
import subprocess
subprocess.call(["mypy", "--ignore-missing-imports", "-m", name])
except UserWarning as e:
sys.stdout.flush()
sys.stderr.flush()
bp.Print("Unexpected Warning:" + str(sys.exc_info()[0]))
res[name] = "error"
traceback.print_exc(file=bp.stdout_)
r = 'Not OK'
except KeyboardInterrupt as e:
sys.stdout.flush()
sys.stderr.flush()
bp.Print("User interruption ")
res[name] = "interrupted"
r = 'Interrupted'
bp.Restore()
raise
except:
sys.stdout.flush()
sys.stderr.flush()
bp.Print("Unexpected Error:" + str(sys.exc_info()[0]))
res[name] = "error"
traceback.print_exc(file=bp.stdout_)
r = 'Not OK'
if stopAtFirstError:
bp.Restore()
raise
if r.lower().startswith('ok'):
bp.Print(TFormat.InGreen(f"OK {name} : {stop_time -start_time:.3f} seconds "))
elif r.lower().startswith('skip'):
bp.Print(TFormat.InYellow(str(r) + " !!!! " + name))
else:
bp.Print(TFormat.InRed(str(r) + " !!!! " + name))
return res
def __tryImportRecursive(subModule, toCheck, stopAtFirstError, modulesToTreat, modulesToSkip) -> None: # pragma: no cover
import importlib
try:
sm = importlib.import_module(subModule)
except ImportError as e:
print(e)
print("please check the value of the _test variable ")
if stopAtFirstError:
raise
sm = None
except Exception:
sm = None
if stopAtFirstError:
raise
cif = getattr(sm, "CheckIntegrity", None)
if cif is not None:
toCheck[subModule] = cif
else:
try:
subSubModules = []
try:
listToTest = getattr(sm, "_test", None)
subSubModules = [subModule + '.' + x for x in listToTest]
except Exception as e:
print(e)
print('Error Loading module : ' + subModule + ' (Current folder'+os.getcwd()+')')
print('-*-*-*-*-*-*> missing _test variable ??? <*-*-*-*-*-*--')
raise
for subSubModule in subSubModules:
if any([x.lower() in ("."+subSubModule.lower()+".") for x in modulesToSkip]):
print("skip module :" + str(subSubModule))
continue
try:
__tryImportRecursive(subSubModule, toCheck, stopAtFirstError, modulesToTreat, modulesToSkip)
except Exception as e:
print(e)
print('Error Loading module : ' + subSubModule + ' (Current folder'+os.getcwd()+')')
print('-*-*-*-*-*-*> missing CheckIntegrity()??? <*-*-*-*-*-*--')
raise
except:
toCheck[subModule] = None
if stopAtFirstError:
raise
def __tryImport(noduleName, bp, stopAtFirstError, modulesToTreat, modulesToSkip) -> Dict: # pragma: no cover
toCheck = {}
try:
__tryImportRecursive(noduleName, toCheck, stopAtFirstError, modulesToTreat, modulesToSkip)
except:
print("Error loading module '" + noduleName + "'")
print("This module will not be tested ")
sys.stdout.flush()
sys.stderr.flush()
bp.Print("Unexpected Error:" + str(sys.exc_info()[0]))
traceback.print_exc(file=bp.stdout_)
if stopAtFirstError:
raise
return toCheck
[docs]
def TestAll(modulesToTreat: List[str] = ['ALL'], modulesToSkip: List = [],
extraToolsBoxes: Optional[List[str]] = None, options=None) -> Dict: # pragma: no cover
"""Core function to test every module present
Parameters
----------
modulesToTreat : list, optional
List of modules to test , by default ['ALL'] means all the modules in the current toolbox
modulesToSkip : list, optional
List of modules to skip every entry is a string used in a find to filter the modules to test, by default []
stopAtFirstError : bool, optional
Stop at first error encountered, by default False
extraToolsBoxes : List[str], optional
name of the extra libraries to test, by default None, this means "Muscat"
dryRun : bool, optional
to just print the code to be executed, by default False
profiling : bool, optional
activate the profiling of the tests, by default False
coverage : Dict, optional
argument to control the coverage checking, by default None
typing : bool, optional
activate the typing checking, by default False
Returns
-------
Dict
_description_
"""
print("")
print("modulesToTreat : " + str(modulesToTreat))
print("modulesToSkip : " + str(modulesToSkip))
print("coverage : " + str(options))
print("extraToolsBoxes : " + str(extraToolsBoxes))
fullOutput = options["fullOutput"]
cov = None
if options["coverage"]:
import coverage as modcoverage
cov = modcoverage.Coverage(omit=['pyexpat', '*__init__.py'])
linesToIgnore: List[str] = ["nt", "posix"]
for line in linesToIgnore:
if os.name == line:
continue
cov.exclude(f"#* only* {line}* coverage")
cov.start()
if options["-L"]:
from Muscat.Helpers.IO.FileTools import TemporaryDirectory
TemporaryDirectory.SetTempPath(os.getcwd())
if options["-P"]:
print('Setting temp output directory to ' + options["-Parg"])
from Muscat.Helpers.IO.FileTools import TemporaryDirectory
TemporaryDirectory.SetTempPath(options["-Parg"])
if options["-g"]:
import Muscat.Helpers.Decorators as Decorators
Decorators.useFroze_itDecorator = True
print("Decorators.useFroze_itDecorator =", Decorators.useFroze_itDecorator)
import Muscat.Helpers.Logger as Logger
Logger.DefaultMuscatOutput()
if options["-v"]:
Logger.VerboseMuscatOutput()
# calls to print, ie import module1
from Muscat.Helpers.PrintBypass import PrintBypass
print("Running Tests : ")
start_time = time.time()
print("--- Begin Test ---")
toCheck = {}
with PrintBypass() as bp:
if extraToolsBoxes is not None:
for tool in extraToolsBoxes:
toCheck.update(__tryImport(tool, bp, options["stopAtFirstError"], modulesToTreat, modulesToSkip))
if not fullOutput:
bp.ToSink()
bp.Print("Sending all the output of the tests to sink")
#import Muscat.Helpers.Logger as Logger
Logger.SilentAllMuscatOutput()
if modulesToTreat[0] != 'ALL':
filtered = dict((k, v) for k, v in toCheck.items() if any(s in k for s in modulesToTreat))
toCheck = filtered
if len(modulesToSkip):
filtered = dict((k, v) for k, v in toCheck.items() if not any(s in k for s in modulesToSkip))
toCheck = filtered
res = __RunAndCheck(toCheck, bp, options["stopAtFirstError"], options["dryRun"], options["profiling"], options["typing"])
if options["coverage"]:
cov.stop()
cov.save()
# create a temp file
if options["localHtml"]:
#tempdir = "./"
tempdir = os.getcwd() + os.sep
else:
from Muscat.Helpers.IO.FileTools import TemporaryDirectory
tempdir = TemporaryDirectory.GetTempPath()
ss = [("*/"+os.sep.join(k.split("."))+"*") for k in toCheck]
cov.html_report(directory=tempdir, include=ss, title="Coverage report of " + " and ".join(extraToolsBoxes))
print('Coverage Report in : ' + tempdir + "index.html")
cov.report(include=ss)
if options["launchBrowser"]:
import webbrowser
webbrowser.open(tempdir+"index.html")
stop_time = time.time()
bp.Print("Total Time : %.3f seconds " % (stop_time - start_time))
print("--- End Test ---")
print(modulesToTreat)
if len(modulesToTreat) == 1 and modulesToTreat[0] == "ALL":
# we verified that all the python files in the repository are tested
CheckIfAllThePresentFilesAreTested(extraToolsBoxes, res)
return res
[docs]
def CheckIfAllThePresentFilesAreTested(extraToolsBoxes, testedFiles) -> None: # pragma: no cover
pythonPaths = os.environ.get('PYTHONPATH', os.getcwd()).split(os.pathsep)
toIgnore = ['.git', # git file
"__pycache__", # python 3
"__init__.py", # infra
"setup.py", # compilation
"docs/conf.py", # documentation
"__main__.py",
"Check.py",
]
testedFiles = testedFiles.keys()
presentFiles = []
for pythonPath in pythonPaths:
for eTB in extraToolsBoxes:
path = pythonPath + os.sep + eTB
for dirname, dirnames, filenames in os.walk(path):
cleanDirName = dirname.replace(pythonPath+os.sep, "") # .replace(os.sep,".")
# print path to all filenames.
for filename in filenames:
if filename in toIgnore:
continue
if len(str(filename)) > 3 and str(filename)[-3:] == ".py":
if len(cleanDirName) == 0:
presentFiles.append(filename)
else:
presentFiles.append(cleanDirName+os.sep+filename)
# Advanced usage:
# editing the 'dirnames' list will stop os.walk() from recursing into there.
for dti in toIgnore:
if dti in dirnames:
dirnames.remove(dti)
for tf in testedFiles:
if tf.replace(".", os.sep)+".py" in presentFiles:
presentFiles.remove(tf.replace(".", os.sep) + ".py")
if len(presentFiles) > 0:
print("Files present in the repository but not tested:")
for file in presentFiles:
print(f" {file}")
[docs]
def CheckIntegrity() -> str:
return "Ok"
[docs]
def GetDefaultOptions(**kwargs):
res = {"-g":False,
"-L": False,
"-P":False,
'-v':False,
"coverage": False, "localHtml":False,
"fullOutput":False,
"typing":False,
"dryRun":False,
"profiling":False,
"stopAtFirstError":False}
res.update(kwargs)
return res
[docs]
def RunTests() -> int: # pragma: no cover
"""Base function to run all tests, all the option are captured from the command line arguments.
this function is intended to be used by the user
python Muscat.Helper.Check.py -c -f -s -e <extraModules> -m <moduleFilter>
python -m Muscat.Helper.Check -c -f -s -e <extraModules> -m <moduleFilter>
options :
-f Full output for all the test
-v Activate maximal level of verbosity (implied -f)
-s Stop at first error
-e To test extra Modules (-e can be repeated)
-m To filter the output by this string (-m can be repeated)
-k Skip test base on string
-d Dry run do not execute anything, only show what will be executed
-c To activate coverage and generate a html report
-cb (coverage with browser and local file index.html generated)
for correct platform dependent coverage you can use the following
comment to disconnect coverage in some platform::
# only nt coverage
# only posix coverage
-l Generation of the coverage (html) report locally (current path)
-b Launch browser after the coverage report generation
-p Activate profiling
-t use mypy to check the typing of every module
-y Generate .pyc when importing modules (default False)
-L Output Locally, Use the current path for all the outputs
-P <dir> Set temporary output directory to
-g to use the Froze decorator during testing
Returns
-------
int
return the number of failed tests
"""
options = GetDefaultOptions()
if len(sys.argv) == 1:
res = TestAll(modulesToTreat=['ALL'], extraToolsBoxes=["Muscat"], options=options) # pragma: no cover
else:
try:
opts, args = getopt.getopt(sys.argv[1:], "thcblfsdpvyLe:m:k:P:g")
except getopt.GetoptError as e:
print(e)
print(Test_Help_String)
sys.exit(2)
coverage = False
extraToolsBoxes = []
modulesToTreat = []
modulesToSkip = []
browser = False
localHtml = False
sys.dont_write_bytecode = True
for opt, arg in opts:
if opt == '-h':
print(Test_Help_String)
sys.exit()
elif opt in ("-c"):
coverage = True
browser = False
elif opt in ("-t"):
options["typing"] = True
os.environ["MYPYPATH"] = os.environ["PYTHONPATH"]
elif opt in ("-l"):
localHtml = True
browser = False
elif opt in ("-b"):
browser = True
elif opt in ("-f"):
options["fullOutput"] = True
elif opt in ("-s"):
options["stopAtFirstError"] = True
elif opt in ("-d"):
options["dryRun"] = True
elif opt in ("-v"):
options["-v"] = True
options["fullOutput"] = True
elif opt in ("-p"):
options["profiling"] = True
elif opt in ("-e"):
extraToolsBoxes.append(arg)
elif opt in ("-m"):
modulesToTreat.append(arg)
elif opt in ("-k"):
modulesToSkip.append(arg)
elif opt in ("-P"):
options["-P"] = True
options["-Parg"] = arg
elif opt in ("-y"):
sys.dont_write_bytecode = False
elif opt in ("-L"):
options["-L"] = True
elif opt in ("-g"):
options["-g"] = True
if len(modulesToTreat) == 0:
modulesToTreat.append("ALL")
if len(extraToolsBoxes) == 0:
extraToolsBoxes.append("Muscat")
options.update({"coverage": coverage, "localHtml": localHtml, "launchBrowser": browser} )
res = TestAll(modulesToTreat=modulesToTreat,
modulesToSkip=modulesToSkip,
options = options,
extraToolsBoxes=extraToolsBoxes)
errors = {}
oks = {}
skipped = {}
for x, y in res.items():
if str(y).lower().startswith("ok"):
oks[x] = y
elif str(y).lower().startswith("skip"):
skipped[x] = y
else:
errors[x] = y
print("Number of test OK : " + str(len(oks)))
print("Number of skipped test : " + str(len(skipped)))
print("Number of test KO : " + str(len(errors)))
nbOks = len(oks)
nbErrors = len(errors)
if nbOks == 0 and nbErrors == 0:
nbOks = 1
print(f"Percentage of OK test : {(nbOks*100.0)/(nbOks+nbErrors):.2f} %")
return errors
if __name__ == '__main__': # pragma: no cover
errors = RunTests()
sys.exit(len(errors))