libghc-tasty-doc 0.11.0.2-1build1 / usr / share / doc / libghc-tasty-doc / html / tasty.txt

-- Hoogle documentation, generated by Haddock
-- See Hoogle, http://www.haskell.org/hoogle/


-- | Modern and extensible testing framework
--   
--   Tasty is a modern testing framework for Haskell. It lets you combine
--   your unit tests, golden tests, QuickCheck/SmallCheck properties, and
--   any other types of tests into a single test suite. See
--   <a>http://documentup.com/feuerbach/tasty</a>.
@package tasty
@version 0.11.0.2


-- | Extensible options. They are used for provider-specific settings,
--   ingredient-specific settings and core settings (such as the test name
--   pattern).
module Test.Tasty.Options

-- | An option is a data type that inhabits the <a>IsOption</a> type class.
class Typeable v => IsOption v where optionCLParser = option parse (long name <> help helpString) where name = untag (optionName :: Tagged v String) helpString = untag (optionHelp :: Tagged v String) parse = str >>= maybe (readerError $ "Could not parse " ++ name) pure <$> parseValue

-- | The value to use if the option was not supplied explicitly
defaultValue :: IsOption v => v

-- | Try to parse an option value from a string
parseValue :: IsOption v => String -> Maybe v

-- | The option name. It is used to form the command line option name, for
--   instance. Therefore, it had better not contain spaces or other fancy
--   characters. It is recommended to use dashes instead of spaces.
optionName :: IsOption v => Tagged v String

-- | The option description or help string. This can be an arbitrary
--   string.
optionHelp :: IsOption v => Tagged v String

-- | A command-line option parser.
--   
--   It has a default implementation in terms of the other methods. You may
--   want to override it in some cases (e.g. add a short flag).
--   
--   Even if you override this, you still should implement all the methods
--   above, to allow alternative interfaces.
--   
--   Do not supply a default value here for this parser! This is because if
--   no value was provided on the command line we may lookup the option
--   e.g. in the environment. But if the parser always succeeds, we have no
--   way to tell whether the user really provided the option on the command
--   line.
optionCLParser :: IsOption v => Parser v

-- | A set of options. Only one option of each type can be kept.
--   
--   If some option has not been explicitly set, the default value is used.
data OptionSet

-- | Set the option value
setOption :: IsOption v => v -> OptionSet -> OptionSet

-- | Change the option value
changeOption :: IsOption v => (v -> v) -> OptionSet -> OptionSet

-- | Query the option value
lookupOption :: IsOption v => OptionSet -> v

-- | Create a singleton <a>OptionSet</a>
singleOption :: IsOption v => v -> OptionSet

-- | The purpose of this data type is to capture the dictionary
--   corresponding to a particular option.
data OptionDescription
Option :: Proxy v -> OptionDescription

-- | Command-line parser to use with flags
flagCLParser :: IsOption v => Maybe Char -> v -> Parser v

-- | Safe read function. Defined here for convenience to use for
--   <a>parseValue</a>.
safeRead :: Read a => String -> Maybe a
instance GHC.Base.Monoid Test.Tasty.Options.OptionSet


-- | API for test providers
module Test.Tasty.Providers

-- | The interface to be implemented by a test provider.
--   
--   The type <tt>t</tt> is the concrete representation of the test which
--   is used by the provider.
class Typeable t => IsTest t

-- | Run the test
--   
--   This method should cleanly catch any exceptions in the code to test,
--   and return them as part of the <a>Result</a>, see <a>FailureReason</a>
--   for an explanation. It is ok for <a>run</a> to raise an exception if
--   there is a problem with the test suite code itself (for example, if a
--   file that should contain example data or expected output is not
--   found).
run :: IsTest t => OptionSet -> t -> (Progress -> IO ()) -> IO Result

-- | The list of options that affect execution of tests of this type
testOptions :: IsTest t => Tagged t [OptionDescription]

-- | <a>Result</a> of a passed test
testPassed :: String -> Result

-- | <a>Result</a> of a failed test
testFailed :: String -> Result

-- | A test result
data Result

-- | Test progress information.
--   
--   This may be used by a runner to provide some feedback to the user
--   while a long-running test is executing.
data Progress
Progress :: String -> Float -> Progress

-- | textual information about the test's progress
[progressText] :: Progress -> String

-- | <a>progressPercent</a> should be a value between 0 and 1. If it's
--   impossible to compute the estimate, use 0.
[progressPercent] :: Progress -> Float

-- | The name of a test or a group of tests
type TestName = String

-- | The main data structure defining a test suite.
--   
--   It consists of individual test cases and properties, organized in
--   named groups which form a tree-like hierarchy.
--   
--   There is no generic way to create a test case. Instead, every test
--   provider (tasty-hunit, tasty-smallcheck etc.) provides a function to
--   turn a test case into a <a>TestTree</a>.
--   
--   Groups can be created using <a>testGroup</a>.
data TestTree

-- | Convert a test to a leaf of the <a>TestTree</a>
singleTest :: IsTest t => TestName -> t -> TestTree


-- | This module contains the core definitions related to ingredients.
--   
--   Ingredients themselves are provided by other modules (usually under
--   the <tt>Test.Tasty.Ingredients.*</tt> hierarchy).
module Test.Tasty.Ingredients

-- | <a>Ingredient</a>s make your test suite tasty.
--   
--   Ingredients represent different actions that you can perform on your
--   test suite. One obvious ingredient that you want to include is one
--   that runs tests and reports the progress and results.
--   
--   Another standard ingredient is one that simply prints the names of all
--   tests.
--   
--   Similar to test providers (see <a>IsTest</a>), every ingredient may
--   specify which options it cares about, so that those options are
--   presented to the user if the ingredient is included in the test suite.
--   
--   An ingredient can choose, typically based on the <a>OptionSet</a>,
--   whether to run. That's what the <a>Maybe</a> is for. The first
--   ingredient that agreed to run does its work, and the remaining
--   ingredients are ignored. Thus, the order in which you arrange the
--   ingredients may matter.
--   
--   Usually, the ingredient which runs the tests is unconditional and thus
--   should be placed last in the list. Other ingredients usually run only
--   if explicitly requested via an option. Their relative order thus
--   doesn't matter.
--   
--   That's all you need to know from an (advanced) user perspective. Read
--   on if you want to create a new ingredient.
--   
--   There are two kinds of ingredients.
--   
--   The first kind is <a>TestReporter</a>. If the ingredient that agrees
--   to run is a <a>TestReporter</a>, then tasty will automatically launch
--   the tests and pass a <a>StatusMap</a> to the ingredient. All the
--   ingredient needs to do then is to process the test results and
--   probably report them to the user in some way (hence the name).
--   
--   <a>TestManager</a> is the second kind of ingredient. It is typically
--   used for test management purposes (such as listing the test names),
--   although it can also be used for running tests (but, unlike
--   <a>TestReporter</a>, it has to launch the tests manually if it wants
--   them to be run). It is therefore more general than
--   <a>TestReporter</a>. <a>TestReporter</a> is provided just for
--   convenience.
--   
--   The function's result should indicate whether all the tests passed.
--   
--   In the <a>TestManager</a> case, it's up to the ingredient author to
--   decide what the result should be. When no tests are run, the result
--   should probably be <a>True</a>. Sometimes, even if some tests run and
--   fail, it still makes sense to return <a>True</a>.
data Ingredient

-- | For the explanation on how the callback works, see the documentation
--   for <a>launchTestTree</a>.
TestReporter :: [OptionDescription] -> (OptionSet -> TestTree -> Maybe (StatusMap -> IO (Time -> IO Bool))) -> Ingredient
TestManager :: [OptionDescription] -> (OptionSet -> TestTree -> Maybe (IO Bool)) -> Ingredient

-- | Run the first <a>Ingredient</a> that agrees to be run.
--   
--   If no one accepts the task, return <a>Nothing</a>. This is usually a
--   sign of misconfiguration.
tryIngredients :: [Ingredient] -> OptionSet -> TestTree -> Maybe (IO Bool)

-- | Return the options which are relevant for the given ingredient.
--   
--   Note that this isn't the same as simply pattern-matching on
--   <a>Ingredient</a>. E.g. options for a <a>TestReporter</a>
--   automatically include <a>NumThreads</a>.
ingredientOptions :: Ingredient -> [OptionDescription]

-- | Like <tt>ingredientOption</tt>, but folds over multiple ingredients.
ingredientsOptions :: [Ingredient] -> [OptionDescription]

-- | All the options relevant for this test suite. This includes the
--   options for the test tree and ingredients, and the core options.
suiteOptions :: [Ingredient] -> TestTree -> [OptionDescription]


-- | This module exports the basic ingredients defined in the
--   <tt>tasty</tt> packages.
--   
--   Note that if <tt>defaultIngredients</tt> from <a>Test.Tasty</a> suits
--   your needs, use that instead of importing this module.
module Test.Tasty.Ingredients.Basic

-- | A simple console UI
consoleTestReporter :: Ingredient

-- | Do not print test results (see README for details)
newtype Quiet
Quiet :: Bool -> Quiet

-- | Report only failed tests
newtype HideSuccesses
HideSuccesses :: Bool -> HideSuccesses

-- | The ingredient that provides the test listing functionality
listingTests :: Ingredient

-- | This option, when set to <a>True</a>, specifies that we should run in
--   the «list tests» mode
newtype ListTests
ListTests :: Bool -> ListTests

-- | Obtain the list of all tests in the suite
testsNames :: OptionSet -> TestTree -> [TestName]

-- | This ingredient doesn't do anything apart from registering additional
--   options.
--   
--   The option values can be accessed using <tt>askOption</tt>.
includingOptions :: [OptionDescription] -> Ingredient


-- | API for test runners
module Test.Tasty.Runners

-- | The main data structure defining a test suite.
--   
--   It consists of individual test cases and properties, organized in
--   named groups which form a tree-like hierarchy.
--   
--   There is no generic way to create a test case. Instead, every test
--   provider (tasty-hunit, tasty-smallcheck etc.) provides a function to
--   turn a test case into a <a>TestTree</a>.
--   
--   Groups can be created using <a>testGroup</a>.
data TestTree

-- | A single test of some particular type
SingleTest :: TestName -> t -> TestTree

-- | Assemble a number of tests into a cohesive group
TestGroup :: TestName -> [TestTree] -> TestTree

-- | Add some options to child tests
PlusTestOptions :: (OptionSet -> OptionSet) -> TestTree -> TestTree

-- | Acquire the resource before the tests in the inner tree start and
--   release it after they finish. The tree gets an <a>IO</a> action which
--   yields the resource, although the resource is shared across all the
--   tests.
WithResource :: (ResourceSpec a) -> (IO a -> TestTree) -> TestTree

-- | Ask for the options and customize the tests based on them
AskOptions :: (OptionSet -> TestTree) -> TestTree

-- | Fold a test tree into a single value.
--   
--   The fold result type should be a monoid. This is used to fold multiple
--   results in a test group. In particular, empty groups get folded into
--   <a>mempty</a>.
--   
--   Apart from pure convenience, this function also does the following
--   useful things:
--   
--   <ol>
--   <li>Keeping track of the current options (which may change due to
--   <a>PlusTestOptions</a> nodes)</li>
--   <li>Filtering out the tests which do not match the patterns</li>
--   </ol>
--   
--   Thus, it is preferred to an explicit recursive traversal of the tree.
--   
--   Note: right now, the patterns are looked up only once, and won't be
--   affected by the subsequent option changes. This shouldn't be a problem
--   in practice; OTOH, this behaviour may be changed later.
foldTestTree :: Monoid b => TreeFold b -> OptionSet -> TestTree -> b

-- | An algebra for folding a <a>TestTree</a>.
--   
--   Instead of constructing fresh records, build upon <a>trivialFold</a>
--   instead. This way your code won't break when new nodes/fields are
--   indroduced.
data TreeFold b
TreeFold :: (forall t. IsTest t => OptionSet -> TestName -> t -> b) -> (TestName -> b -> b) -> (forall a. ResourceSpec a -> (IO a -> b) -> b) -> TreeFold b
[foldSingle] :: TreeFold b -> forall t. IsTest t => OptionSet -> TestName -> t -> b
[foldGroup] :: TreeFold b -> TestName -> b -> b
[foldResource] :: TreeFold b -> forall a. ResourceSpec a -> (IO a -> b) -> b

-- | <a>trivialFold</a> can serve as the basis for custom folds. Just
--   override the fields you need.
--   
--   Here's what it does:
--   
--   <ul>
--   <li>single tests are mapped to <a>mempty</a> (you probably do want to
--   override that)</li>
--   <li>test groups are returned unmodified</li>
--   <li>for a resource, an IO action that throws an exception is passed
--   (you want to override this for runners/ingredients that execute
--   tests)</li>
--   </ul>
trivialFold :: Monoid b => TreeFold b

-- | <a>ResourceSpec</a> describes how to acquire a resource (the first
--   field) and how to release it (the second field).
data ResourceSpec a
ResourceSpec :: (IO a) -> (a -> IO ()) -> ResourceSpec a

-- | Monoid generated by <a>*&gt;</a>
newtype Traversal f
Traversal :: f () -> Traversal f
[getTraversal] :: Traversal f -> f ()

-- | Monoid generated by <tt><a>liftA2</a> (<a>&lt;&gt;</a>)</tt>
newtype Ap f a
Ap :: f a -> Ap f a
[getApp] :: Ap f a -> f a

-- | <a>Ingredient</a>s make your test suite tasty.
--   
--   Ingredients represent different actions that you can perform on your
--   test suite. One obvious ingredient that you want to include is one
--   that runs tests and reports the progress and results.
--   
--   Another standard ingredient is one that simply prints the names of all
--   tests.
--   
--   Similar to test providers (see <a>IsTest</a>), every ingredient may
--   specify which options it cares about, so that those options are
--   presented to the user if the ingredient is included in the test suite.
--   
--   An ingredient can choose, typically based on the <a>OptionSet</a>,
--   whether to run. That's what the <a>Maybe</a> is for. The first
--   ingredient that agreed to run does its work, and the remaining
--   ingredients are ignored. Thus, the order in which you arrange the
--   ingredients may matter.
--   
--   Usually, the ingredient which runs the tests is unconditional and thus
--   should be placed last in the list. Other ingredients usually run only
--   if explicitly requested via an option. Their relative order thus
--   doesn't matter.
--   
--   That's all you need to know from an (advanced) user perspective. Read
--   on if you want to create a new ingredient.
--   
--   There are two kinds of ingredients.
--   
--   The first kind is <a>TestReporter</a>. If the ingredient that agrees
--   to run is a <a>TestReporter</a>, then tasty will automatically launch
--   the tests and pass a <a>StatusMap</a> to the ingredient. All the
--   ingredient needs to do then is to process the test results and
--   probably report them to the user in some way (hence the name).
--   
--   <a>TestManager</a> is the second kind of ingredient. It is typically
--   used for test management purposes (such as listing the test names),
--   although it can also be used for running tests (but, unlike
--   <a>TestReporter</a>, it has to launch the tests manually if it wants
--   them to be run). It is therefore more general than
--   <a>TestReporter</a>. <a>TestReporter</a> is provided just for
--   convenience.
--   
--   The function's result should indicate whether all the tests passed.
--   
--   In the <a>TestManager</a> case, it's up to the ingredient author to
--   decide what the result should be. When no tests are run, the result
--   should probably be <a>True</a>. Sometimes, even if some tests run and
--   fail, it still makes sense to return <a>True</a>.
data Ingredient

-- | For the explanation on how the callback works, see the documentation
--   for <a>launchTestTree</a>.
TestReporter :: [OptionDescription] -> (OptionSet -> TestTree -> Maybe (StatusMap -> IO (Time -> IO Bool))) -> Ingredient
TestManager :: [OptionDescription] -> (OptionSet -> TestTree -> Maybe (IO Bool)) -> Ingredient

-- | Time in seconds. Used to measure how long the tests took to run.
type Time = Double

-- | Run the first <a>Ingredient</a> that agrees to be run.
--   
--   If no one accepts the task, return <a>Nothing</a>. This is usually a
--   sign of misconfiguration.
tryIngredients :: [Ingredient] -> OptionSet -> TestTree -> Maybe (IO Bool)

-- | Return the options which are relevant for the given ingredient.
--   
--   Note that this isn't the same as simply pattern-matching on
--   <a>Ingredient</a>. E.g. options for a <a>TestReporter</a>
--   automatically include <a>NumThreads</a>.
ingredientOptions :: Ingredient -> [OptionDescription]

-- | Like <tt>ingredientOption</tt>, but folds over multiple ingredients.
ingredientsOptions :: [Ingredient] -> [OptionDescription]

-- | A simple console UI
consoleTestReporter :: Ingredient

-- | The ingredient that provides the test listing functionality
listingTests :: Ingredient

-- | This option, when set to <a>True</a>, specifies that we should run in
--   the «list tests» mode
newtype ListTests
ListTests :: Bool -> ListTests

-- | Obtain the list of all tests in the suite
testsNames :: OptionSet -> TestTree -> [TestName]

-- | Generate a command line parser from a list of option descriptions
optionParser :: [OptionDescription] -> Parser OptionSet

-- | The command line parser for the test suite
suiteOptionParser :: [Ingredient] -> TestTree -> Parser OptionSet

-- | Parse the command line arguments and run the tests using the provided
--   ingredient list
defaultMainWithIngredients :: [Ingredient] -> TestTree -> IO ()

-- | Current status of a test
data Status

-- | test has not started running yet
NotStarted :: Status

-- | test is being run
Executing :: Progress -> Status

-- | test finished with a given result
Done :: Result -> Status

-- | A test result
data Result
Result :: Outcome -> String -> String -> Time -> Result

-- | Did the test fail? If so, why?
[resultOutcome] :: Result -> Outcome

-- | <a>resultDescription</a> may contain some details about the test. For
--   a passed test it's ok to leave it empty. Providers like SmallCheck and
--   QuickCheck use it to provide information about how many tests were
--   generated.
--   
--   For a failed test, <a>resultDescription</a> should typically provide
--   more information about the failure.
[resultDescription] :: Result -> String

-- | The short description printed in the test run summary, usually
--   <tt>OK</tt> or <tt>FAIL</tt>.
[resultShortDescription] :: Result -> String

-- | How long it took to run the test, in seconds.
[resultTime] :: Result -> Time

-- | Outcome of a test run
--   
--   Note: this is isomorphic to <tt><a>Maybe</a>
--   <a>FailureReason</a></tt>. You can use the <tt>generic-maybe</tt>
--   package to exploit that.
data Outcome

-- | test succeeded
Success :: Outcome

-- | test failed because of the <a>FailureReason</a>
Failure :: FailureReason -> Outcome

-- | If a test failed, <a>FailureReason</a> describes why
data FailureReason

-- | test provider indicated failure of the code to test, either because
--   the tested code returned wrong results, or raised an exception
TestFailed :: FailureReason

-- | the test code itself raised an exception. Typical cases include
--   missing example input or output files.
--   
--   Usually, providers do not have to implement this, as their <a>run</a>
--   method may simply raise an exception.
TestThrewException :: SomeException -> FailureReason

-- | test didn't complete in allotted time
TestTimedOut :: Integer -> FailureReason

-- | <a>True</a> for a passed test, <a>False</a> for a failed one.
resultSuccessful :: Result -> Bool

-- | Test progress information.
--   
--   This may be used by a runner to provide some feedback to the user
--   while a long-running test is executing.
data Progress
Progress :: String -> Float -> Progress

-- | textual information about the test's progress
[progressText] :: Progress -> String

-- | <a>progressPercent</a> should be a value between 0 and 1. If it's
--   impossible to compute the estimate, use 0.
[progressPercent] :: Progress -> Float

-- | Mapping from test numbers (starting from 0) to their status variables.
--   
--   This is what an ingredient uses to analyse and display progress, and
--   to detect when tests finish.
type StatusMap = IntMap (TVar Status)

-- | Start running all the tests in a test tree in parallel, without
--   blocking the current thread. The number of test running threads is
--   determined by the <a>NumThreads</a> option.
launchTestTree :: OptionSet -> TestTree -> (StatusMap -> IO (Time -> IO a)) -> IO a

-- | Number of parallel threads to use for running tests.
--   
--   Note that this is <i>not</i> included in <a>coreOptions</a>. Instead,
--   it's automatically included in the options for any
--   <tt>TestReporter</tt> ingredient by <tt>ingredientOptions</tt>,
--   because the way test reporters are handled already involves
--   parallelism. Other ingredients may also choose to include this option.
newtype NumThreads
NumThreads :: Int -> NumThreads
[getNumThreads] :: NumThreads -> Int

-- | All the options relevant for this test suite. This includes the
--   options for the test tree and ingredients, and the core options.
suiteOptions :: [Ingredient] -> TestTree -> [OptionDescription]

-- | The list of all core options, i.e. the options not specific to any
--   provider or ingredient, but to tasty itself. Currently contains
--   <a>TestPattern</a> and <a>Timeout</a>.
coreOptions :: [OptionDescription]

-- | A pattern to filter tests. For the syntax description, see
--   <a>http://documentup.com/feuerbach/tasty#using-patterns</a>
data TestPattern

-- | Parse a pattern
parseTestPattern :: String -> TestPattern

-- | A pattern that matches anything.
noPattern :: TestPattern

-- | Test a path (which is the sequence of group titles, possibly followed
--   by the test title) against a pattern
testPatternMatches :: TestPattern -> [String] -> Bool

-- | Catch possible exceptions that may arise when evaluating a string. For
--   normal (total) strings, this is a no-op.
--   
--   This function should be used to display messages generated by the test
--   suite (such as test result descriptions).
--   
--   See e.g. <a>https://github.com/feuerbach/tasty/issues/25</a>
formatMessage :: String -> IO String


-- | This module defines the main data types and functions needed to use
--   Tasty.
module Test.Tasty

-- | The name of a test or a group of tests
type TestName = String

-- | The main data structure defining a test suite.
--   
--   It consists of individual test cases and properties, organized in
--   named groups which form a tree-like hierarchy.
--   
--   There is no generic way to create a test case. Instead, every test
--   provider (tasty-hunit, tasty-smallcheck etc.) provides a function to
--   turn a test case into a <a>TestTree</a>.
--   
--   Groups can be created using <a>testGroup</a>.
data TestTree

-- | Create a named group of test cases or other groups
testGroup :: TestName -> [TestTree] -> TestTree

-- | Parse the command line arguments and run the tests
defaultMain :: TestTree -> IO ()

-- | Parse the command line arguments and run the tests using the provided
--   ingredient list
defaultMainWithIngredients :: [Ingredient] -> TestTree -> IO ()

-- | List of the default ingredients. This is what <a>defaultMain</a> uses.
--   
--   At the moment it consists of <a>listingTests</a> and
--   <a>consoleTestReporter</a>.
defaultIngredients :: [Ingredient]

-- | This ingredient doesn't do anything apart from registering additional
--   options.
--   
--   The option values can be accessed using <tt>askOption</tt>.
includingOptions :: [OptionDescription] -> Ingredient

-- | Locally adjust the option value for the given test subtree
adjustOption :: IsOption v => (v -> v) -> TestTree -> TestTree

-- | Locally set the option value for the given test subtree
localOption :: IsOption v => v -> TestTree -> TestTree

-- | Customize the test tree based on the run-time options
askOption :: IsOption v => (v -> TestTree) -> TestTree

-- | Timeout to be applied to individual tests
data Timeout

-- | <a>String</a> is the original representation of the timeout (such as
--   <tt>"0.5m"</tt>), so that we can print it back. <a>Integer</a> is the
--   number of microseconds.
Timeout :: Integer -> String -> Timeout
NoTimeout :: Timeout

-- | A shortcut for creating <a>Timeout</a> values
mkTimeout :: Integer -> Timeout

-- | Acquire the resource to run this test (sub)tree and release it
--   afterwards
withResource :: IO a -> (a -> IO ()) -> (IO a -> TestTree) -> TestTree