Merge "Refactor unit tests"

tests/engines/LuaCommon/CommonTests.lua

@@ -1,86 +1,26 @@
--- Parts are based on lua-TestMore (Copyright © 2009-2012 François Perrad, MIT
--- license)
+local testframework = require 'Module:TestFramework'
 
 local test = {}
 
-local function is_deeply (got, expected, name)
-    if type(got) ~= 'table' then
-        return false
-    elseif type(expected) ~= 'table' then
-        error("expected value isn't a table : " .. tostring(expected))
-    end
-    local msg1
-    local msg2
-
-    local function deep_eq (t1, t2, key_path)
-        if t1 == t2 then
-            return true
-        end
-        for k, v2 in pairs(t2) do
-            local v1 = t1[k]
-            if type(v1) == 'table' and type(v2) == 'table' then
-                local r = deep_eq(v1, v2, key_path .. "." .. tostring(k))
-                if not r then
-                    return false
-                end
-            else
-                if v1 ~= v2 then
-                    key_path = key_path .. "." .. tostring(k)
-                    msg1 = "     got" .. key_path .. ": " .. tostring(v1)
-                    msg2 = "expected" .. key_path .. ": " .. tostring(v2)
-                    return false
-                end
-            end
-        end
-        for k in pairs(t1) do
-            local v2 = t2[k]
-            if v2 == nil then
-                key_path = key_path .. "." .. tostring(k)
-                msg1 = "     got" .. key_path .. ": " .. tostring(t1[k])
-                msg2 = "expected" .. key_path .. ": " .. tostring(v2)
-                return false
-            end
-        end
-        return true
-    end -- deep_eq
-
-	return deep_eq(got, expected, '')
-end
-
-function test.getTests( engine )
-	return {
-		{ 'clone1', 'ok' },
-		{ 'clone2', 'ok' },
-		{ 'clone3', 'ok' },
-		{ 'clone4', 'ok' },
-		{ 'setfenv1', { error = '%s cannot set the global %s' } },
-		{ 'setfenv2', { error = '%s cannot set an environment %s' } },
-		{ 'setfenv3', { error = '%s cannot set the requested environment%s' } },
-		{ 'setfenv4', 'ok' },
-		{ 'setfenv5', 'ok' },
-		{ 'setfenv6', { error = '%s cannot be called on a protected function' } },
-		{ 'setfenv7', { error = '%s can only be called with a function%s' } },
-		{ 'getfenv1', 'ok' },
-		{ 'getfenv2', { error = '%s cannot get the global environment' } },
-		{ 'getfenv3', { error = '%Sno function environment for tail call %s' } },
-	}
-end
-
 function test.clone1()
 	local x = 1
 	local y = mw.clone( x )
-	assert( x == y )
-	return 'ok'
+	return ( x == y )
 end
 
 function test.clone2()
 	local x = { 'a' }
 	local y = mw.clone( x )
 	assert( x ~= y )
-	assert( is_deeply( y, x ) )
+	return testframework.deepEquals( x, y )
+end
+
+function test.clone2b()
+	local x = { 'a' }
+	local y = mw.clone( x )
+	assert( x ~= y )
 	y[2] = 'b'
-	assert( not is_deeply( y, x ) )
-	return 'ok'
+	return testframework.deepEquals( x, y )
 end
 
 function test.clone3()
@@ -89,8 +29,7 @@ function test.clone3()
 	setmetatable( x, mt )
 	local y = mw.clone( x )
 	assert( getmetatable( x ) ~= getmetatable( y ) )
-	assert( is_deeply( getmetatable( y ), getmetatable( x ) ) )
-	return 'ok'
+	return testframework.deepEquals( getmetatable( x ), getmetatable( y ) )
 end
 
 function test.clone4()
@@ -98,8 +37,7 @@ function test.clone4()
 	x.x = x
 	local y = mw.clone( x )
 	assert( x ~= y )
-	assert( y == y.x )
-	return 'ok'
+	return y == y.x
 end
 
 function test.setfenv1()
@@ -120,8 +58,8 @@ function test.setfenv3()
 end
 
 function test.setfenv4()
-	-- Set an unprotected environment at a higher stack level than a protected 
-	-- environment. It's assumed that any higher-level environment will protect 
+	-- Set an unprotected environment at a higher stack level than a protected
+	-- environment. It's assumed that any higher-level environment will protect
 	-- itself with its own setfenv wrapper, so this succeeds.
 	local function level3()
 		local function level2()
@@ -129,7 +67,7 @@ function test.setfenv4()
 			local function level1()
 				setfenv( 3, {} )
 			end
-			
+
 			setfenv( level1, env )()
 		end
 		local protected = {mw = mw}
@@ -165,10 +103,6 @@ function test.setfenv7()
 	setfenv( {}, {} )
 end
 
-function test.setfenv8()
-	setfenv( 2, {} )
-end
-
 function test.getfenv1()
 	assert( getfenv( 1 ) == _G )
 	return 'ok'
@@ -187,7 +121,62 @@ function test.getfenv3()
 		return foo()
 	end
 
-	bar()
+	-- The "at level #" bit varies between environments, so
+	-- catch the error and strip that part out
+	local ok, err = pcall( bar )
+	if not ok then
+		err = string.gsub( err, '^%S+:%d+: ', '' )
+		err = string.gsub( err, ' at level %d$', '' )
+		error( err )
+	end
 end
 
-return test
+return testframework.getTestProvider( {
+	{ name = 'clone', func = test.clone1,
+	  expect = { true },
+	},
+	{ name = 'clone table', func = test.clone2,
+	  expect = { true },
+	},
+	{ name = 'clone table then modify', func = test.clone2b,
+	  expect = { false, { 2 }, nil, 'b' },
+	},
+	{ name = 'clone table with metatable', func = test.clone3,
+	  expect = { true },
+	},
+	{ name = 'clone recursive table', func = test.clone4,
+	  expect = { true },
+	},
+
+	{ name = 'setfenv global', func = test.setfenv1,
+	  expect = "'setfenv' cannot set the global environment, it is protected",
+	},
+	{ name = 'setfenv invalid level', func = test.setfenv2,
+	  expect = "'setfenv' cannot set an environment at a level greater than 10",
+	},
+	{ name = 'setfenv invalid environment', func = test.setfenv3,
+	  expect = "'setfenv' cannot set the requested environment, it is protected",
+	},
+	{ name = 'setfenv on unprotected past protected', func = test.setfenv4,
+	  expect = { 'ok' },
+	},
+	{ name = 'setfenv from inside protected', func = test.setfenv5,
+	  expect = { 'ok' },
+	},
+	{ name = 'setfenv protected function', func = test.setfenv6,
+	  expect = "'setfenv' cannot be called on a protected function",
+	},
+	{ name = 'setfenv on a non-function', func = test.setfenv7,
+	  expect = "'setfenv' can only be called with a function or integer as the first argument",
+	},
+
+	{ name = 'getfenv(1)', func = test.getfenv1,
+	  expect = { 'ok' },
+	},
+	{ name = 'getfenv(0)', func = test.getfenv2,
+	  expect = "'getfenv' cannot get the global environment",
+	},
+	{ name = 'getfenv with tail call', func = test.getfenv3,
+	  expect = "no function environment for tail call",
+	},
+} )

tests/engines/LuaCommon/LuaDataProvider.php

@@ -0,0 +1,49 @@
+<?php
+
+class LuaDataProvider implements Iterator {
+	protected $engine = null;
+	protected $exports = null;
+	protected $key = 1;
+
+	public function __construct( $engine, $moduleName ) {
+		$this->engine = $engine;
+		$this->key = 1;
+		$module = $engine->fetchModuleFromParser(
+			Title::makeTitle( NS_MODULE, $moduleName )
+		);
+		if ( $module === null ) {
+			throw new Exception( "Failed to load module $moduleName" );
+		}
+		$this->exports = $module->execute();
+	}
+
+	public function destroy() {
+		$this->engine = null;
+		$this->exports = null;
+	}
+
+	public function rewind() {
+		$this->key = 1;
+	}
+
+	public function valid() {
+		return $this->key <= $this->exports['count'];
+	}
+
+	public function key() {
+		return $this->key;
+	}
+
+	public function next() {
+		$this->key++;
+	}
+
+	public function current() {
+		return $this->engine->getInterpreter()->callFunction( $this->exports['provide'], $this->key );
+	}
+
+	public function run( $key ) {
+		list( $ret ) = $this->engine->getInterpreter()->callFunction( $this->exports['run'], $key );
+		return $ret;
+	}
+}

tests/engines/LuaCommon/LuaEngineTest.php

@@ -1,6 +1,15 @@
 <?php
 
+// To add additional test modules, add the module to getTestModules() and
+// implement a data provider method and test method, using provideCommonTests()
+// and testCommonTests() as a template.
+
+require_once( __DIR__ . '/LuaDataProvider.php' );
+
 abstract class Scribunto_LuaEngineTest extends MediaWikiTestCase {
+	private $engine = null;
+	private $dataProviders = array();
+	private $luaTestName = null;
 
 	abstract function newEngine( $opts = array() );
 
@@ -13,22 +22,28 @@ abstract class Scribunto_LuaEngineTest extends MediaWikiTestCase {
 		}
 	}
 
+	function tearDown() {
+		foreach ( $this->dataProviders as $k => $p ) {
+			$p->destroy();
+		}
+		$this->dataProviders = array();
+		if ( $this->engine ) {
+			$this->engine->destroy();
+			$this->engine = null;
+		}
+		parent::tearDown();
+	}
+
 	function getEngine() {
+		if ( $this->engine ) {
+			return $this->engine;
+		}
 		$parser = new Parser;
 		$options = new ParserOptions;
 		$options->setTemplateCallback( array( $this, 'templateCallback' ) );
 		$parser->startExternalParse( Title::newMainPage(), $options, Parser::OT_HTML, true );
-		return $this->newEngine( array( 'parser' => $parser ) );
-	}
-
-	function getFrame( $engine ) {
-		return $engine->getParser()->getPreprocessor()->newFrame();
-	}
-
-	function getTestModules() {
-		return array(
-			'CommonTests' => dirname( __FILE__ ) . '/CommonTests.lua'
-		);
+		$this->engine = $this->newEngine( array( 'parser' => $parser ) );
+		return $this->engine;
 	}
 
 	function templateCallback( $title, $parser ) {
@@ -46,52 +61,43 @@ abstract class Scribunto_LuaEngineTest extends MediaWikiTestCase {
 		return Parser::statelessFetchTemplate( $title, $parser );
 	}
 
-	function getTestModuleName() {
-		return 'CommonTests';
-	}
-
-	function getTestModule( $engine, $moduleName ) {
-		return $engine->fetchModuleFromParser( 
-			Title::makeTitle( NS_MODULE, $moduleName ) );
-	}
-
-	function testProvider() {
-		$tests = $this->provideLua();
-		$this->assertGreaterThan( 2, count( $tests ) );
-	}
-
-	function provideLua() {
-		$engine = $this->getEngine();
-		$allTests = array();
-		foreach ( $this->getTestModules() as $moduleName => $fileName ) {
-			$module = $this->getTestModule( $engine, $moduleName );
-			$exports = $module->execute();
-			$result = $engine->getInterpreter()->callFunction( $exports['getTests'] );
-			$moduleTests = $result[0];
-			foreach ( $moduleTests as $test ) {
-				array_unshift( $test, $moduleName );
-				$allTests[] = $test;
-			}
+	function toString() {
+		// When running tests written in Lua, return a nicer representation in
+		// the failure message.
+		if ( $this->luaTestName ) {
+			return $this->luaTestName;
 		}
-		return $allTests;
+		return parent::toString();
 	}
 
-	/** @dataProvider provideLua */
-	function testLua( $moduleName, $testName, $expected ) {
-		$engine = $this->getEngine();
-		$module = $this->getTestModule( $engine, $moduleName );
-		if ( is_array( $expected ) && isset( $expected['error'] ) ) {
-			$caught = false;
-			try {
-				$ret = $module->invoke( $testName, $this->getFrame( $engine ) );
-			} catch ( Scribunto_LuaError $e ) {
-				$caught = true;
-				$this->assertStringMatchesFormat( $expected['error'], $e->getLuaMessage() );
-			}
-			$this->assertTrue( $caught, 'expected an exception' );
-		} else {
-			$ret = $module->invoke( $testName, $this->getFrame( $engine ) );
-			$this->assertSame( $expected, $ret );
+	function getTestModules() {
+		return array(
+			'TestFramework' => __DIR__ . '/TestFramework.lua',
+			'CommonTests' => __DIR__ . '/CommonTests.lua',
+		);
+	}
+
+	function getTestProvider( $moduleName ) {
+		if ( !isset( $this->dataProviders[$moduleName] ) ) {
+			$this->dataProviders[$moduleName] = new LuaDataProvider( $this->getEngine(), $moduleName );
 		}
+		return $this->dataProviders[$moduleName];
+	}
+
+	function runTestProvider( $moduleName, $key, $testName, $expected ) {
+		$this->luaTestName = "{$moduleName}[$key]: $testName";
+		$dataProvider = $this->getTestProvider( $moduleName );
+		$actual = $dataProvider->run( $key );
+		$this->assertSame( $expected, $actual );
+		$this->luaTestName = null;
+	}
+
+	function provideCommonTests() {
+		return $this->getTestProvider( 'CommonTests' );
+	}
+
+	/** @dataProvider provideCommonTests */
+	function testCommonTests( $key, $testName, $expected ) {
+		$this->runTestProvider( 'CommonTests', $key, $testName, $expected );
 	}
 }

tests/engines/LuaCommon/TestFramework.lua

@@ -0,0 +1,207 @@
+testframework = testframework or {}
+
+-- Return a string represetation of a value, including the deep structure of a table
+local function deepToString( val, indent, done )
+	done = done or {}
+	indent = indent or 0
+
+	local tp = type( val )
+	if tp == 'string' then
+		return string.format( "%q", val )
+	elseif tp == 'table' then
+		if done[val] then return '{ ... }' end
+		done[val] = true
+		local sb = { '{\n' }
+		local donekeys = {}
+		for key, value in ipairs( val ) do
+			donekeys[key] = true
+			sb[#sb + 1] = string.rep( " ", indent + 2 )
+			sb[#sb + 1] = deepToString( value, indent + 2, done )
+			sb[#sb + 1] = ",\n"
+		end
+		local keys = {}
+		for key in pairs( val ) do
+			if not donekeys[key] then
+				keys[#keys + 1] = key
+			end
+		end
+		table.sort( keys )
+		for i = 1, #keys do
+			local key = keys[i]
+			sb[#sb + 1] = string.rep( " ", indent + 2 )
+			if type( key ) == 'table' then
+				sb[#sb + 1] = '[{ ... }] = '
+			else
+				sb[#sb + 1] = '['
+				sb[#sb + 1] = deepToString( key, indent + 3, done )
+				sb[#sb + 1] = '] = '
+			end
+			sb[#sb + 1] = deepToString( val[key], indent + 2, done )
+			sb[#sb + 1] = ",\n"
+		end
+		sb[#sb + 1] = string.rep( " ", indent )
+		sb[#sb + 1] = "}"
+		return table.concat( sb )
+	else
+		return tostring( val )
+	end
+end
+testframework.deepToString = deepToString
+
+-- Test whether two objects are equal, including the deep structure of a table.
+-- Returns 4 values:
+--  boolean  equal?
+--  list     key path to first inequality
+--  mixed    value from 'a' for key path
+--  mixed    value from 'b' for key path
+local function deepEquals( a, b, keypath, done )
+	-- Simple equality
+	if a == b then
+		return true
+	end
+
+	keypath = keypath or {}
+	done = done or {}
+
+	-- Must be equal types to be equal
+	local tp = type( a )
+	if type( b ) ~= tp then
+		return false, keypath, a, b
+	end
+
+	-- Special tests for certain types
+
+	if tp == 'number' then
+		-- For test framework purposes, NaNs are equivalent. Lua has no
+		-- standard "isNaN" function, but only NaN will return true for
+		-- "x ~= x".
+		if a ~= a and b ~= b then
+			return true
+		end
+
+		return false, keypath, a, b
+	end
+
+	if tp == 'table' then
+		-- To avoid recursion, see if we've seen this pair of tables before. If
+		-- so, they must be equal or the test would have failed the first time we saw them.
+		done[a] = done[a] or {}
+		done[b] = done[b] or {}
+		if done[a][b] or done[b][a] then
+			return true
+		end
+
+		-- Not seen before, record them and compare key by key.
+		done[a][b] = true
+
+		local n = #keypath + 1
+		-- First, check if the values for all keys in 'a' are equal in 'b'.
+		for k in pairs( a ) do
+			keypath[n] = k
+			local ok, kp, aa, bb = deepEquals( a[k], b[k], keypath, done )
+			if not ok then
+				return false, kp, aa, bb
+			end
+		end
+		keypath[n] = nil
+
+		-- Then check if there are any keys in 'b' that don't exist in 'a'.
+		for k, v in pairs( b ) do
+			if a[k] == nil then
+				keypath[n] = k
+				return false, keypath, nil, v
+			end
+		end
+
+		-- Ok, all keys equal so it must match.
+		return true
+	end
+
+	-- Ok, they're not equal
+	return false, keypath, a, b
+end
+testframework.deepEquals = deepEquals
+
+---- Test types available ---
+-- Each type has a formatter and an executor:
+--  Formatters take 1 arg: expected return value from the function.
+--  Executors take 2 args: function and arguments.
+--  Both return a string. The test passes if the two strings match.
+testframework.types = testframework.types or {}
+
+-- Execute a function and assert expected results
+-- Expected value is a list of return values, or a string error message
+testframework.types.Normal = {
+	format = function ( expect )
+		if type( expect ) == 'string' then
+			return 'ERROR: ' .. expect
+		else
+			return deepToString( expect )
+		end
+	end,
+	exec = function ( func, args )
+		local got = { pcall( func, unpack( args ) ) }
+		if table.remove( got, 1 ) then
+			return deepToString( got )
+		else
+			got = string.gsub( got[1], '^%S+:%d+: ', '' )
+			return 'ERROR: ' .. got
+		end
+	end
+}
+
+-- Execute an iterator-returning function and assert expected results from each
+-- iteration.
+-- Expected value is a list of return value lists.
+testframework.types.Iterator = {
+	format = function ( expect )
+		local sb = {}
+		for i = 1, #expect do
+			sb[i] = '[iteration ' .. i .. ']:\n' .. deepToString( expect[i] )
+		end
+		return table.concat( sb, '\n\n' )
+	end,
+	exec = function ( func, args )
+		local sb = {}
+		local i = 0
+		local f, s, var = func( unpack( args ) )
+		while true do
+			local got = { f( s, var ) }
+			var = got[1]
+			if var == nil then break end
+			i = i + 1
+			sb[i] = '[iteration ' .. i .. ']:\n' .. deepToString( got )
+		end
+		return table.concat( sb, '\n\n' )
+	end
+}
+
+-- This takes a list of tests to run, and returns the object used by PHP to
+-- call them.
+--
+-- Each test is a table with the following keys:
+--  name: Name of the test
+--  expect: Table of results expected
+--  func: Function to execute
+--  args: (optional) Table of args to be unpacked and passed to the function
+--  type: (optional) Formatter/Executor name, default "Normal"
+function testframework.getTestProvider( tests )
+	return {
+		count = #tests,
+
+		provide = function ( n )
+			local t = tests[n]
+			return n, t.name, testframework.types[t.type or 'Normal'].format( t.expect )
+		end,
+
+		run = function ( n )
+			local t = tests[n]
+			if not t then
+				return 'Test ' .. name .. ' does not exist'
+			end
+			return testframework.types[t.type or 'Normal'].exec( t.func, t.args or {} )
+		end,
+	}
+end
+
+return testframework

tests/engines/LuaSandbox/LuaSandboxEngineTest.php

@@ -1,7 +1,7 @@
 <?php
 
 if ( php_sapi_name() !== 'cli' ) exit;
-require_once( dirname( __FILE__ ) .'/../LuaCommon/LuaEngineTest.php' );
+require_once( __DIR__ . '/../LuaCommon/LuaEngineTest.php' );
 
 class Scribunto_LuaSandboxEngineTest extends Scribunto_LuaEngineTest {
 	var $stdOpts = array(
@@ -17,8 +17,17 @@ class Scribunto_LuaSandboxEngineTest extends Scribunto_LuaEngineTest {
 
 	function getTestModules() {
 		return parent::getTestModules() + array(
-			'SandboxTests' => dirname( __FILE__ ) . '/SandboxTests.lua'
+			'SandboxTests' => __DIR__ . '/SandboxTests.lua'
 		);
 	}
+
+	function provideSandboxTests() {
+		return $this->getTestProvider( 'SandboxTests' );
+	}
+
+	/** @dataProvider provideSandboxTests */
+	function testSandboxTests( $key, $testName, $expected ) {
+		$this->runTestProvider( 'SandboxTests', $key, $testName, $expected );
+	}
 }
 

tests/engines/LuaSandbox/SandboxTests.lua

@@ -1,19 +1,18 @@
-local test = require( 'Module:CommonTests' )
-local sbtest = {}
+local testframework = require( 'Module:TestFramework' )
 
-function sbtest.getTests()
-	return {
-		{ 'setfenv1', { error = '%sinvalid level%s' } },
-		{ 'getfenv1', { error = '%sinvalid level%s' } },
-	}
+local function setfenv1()
+	setfenv( 5, {} )
 end
 
-function sbtest.setfenv1()
-	setfenv( 3, {} )
+local function getfenv1()
+	assert( getfenv( 5 ) == nil )
 end
 
-function sbtest.getfenv1()
-	assert( getfenv( 3 ) == nil )
-end
-
-return sbtest
+return testframework.getTestProvider( {
+	{ name = 'setfenv invalid level', func = setfenv1,
+	  expect = "bad argument #1 to 'old_getfenv' (invalid level)",
+	},
+	{ name = 'getfenv invalid level', func = getfenv1,
+	  expect = "bad argument #1 to 'old_getfenv' (invalid level)",
+	},
+} )

tests/engines/LuaStandalone/LuaStandaloneEngineTest.php

@@ -1,7 +1,7 @@
 <?php
 
 if ( php_sapi_name() !== 'cli' ) exit;
-require_once( dirname( __FILE__ ) .'/../LuaCommon/LuaEngineTest.php' );
+require_once( __DIR__ . '/../LuaCommon/LuaEngineTest.php' );
 
 class Scribunto_LuaStandaloneEngineTest extends Scribunto_LuaEngineTest {
 	var $stdOpts = array(
@@ -16,11 +16,20 @@ class Scribunto_LuaStandaloneEngineTest extends Scribunto_LuaEngineTest {
 		$opts = $opts + $this->stdOpts;
 		return new Scribunto_LuaStandaloneEngine( $opts );
 	}
-	
+
 	function getTestModules() {
 		return parent::getTestModules() + array(
-			'StandloneTests' => dirname( __FILE__ ) . '/StandaloneTests.lua'
+			'StandaloneTests' => __DIR__ . '/StandaloneTests.lua'
 		);
 	}
+
+	function provideStandaloneTests() {
+		return $this->getTestProvider( 'StandaloneTests' );
+	}
+
+	/** @dataProvider provideStandaloneTests */
+	function testStandaloneTests( $key, $testName, $expected ) {
+		$this->runTestProvider( 'StandaloneTests', $key, $testName, $expected );
+	}
 }
 

tests/engines/LuaStandalone/StandaloneTests.lua

@@ -1,20 +1,28 @@
-local test = require( 'Module:CommonTests' )
-local satest = {}
+local testframework = require( 'Module:TestFramework' )
 
-function satest.getTests()
-	return {
-		{ 'setfenv1', { error = '%s cannot set the requested environment%s' } },
-		{ 'getfenv1', 'ok' },
-	}
+local function setfenv1()
+	local ok, err = pcall( function()
+		setfenv( 2, {} )
+	end )
+	if not ok then
+		err = string.gsub( err, '^%S+:%d+: ', '' )
+		error( err )
+	end
 end
 
-function satest.setfenv1()
-	setfenv( 4, {} )
+local function getfenv1()
+	local env
+	pcall( function()
+		env = getfenv( 2 )
+	end )
+	return env
 end
 
-function satest.getfenv1()
-	assert( getfenv( 4 ) == nil )
-	return 'ok'
-end
-
-return satest
+return testframework.getTestProvider( {
+	{ name = 'setfenv on a C function', func = setfenv1,
+	  expect = "'setfenv' cannot set the requested environment, it is protected",
+	},
+	{ name = 'getfenv on a C function', func = getfenv1,
+	  expect = { nil },
+	},
+} )