Improve coverage for AbuseFilterTokenizer

This will make tokenizer almost fully covered. The only uncovered parts
are the one with cache and an else condition which I think won't ever be
executed, and thus added a comment for that. Also, remove an obsolete
xxx comment from ComputedVariable (fixed in
I8e420f0259ef6c9e579f7a00beb58f28af9da37d)

Bug: T201193
Change-Id: I6e9a73aa9e437f096f6a1e20d53a7cb50e5ed85d

includes/AFComputedVariable.php

@@ -206,7 +206,6 @@ class AFComputedVariable {
 				if ( $article->getContentModel() === CONTENT_MODEL_WIKITEXT ) {
 					$textVar = $parameters['text-var'];
 
-					// XXX: Use prepareContentForEdit. But we need a Content object for that.
 					$new_text = $vars->getVar( $textVar )->toString();
 					$content = ContentHandler::makeContent( $new_text, $article->getTitle() );
 					$editInfo = $article->prepareContentForEdit( $content );

includes/parser/AbuseFilterTokenizer.php

@@ -259,6 +259,7 @@ class AbuseFilterTokenizer {
 				$offset += 2;
 
 			} else {
+				// Should never happen
 				$token .= $code[$offset];
 				$offset++;
 			}

tests/parserTests/string.t

@@ -1 +1,6 @@
-"a\tb" = "a	b" & "a\qb" = "a\qb"
+"a\tb" === "a	b" &
+"a\qb" === "a\qb" &
+"a\"b" === 'a"b' &
+"a\rb" !== "a\r\nb" &
+"\x66\x6f\x6f" === "foo" &
+"some\xstring" === "somexstring"