/*
 * Copyright 2007 Kasper B. Graversen
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 *     http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.supercsv.io;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.supercsv.prefs.CsvPreference.EXCEL_PREFERENCE;

import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.List;

import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.supercsv.comment.CommentMatches;
import org.supercsv.comment.CommentStartsWith;
import org.supercsv.exception.SuperCsvException;
import org.supercsv.prefs.CsvPreference;

public class TokenizerTest {
	
	private static final CsvPreference NORMAL_PREFERENCE = EXCEL_PREFERENCE;
	private static final CsvPreference SPACES_NEED_QUOTES_PREFERENCE = new CsvPreference.Builder(EXCEL_PREFERENCE)
		.surroundingSpacesNeedQuotes(true).build();
	private static final CsvPreference DONT_IGNORE_EMPTY_LINES_PREFERENCE = new CsvPreference.Builder(EXCEL_PREFERENCE)
		.ignoreEmptyLines(false).build();
	
	private Tokenizer tokenizer;
	private List<String> columns;
	
	/**
	 * Sets up the columns List for the test.
	 * 
	 * @throws Exception
	 */
	@Before
	public void setUp() {
		columns = new ArrayList<String>();
	}
	
	/**
	 * Tidies up after the test.
	 */
	@After
	public void tearDown() throws IOException {
		if( tokenizer != null ) {
			tokenizer.close();
		}
	}
	
	/**
	 * Creates a Tokenizer with the input and preferences.
	 * 
	 * @param input
	 *            the input String
	 * @param preference
	 *            the preferences
	 * @return the Tokenizer
	 */
	private static Tokenizer createTokenizer(String input, CsvPreference preference) {
		final Reader r = input != null ? new StringReader(input) : null;
		return new Tokenizer(r, preference);
	}
	
	/**
	 * Tests the constructor with a null Reader (should throw an Exception).
	 */
	@Test(expected = NullPointerException.class)
	public void testConstructorWithNullReader() throws Exception {
		createTokenizer(null, NORMAL_PREFERENCE);
	}
	
	/**
	 * Tests the constructor with a null CsvPreference (should throw an Exception).
	 */
	@Test(expected = NullPointerException.class)
	public void testConstructorWithNullPreferences() throws Exception {
		createTokenizer("", null);
	}
	
	/**
	 * Tests the readColumns() method with null List (should throw an Exception).
	 */
	@Test(expected = NullPointerException.class)
	public void testReadColumnsWithNullList() throws Exception {
		tokenizer = createTokenizer("", NORMAL_PREFERENCE);
		tokenizer.readColumns(null);
	}
	
	/**
	 * Tests the getPreferences() method.
	 */
	@Test()
	public void testGetPreferences() throws Exception {
		tokenizer = createTokenizer("", NORMAL_PREFERENCE);
		CsvPreference prefs = tokenizer.getPreferences();
		assertEquals(NORMAL_PREFERENCE.getDelimiterChar(), prefs.getDelimiterChar());
		assertEquals(NORMAL_PREFERENCE.getEndOfLineSymbols(), prefs.getEndOfLineSymbols());
		assertEquals(NORMAL_PREFERENCE.getQuoteChar(), prefs.getQuoteChar());
		assertEquals(NORMAL_PREFERENCE.isSurroundingSpacesNeedQuotes(), prefs.isSurroundingSpacesNeedQuotes());
	}
	
	/**
	 * Tests the readColumns() method with no data.
	 */
	@Test
	public void testReadColumnsWithNoData() throws Exception {
		final String input = "";
		tokenizer = createTokenizer(input, NORMAL_PREFERENCE);
		tokenizer.readColumns(columns);
		assertTrue(columns.isEmpty());
		assertEquals(input, tokenizer.getUntokenizedRow());
	}
	
	/**
	 * Tests that the readColumns() method skips over empty lines.
	 */
	@Test
	public void testEmptyLines() throws Exception {
		
		final String input = "\n\nthis is the third line\n";
		tokenizer = createTokenizer(input, NORMAL_PREFERENCE);
		tokenizer.readColumns(columns);
		assertTrue(columns.size() == 1);
		assertEquals("this is the third line", columns.get(0));
		assertEquals(3, tokenizer.getLineNumber());
		assertEquals("this is the third line", tokenizer.getUntokenizedRow());
	}
	
	/**
	 * Tests that the readColumns() method doesn't skip over empty lines if the ignoreEmptyLines
	 * preference is disabled.
	 */
	@Test
	public void testEmptyLinesWithIgnoreEmptyLines() throws Exception {
		
		final String input = "\nthis is the second line\n\n";
		tokenizer = createTokenizer(input, DONT_IGNORE_EMPTY_LINES_PREFERENCE);
		tokenizer.readColumns(columns);
		assertTrue(columns.size() == 1);
		assertNull(columns.get(0));
		assertEquals(1, tokenizer.getLineNumber());
		assertEquals("", tokenizer.getUntokenizedRow());
		
		tokenizer.readColumns(columns);
		assertTrue(columns.size() == 1);
		assertEquals("this is the second line", columns.get(0));
		assertEquals(2, tokenizer.getLineNumber());
		assertEquals("this is the second line", tokenizer.getUntokenizedRow());
		
		tokenizer.readColumns(columns);
		assertTrue(columns.size() == 1);
		assertNull(columns.get(0));
		assertEquals(3, tokenizer.getLineNumber());
		assertEquals("", tokenizer.getUntokenizedRow());
	}
	
	/**
	 * Tests the readColumns() method a quoted section has text surrounding it. This is not technically valid CSV, but
	 * the tokenizer is lenient enough to allow it (it will just unescape the quoted section).
	 */
	@Test
	public void testQuotedFieldWithSurroundingText() throws Exception {
		
		final String input = "surrounding \"quoted\" text";
		tokenizer = createTokenizer(input, NORMAL_PREFERENCE);
		tokenizer.readColumns(columns);
		assertTrue(columns.size() == 1);
		assertEquals("surrounding quoted text", columns.get(0));
		assertEquals(1, tokenizer.getLineNumber());
		assertEquals(input, tokenizer.getUntokenizedRow());
		
		// same result when surrounding spaces require quotes
		tokenizer = createTokenizer(input, SPACES_NEED_QUOTES_PREFERENCE);
		tokenizer.readColumns(columns);
		assertTrue(columns.size() == 1);
		assertEquals("surrounding quoted text", columns.get(0));
		assertEquals(1, tokenizer.getLineNumber());
		assertEquals(input, tokenizer.getUntokenizedRow());
	}
	
	/**
	 * Tests the readColumns() method when a quoted section with text after it. This is not technically valid CSV, but
	 * the tokenizer is lenient enough to allow it (it will just unescape the quoted section).
	 */
	@Test
	public void testQuotedFieldWithTextAfter() throws Exception {
		
		// illegal char after quoted section
		final String input = "\"quoted on 2 lines\nand afterward some\" text";
		tokenizer = createTokenizer(input, NORMAL_PREFERENCE);
		tokenizer.readColumns(columns);
		assertEquals(1, columns.size());
		assertEquals("quoted on 2 lines\nand afterward some text", columns.get(0));
		assertEquals(2, tokenizer.getLineNumber());
		assertEquals(input, tokenizer.getUntokenizedRow());
		
		// should have exactly the same result when surrounding spaces need quotes
		tokenizer = createTokenizer(input, SPACES_NEED_QUOTES_PREFERENCE);
		tokenizer.readColumns(columns);
		assertEquals(1, columns.size());
		assertEquals("quoted on 2 lines\nand afterward some text", columns.get(0));
		assertEquals(2, tokenizer.getLineNumber());
		assertEquals(input, tokenizer.getUntokenizedRow());
	}
	
	/**
	 * Tests the readColumns() method with a single quoted newline.
	 */
	@Test
	public void testQuotedNewline() throws Exception {
		
		final String input = "\"\n\"";
		tokenizer = createTokenizer(input, NORMAL_PREFERENCE);
		tokenizer.readColumns(columns);
		assertTrue(columns.size() == 1);
		assertEquals("\n", columns.get(0));
		assertEquals(input, tokenizer.getUntokenizedRow());
		
		// same input when surrounding spaces require quotes (results should be identical)
		tokenizer = createTokenizer(input, SPACES_NEED_QUOTES_PREFERENCE);
		tokenizer.readColumns(columns);
		assertTrue(columns.size() == 1);
		assertEquals("\n", columns.get(0));
		assertEquals(input, tokenizer.getUntokenizedRow());
	}
	
	/**
	 * Tests the readColumns() method with a variety of quoted newlines.
	 */
	@Test
	public void testQuotedNewlines() throws Exception {
		
		final String input = "\"one line\",\"two\nlines\",\"three\nlines\n!\"";
		tokenizer = createTokenizer(input, NORMAL_PREFERENCE);
		tokenizer.readColumns(columns);
		assertTrue(columns.size() == 3);
		assertEquals("one line", columns.get(0));
		assertEquals("two\nlines", columns.get(1));
		assertEquals("three\nlines\n!", columns.get(2));
		assertEquals(input, tokenizer.getUntokenizedRow());
		
		// same input when surrounding spaces require quotes (results should be identical)
		tokenizer = createTokenizer(input, SPACES_NEED_QUOTES_PREFERENCE);
		tokenizer.readColumns(columns);
		assertTrue(columns.size() == 3);
		assertEquals("one line", columns.get(0));
		assertEquals("two\nlines", columns.get(1));
		assertEquals("three\nlines\n!", columns.get(2));
		assertEquals(input, tokenizer.getUntokenizedRow());
	}
	
	
	/**
	 * Tests the readColumns() method when a quoted field has consecutive newlines.
	 */
	@Test
	public void testQuotedTextWithConsecutiveNewLines() throws Exception {
		
		// second field has consecutive newlines
		final String input = "one, \"multiline\n\n\ntext\"";
		tokenizer = createTokenizer(input, NORMAL_PREFERENCE);
		tokenizer.readColumns(columns);
		assertEquals(2, columns.size());
		assertEquals("one", columns.get(0));
		assertEquals(" multiline\n\n\ntext", columns.get(1));
		assertEquals(4, tokenizer.getLineNumber());
		assertEquals(input, tokenizer.getUntokenizedRow());
		
		// should have exactly the same result when surrounding spaces need quotes
		tokenizer = createTokenizer(input, SPACES_NEED_QUOTES_PREFERENCE);
		tokenizer.readColumns(columns);
		assertTrue(columns.size() == 2);
		assertEquals("one", columns.get(0));
		assertEquals("multiline\n\n\ntext", columns.get(1));
		assertEquals(input, tokenizer.getUntokenizedRow());
	}
	
	/**
	 * Tests the readColumns() method when EOF is reached within quote scope.
	 */
	@Test
	public void testQuotedFieldWithUnexpectedEOF() throws Exception {
		
		// EOF reached within quote scope
		final String input = "\"quoted spanning\ntwo lines with EOF reached before another quote";
		tokenizer = createTokenizer(input, NORMAL_PREFERENCE);
		try {
			tokenizer.readColumns(columns);
			fail("should have thrown SuperCsvException");
		}
		catch(SuperCsvException e) {
			assertEquals("unexpected end of file while reading quoted column beginning on line 1 and ending on line 2",
				e.getMessage());
		}
	}

	/**
	 * Tests the readColumns() method when a newline is reached in quote
	 * scoped when a single line is only supposed to be read
	 */
	@Test
	public void testQuotedFieldWithUnexpectedNewline() throws Exception {

		// Row 2 has a missing trailing quote
		final String input = "col1,col2\n" +
				"\"foo\",\"bar\n" +
				"\"baz\",\"zoo\"\n" +
				"\"aaa\",\"bbb\"";
		CsvPreference pref = new CsvPreference.Builder(NORMAL_PREFERENCE)
				.maxLinesPerRow(1).build();

		tokenizer = createTokenizer(input, pref);
		try {
			boolean first = tokenizer.readColumns(columns);
			assertEquals(true , first);

			tokenizer.readColumns(columns);
			fail("should have thrown SuperCsvException");
		}
		catch(SuperCsvException e) {
			assertEquals("unexpected end of line while reading quoted column on line 2",
					e.getMessage());
		}
	}

	/**
	 * Tests the readColumns() method when a newline is reached in quote
	 * scoped when two lines are only supposed to be read
	 */
	@Test
	public void testQuotedFieldWithTwoMaxLines() throws Exception {

		// Row 2 has a missing trailing quote
		final String input = "col1,col2\n" +
				"\"foo\",\"bar\n" +
				"baz,zoo\n" +
				"aaa,bbb";
		CsvPreference pref = new CsvPreference.Builder(NORMAL_PREFERENCE)
				.maxLinesPerRow(2).build();

		tokenizer = createTokenizer(input, pref);
		try {
			boolean first = tokenizer.readColumns(columns);
			assertEquals(true , first);

			boolean second = tokenizer.readColumns(columns);
			assertEquals(true , second);

			tokenizer.readColumns(columns);
			fail("should have thrown SuperCsvException");
		}
		catch(SuperCsvException e) {
			assertEquals("max number of lines to read exceeded while reading quoted column beginning on line 2 and ending on line 3",
					e.getMessage());
		}
	}

	@Test
	public void testQuotedFieldWithUnexpectedNewlineNoNextLineRead() throws Exception {

		// Row 2 has a missing trailing quote
		final String input = "col1,col2\n" +
				"\"foo\",\"bar\n" +
				"\"baz\",\"zoo\"\n" +
				"\"aaa\",\"bbb\"";
		CsvPreference pref = new CsvPreference.Builder(NORMAL_PREFERENCE)
				.maxLinesPerRow(1).build();

		tokenizer = createTokenizer(input, pref);
		try {
			final boolean first = tokenizer.readColumns(columns);
			assertEquals(true , first);
			assertEquals("[col1, col2]" , columns.toString());

			tokenizer.readColumns(columns);
			fail("should have thrown SuperCsvException");
		}
		catch(SuperCsvException e) {
			assertEquals("unexpected end of line while reading quoted column on line 2",
					e.getMessage());
		}
		final boolean third = tokenizer.readColumns(columns);
		assertEquals(true , third);
		assertEquals("[baz, zoo]" , columns.toString());

		final boolean fourth = tokenizer.readColumns(columns);
		assertEquals(true , fourth);
		assertEquals("[aaa, bbb]" , columns.toString());

		//line 4 was the last 
		final boolean fifth = tokenizer.readColumns(columns);
		assertEquals(false , fifth);
	}

	/**
	 * Tests the readColumns() method when a newline is reached in quote
	 * scoped when two lines are only supposed to be read
	 */
	@Test
	public void testQuotedFieldWithTwoMaxLinesNoMoreLinesRead() throws Exception {

		// Row 2 has a missing trailing quote
		final String input = "col1,col2\n" +
				"\"foo,bar\n" +
				"baz,zoo\n" +
				"aaa,bbb";
		CsvPreference pref = new CsvPreference.Builder(NORMAL_PREFERENCE)
				.maxLinesPerRow(2).build();

		tokenizer = createTokenizer(input, pref);
		try {
			boolean first = tokenizer.readColumns(columns);
			assertEquals(true , first);
			assertEquals("[col1, col2]" , columns.toString());
			

			boolean second = tokenizer.readColumns(columns);
			assertEquals(true , second);
			assertEquals("[\"foo,bar]" , columns.toString());


			tokenizer.readColumns(columns);
			fail("should have thrown SuperCsvException");
		}
		catch(SuperCsvException e) {
			assertEquals("max number of lines to read exceeded while reading quoted column beginning on line 2 and ending on line 3",
					e.getMessage());
		}
		boolean fourth = tokenizer.readColumns(columns);
		assertEquals(true , fourth);
		assertEquals("[aaa, bbb]" , columns.toString());
		
	}

	/**
	 * Tests the readColumns() method with a leading space before the first quoted field. This is not technically valid
	 * CSV, but the tokenizer is lenient enough to allow it. The leading spaces will be trimmed off when surrounding
	 * spaces require quotes, otherwise they will be part of the field.
	 */
	@Test
	public void testQuotedFirstFieldWithLeadingSpace() throws Exception {
		
		// leading spaces should be preserved
		final String input = "  \"quoted with leading spaces\",two";
		tokenizer = createTokenizer(input, NORMAL_PREFERENCE);
		tokenizer.readColumns(columns);
		assertTrue(columns.size() == 2);
		assertEquals("  quoted with leading spaces", columns.get(0));
		assertEquals("two", columns.get(1));
		assertEquals(input, tokenizer.getUntokenizedRow());
		
		// same input when surrounding spaces require quotes (leading spaces trimmed)
		tokenizer = createTokenizer(input, SPACES_NEED_QUOTES_PREFERENCE);
		tokenizer.readColumns(columns);
		assertTrue(columns.size() == 2);
		assertEquals("quoted with leading spaces", columns.get(0));
		assertEquals("two", columns.get(1));
		assertEquals(input, tokenizer.getUntokenizedRow());
	}
	
	/**
	 * Tests the readColumns() method with a leading space before the last quoted field. This is not technically valid
	 * CSV, but the tokenizer is lenient enough to allow it. The leading spaces will be trimmed off when surrounding
	 * spaces require quotes, otherwise they will be part of the field.
	 */
	@Test
	public void testQuotedLastFieldWithLeadingSpace() throws Exception {
		
		// last field has a leading space before quote (should be preserved)
		final String input = "one,two,  \"quoted with leading spaces\"";
		tokenizer = createTokenizer(input, NORMAL_PREFERENCE);
		tokenizer.readColumns(columns);
		assertTrue(columns.size() == 3);
		assertEquals("one", columns.get(0));
		assertEquals("two", columns.get(1));
		assertEquals("  quoted with leading spaces", columns.get(2));
		assertEquals(input, tokenizer.getUntokenizedRow());
		
		// leading space should be trimmed off
		tokenizer = createTokenizer(input, SPACES_NEED_QUOTES_PREFERENCE);
		tokenizer.readColumns(columns);
		assertTrue(columns.size() == 3);
		assertEquals("one", columns.get(0));
		assertEquals("two", columns.get(1));
		assertEquals("quoted with leading spaces", columns.get(2));
		assertEquals(input, tokenizer.getUntokenizedRow());
	}
	
	/**
	 * Tests the readColumns() method with a trailing space after the first quoted field. This is not technically valid
	 * CSV, but the tokenizer is lenient enough to allow it. The trailing spaces will be trimmed off when surrounding
	 * spaces require quotes, otherwise they will be part of the field.
	 */
	@Test
	public void testQuotedFirstFieldWithTrailingSpace() throws Exception {
		
		// first field has a leading space before quote (should be preserved)
		final String input = "\"quoted with trailing spaces\"  ,two";
		tokenizer = createTokenizer(input, NORMAL_PREFERENCE);
		tokenizer.readColumns(columns);
		assertTrue(columns.size() == 2);
		assertEquals("quoted with trailing spaces  ", columns.get(0));
		assertEquals("two", columns.get(1));
		assertEquals(input, tokenizer.getUntokenizedRow());
		
		// trailing spaces should be trimmed
		tokenizer = createTokenizer(input, SPACES_NEED_QUOTES_PREFERENCE);
		tokenizer.readColumns(columns);
		assertTrue(columns.size() == 2);
		assertEquals("quoted with trailing spaces", columns.get(0));
		assertEquals("two", columns.get(1));
		assertEquals(input, tokenizer.getUntokenizedRow());
	}
	
	/**
	 * Tests the readColumns() method with a trailing space after the last quoted field. This is not technically valid
	 * CSV, but the tokenizer is lenient enough to allow it. The trailing spaces will be trimmed off when surrounding
	 * spaces require quotes, otherwise they will be part of the field.
	 */
	@Test
	public void testQuotedLastFieldWithTrailingSpace() throws Exception {
		
		// last field has a leading space before quote (should be preserved)
		final String input = "one,two,\"quoted with trailing spaces\"  ";
		tokenizer = createTokenizer(input, NORMAL_PREFERENCE);
		tokenizer.readColumns(columns);
		assertTrue(columns.size() == 3);
		assertEquals("one", columns.get(0));
		assertEquals("two", columns.get(1));
		assertEquals("quoted with trailing spaces  ", columns.get(2));
		assertEquals(input, tokenizer.getUntokenizedRow());
		
		// trailing spaces should be trimmed off
		tokenizer = createTokenizer(input, SPACES_NEED_QUOTES_PREFERENCE);
		tokenizer.readColumns(columns);
		assertTrue(columns.size() == 3);
		assertEquals("one", columns.get(0));
		assertEquals("two", columns.get(1));
		assertEquals("quoted with trailing spaces", columns.get(2));
		assertEquals(input, tokenizer.getUntokenizedRow());
	}
	
	/**
	 * Tests the readColumns() method with a variety of quoted spaces.
	 */
	@Test
	public void testQuotedSpaces() throws Exception {
		
		final String input = "\" one \",\"  two  \",\"   three   \"";
		tokenizer = createTokenizer(input, NORMAL_PREFERENCE);
		tokenizer.readColumns(columns);
		assertTrue(columns.size() == 3);
		assertEquals(" one ", columns.get(0));
		assertEquals("  two  ", columns.get(1));
		assertEquals("   three   ", columns.get(2));
		assertEquals(input, tokenizer.getUntokenizedRow());
		
		// same input when surrounding spaces require quotes (results should be identical)
		tokenizer = createTokenizer(input, SPACES_NEED_QUOTES_PREFERENCE);
		tokenizer.readColumns(columns);
		assertTrue(columns.size() == 3);
		assertEquals(" one ", columns.get(0));
		assertEquals("  two  ", columns.get(1));
		assertEquals("   three   ", columns.get(2));
		assertEquals(input, tokenizer.getUntokenizedRow());
	}
	
	/**
	 * Tests the readColumns() method with a variety of unquoted spaces.
	 */
	@Test
	public void testSpaces() throws Exception {
		
		final String input = " one ,  two  ,   three   ";
		tokenizer = createTokenizer(input, NORMAL_PREFERENCE);
		tokenizer.readColumns(columns);
		assertTrue(columns.size() == 3);
		assertEquals(" one ", columns.get(0));
		assertEquals("  two  ", columns.get(1));
		assertEquals("   three   ", columns.get(2));
		assertEquals(input, tokenizer.getUntokenizedRow());
		
		// same input when surrounding spaces require quotes
		tokenizer = createTokenizer(input, SPACES_NEED_QUOTES_PREFERENCE);
		tokenizer.readColumns(columns);
		assertTrue(columns.size() == 3);
		assertEquals("one", columns.get(0));
		assertEquals("two", columns.get(1));
		assertEquals("three", columns.get(2));
		assertEquals(input, tokenizer.getUntokenizedRow());
	}
	
	/**
	 * Tests the readColumns() method with a variety of spaces and tabs.
	 */
	@Test
	public void testSpacesAndTabs() throws Exception {
		
		// tabs should never be trimmed
		final String input = "\t, \tone\t ,  \ttwo\t  ,   \tthree\t   ";
		tokenizer = createTokenizer(input, NORMAL_PREFERENCE);
		tokenizer.readColumns(columns);
		assertTrue(columns.size() == 4);
		assertEquals("\t", columns.get(0));
		assertEquals(" \tone\t ", columns.get(1));
		assertEquals("  \ttwo\t  ", columns.get(2));
		assertEquals("   \tthree\t   ", columns.get(3));
		assertEquals(input, tokenizer.getUntokenizedRow());
		
		// same input when surrounding spaces require quotes
		tokenizer = createTokenizer(input, SPACES_NEED_QUOTES_PREFERENCE);
		tokenizer.readColumns(columns);
		assertTrue(columns.size() == 4);
		assertEquals("\t", columns.get(0));
		assertEquals("\tone\t", columns.get(1));
		assertEquals("\ttwo\t", columns.get(2));
		assertEquals("\tthree\t", columns.get(3));
		assertEquals(input, tokenizer.getUntokenizedRow());
	}
	
	/**
	 * Tests the readColumns() method with spaces between words.
	 */
	@Test
	public void testSpacesBetweenWords() throws Exception {
		
		final String input = " one partridge ,  two turtle doves  ,   three french hens   ";
		tokenizer = createTokenizer(input, NORMAL_PREFERENCE);
		tokenizer.readColumns(columns);
		assertTrue(columns.size() == 3);
		assertEquals(" one partridge ", columns.get(0));
		assertEquals("  two turtle doves  ", columns.get(1));
		assertEquals("   three french hens   ", columns.get(2));
		assertEquals(input, tokenizer.getUntokenizedRow());
		
		// same input when surrounding spaces require quotes
		tokenizer = createTokenizer(input, SPACES_NEED_QUOTES_PREFERENCE);
		tokenizer.readColumns(columns);
		assertTrue(columns.size() == 3);
		assertEquals("one partridge", columns.get(0));
		assertEquals("two turtle doves", columns.get(1));
		assertEquals("three french hens", columns.get(2));
		assertEquals(input, tokenizer.getUntokenizedRow());
	}
	
	/**
	 * Tests that the CommentStartsWith comment matcher works (comments are skipped).
	 */
	@Test
	public void testSkipCommentsStartsWith() throws IOException {
		
		final CsvPreference commentsStartWithPrefs = new CsvPreference.Builder(EXCEL_PREFERENCE).skipComments(
			new CommentStartsWith("#")).build();
		
		final String input = "#comment\nnot,a,comment\n# another comment\nalso,not,comment";
		final Tokenizer tokenizer = createTokenizer(input, commentsStartWithPrefs);
		tokenizer.readColumns(columns);
		assertTrue(columns.size() == 3);
		assertEquals("not", columns.get(0));
		assertEquals("a", columns.get(1));
		assertEquals("comment", columns.get(2));
		
		tokenizer.readColumns(columns);
		assertTrue(columns.size() == 3);
		assertEquals("also", columns.get(0));
		assertEquals("not", columns.get(1));
		assertEquals("comment", columns.get(2));
		
		assertFalse(tokenizer.readColumns(columns));
	}
	
	/**
	 * Tests that the CommentMatches comment matcher works (comments are skipped).
	 */
	@Test
	public void testSkipCommentsMatches() throws IOException {
		
		final CsvPreference commentsMatchesPrefs = new CsvPreference.Builder(EXCEL_PREFERENCE).skipComments(
			new CommentMatches("<!--.*-->")).build();
		
		final String input = "<!--comment-->\nnot,a,comment\n<!-- another comment-->\nalso,not,comment";
		final Tokenizer tokenizer = createTokenizer(input, commentsMatchesPrefs);
		tokenizer.readColumns(columns);
		assertTrue(columns.size() == 3);
		assertEquals("not", columns.get(0));
		assertEquals("a", columns.get(1));
		assertEquals("comment", columns.get(2));
		
		tokenizer.readColumns(columns);
		assertTrue(columns.size() == 3);
		assertEquals("also", columns.get(0));
		assertEquals("not", columns.get(1));
		assertEquals("comment", columns.get(2));
		
		assertFalse(tokenizer.readColumns(columns));
		
	}
	
}
