from __future__ import print_function
import six
from twisted.trial import unittest
from twisted.python.failure import Failure
from twisted.python.components import registerAdapter
from twisted.internet import defer

from foolscap.tokens import ISlicer, Violation, BananaError
from foolscap.tokens import BananaFailure, tokenNames, \
     OPEN, CLOSE, ABORT, INT, LONGINT, NEG, LONGNEG, FLOAT, STRING
from foolscap import slicer, schema, storage, banana, vocab
from foolscap.eventual import fireEventually, flushEventualQueue
from foolscap.slicers.allslicers import RootSlicer, DictUnslicer, TupleUnslicer
from foolscap.constraint import IConstraint
from foolscap.banana import int2b128, long_to_bytes
from ..util import long_type

import io
import struct
from decimal import Decimal

#log.startLogging(sys.stderr)

# some utility functions to manually assemble bytestreams

def bOPEN(opentype, count):
    opentype = six.ensure_binary(opentype)
    assert count < 128
    return six.int2byte(count) + b"\x88" + six.int2byte(len(opentype)) + b"\x82" + opentype
def bCLOSE(count):
    assert count < 128
    return six.int2byte(count) + b"\x89"
def bINT(num):
    if num >=0:
        assert num < 128
        return six.int2byte(num) + b"\x81"
    num = -num
    assert num < 128
    return six.int2byte(num) + b"\x83"
def bSTR(b):
    b = six.ensure_binary(b)
    assert len(b) < 128
    return six.int2byte(len(b)) + b"\x82" + b
def bERROR(b):
    assert isinstance(b, bytes)
    assert len(b) < 128
    return six.int2byte(len(b)) + b"\x8d" + b
def bABORT(count):
    assert count < 128
    return six.int2byte(count) + b"\x8A"
# DecodeTest (24): turns tokens into objects, tests objects and UFs
# EncodeTest (13): turns objects/instance into tokens, tests tokens
# FailedInstanceTests (2): 1:turn instances into tokens and fail, 2:reverse

# ByteStream (3): turn object into bytestream, test bytestream
# InboundByteStream (14): turn bytestream into object, check object
#                         with or without constraints
# ThereAndBackAgain (20): encode then decode object, check object

# VocabTest1 (2): test setOutgoingVocabulary and an inbound Vocab sequence
# VocabTest2 (1): send object, test bytestream w/vocab-encoding
# Sliceable (2): turn instance into tokens (with ISliceable, test tokens

def tOPEN(count):
    return ("OPEN", count)
def tCLOSE(count):
    return ("CLOSE", count)
tABORT = ("ABORT",)

class TokenBanana(banana.Banana):
    """this Banana formats tokens as strings, numbers, and ('OPEN',) tuples
    instead of bytes. Used for testing purposes."""

    def sendOpen(self):
        openID = self.openCount
        self.openCount += 1
        self.sendToken(("OPEN", openID))
        return openID

    def sendToken(self, token):
        #print token
        self.tokens.append(token)

    def sendClose(self, openID):
        self.sendToken(("CLOSE", openID))

    def sendAbort(self, count=0):
        self.sendToken(("ABORT",))

    def sendError(self, msg):
        #print "TokenBanana.sendError(%s)" % msg
        pass

    def testSlice(self, obj):
        assert len(self.slicerStack) == 1
        assert isinstance(self.slicerStack[0][0], RootSlicer)
        self.tokens = []
        d = self.send(obj)
        d.addCallback(self._testSlice_1)
        return d
    def _testSlice_1(self, res):
        assert len(self.slicerStack) == 1
        assert not self.rootSlicer.sendQueue
        assert isinstance(self.slicerStack[0][0], RootSlicer)
        return self.tokens

    def __del__(self):
        assert not self.rootSlicer.sendQueue

def untokenize(tokens):
    data = []
    for t in tokens:
        if isinstance(t, tuple):
            if t[0] == "OPEN":
                int2b128(t[1], data.append)
                data.append(OPEN)
            elif t[0] == "CLOSE":
                int2b128(t[1], data.append)
                data.append(CLOSE)
            elif t[0] == "ABORT":
                data.append(ABORT)
            else:
                raise RuntimeError("bad token")
        else:
            if isinstance(t, six.integer_types):
                if t >= 2**31:
                    s = long_to_bytes(t)
                    int2b128(len(s), data.append)
                    data.append(LONGINT)
                    data.append(s)
                elif t >= 0:
                    int2b128(t, data.append)
                    data.append(INT)
                elif -t > 2**31: # NEG is [-2**31, 0)
                    s = long_to_bytes(-t)
                    int2b128(len(s), data.append)
                    data.append(LONGNEG)
                    data.append(s)
                else:
                    int2b128(-t, data.append)
                    data.append(NEG)
            elif isinstance(t, float):
                data.append(FLOAT)
                data.append(struct.pack("!d", t))
            elif isinstance(t, six.string_types) or isinstance(t, six.binary_type):
                t = six.ensure_binary(t)
                int2b128(len(t), data.append)
                data.append(STRING)
                data.append(t)
            else:
                raise BananaError("could not send object: %s" % repr(t))
    return b"".join(data)

class UnbananaTestMixin:
    def setUp(self):
        self.hangup = False
        self.banana = storage.StorageBanana()
        self.banana.slicerClass = storage.StorageRootSlicer
        self.banana.unslicerClass = storage.StorageRootUnslicer
        self.banana.connectionMade()
    def tearDown(self):
        if not self.hangup:
            self.assertTrue(len(self.banana.receiveStack) == 1)
            self.assertTrue(isinstance(self.banana.receiveStack[0],
                                       storage.StorageRootUnslicer))

    def do(self, tokens):
        self.banana.violation = None
        self.banana.disconnectReason = None
        self.assertTrue(len(self.banana.receiveStack) == 1)
        self.assertTrue(isinstance(self.banana.receiveStack[0],
                                   storage.StorageRootUnslicer))
        data = untokenize(tokens)
        results = []
        d = self.banana.prepare()
        d.addCallback(results.append)
        self.banana.dataReceived(data)
        # we expect everything here to be synchronous
        if len(results) == 1:
            return results[0]
        self.assertTrue(self.banana.violation or self.banana.disconnectReason)
        return None

    def shouldFail(self, tokens):
        obj = self.do(tokens)
        self.assertTrue(obj is None, "object was produced: %s" % obj)
        self.assertTrue(self.banana.violation, "didn't fail, ret=%s" % obj)
        self.assertFalse(self.banana.disconnectReason,
                    "connection was dropped: %s" % \
                    self.banana.disconnectReason)
        return self.banana.violation

    def shouldDropConnection(self, tokens):
        self.banana.logReceiveErrors = False
        try:
            obj = self.do(tokens)
            self.fail("connection was supposed to be dropped, got obj=%s"
                      % (obj,))
        except BananaError:
            f = self.banana.disconnectReason
            if not isinstance(f, Failure):
                self.fail("disconnectReason wasn't a Failure: %s" % f)
            if not f.check(BananaError):
                self.fail("wrong exception type: %s" % f)
            self.hangup = True # to stop the tearDown check
            self.assertFalse(self.banana.violation)
            return f


    def failIfBananaFailure(self, res):
        if isinstance(res, BananaFailure):
            # something went wrong
            print("There was a failure while Unbananaing '%s':" % res.where)
            print(res.getTraceback())
            self.fail("BananaFailure")

    def checkBananaFailure(self, res, where, failtype=None):
        print(res)
        self.assertTrue(isinstance(res, BananaFailure))
        if failtype:
            self.assertTrue(res.failure,
                            "No Failure object in BananaFailure")
            if not res.check(failtype):
                print("Wrong exception (wanted '%s'):" % failtype)
                print(res.getTraceback())
                self.fail("Wrong exception (wanted '%s'):" % failtype)
        self.assertEqual(res.where, where)
        self.banana.object = None # to stop the tearDown check TODO ??

class TestTransport(io.BytesIO):
    disconnectReason = None
    def loseConnection(self):
        pass

class _None: pass

class TestBananaMixin:
    def setUp(self):
        self.makeBanana()

    def makeBanana(self):
        self.banana = storage.StorageBanana()
        self.banana.slicerClass = storage.StorageRootSlicer
        self.banana.unslicerClass = storage.StorageRootUnslicer
        self.banana.transport = TestTransport()
        self.banana.connectionMade()

    def encode(self, obj):
        d = self.banana.send(obj)
        d.addCallback(lambda res: self.banana.transport.getvalue())
        return d

    def clearOutput(self):
        self.banana.transport = TestTransport()

    def decode(self, stream):
        self.banana.violation = None
        results = []
        d = self.banana.prepare()
        d.addCallback(results.append)
        self.banana.dataReceived(stream)
        # we expect everything here to be synchronous
        if len(results) == 1:
            return results[0]
        self.assertTrue(self.banana.violation or self.banana.disconnectReason)
        return None

    def shouldDecode(self, stream):
        obj = self.decode(stream)
        self.assertFalse(self.banana.violation)
        self.assertFalse(self.banana.disconnectReason)
        self.assertEqual(len(self.banana.receiveStack), 1)
        return obj

    def shouldFail(self, stream):
        obj = self.decode(stream)
        # Violations on a StorageBanana will continue to decode objects, but
        # will set b.violation, which we can examine afterwards
        self.assertEqual(obj, None)
        self.assertFalse(self.banana.disconnectReason,
                    "connection was dropped: %s" % \
                    self.banana.disconnectReason)
        self.assertEqual(len(self.banana.receiveStack), 1)
        f = self.banana.violation
        if not f:
            self.fail("didn't fail")
        if not isinstance(f, BananaFailure):
            self.fail("violation wasn't a BananaFailure: %s" % f)
        if not f.check(Violation):
            self.fail("wrong exception type: %s" % f)
        return f

    def shouldDropConnection(self, stream):
        self.banana.logReceiveErrors = False # trial hooks log.err
        try:
            obj = self.decode(stream)
            self.fail("decode worked! got '%s', expected dropConnection" \
                      % (obj,))
        except BananaError:
            # the receiveStack is allowed to be non-empty here, since we've
            # dropped the connection anyway
            f = self.banana.disconnectReason
            if not f:
                self.fail("didn't fail")
            if not isinstance(f, Failure):
                self.fail("disconnectReason wasn't a Failure: %s" % f)
            if not f.check(BananaError):
                self.fail("wrong exception type: %s" % f)
            self.makeBanana() # need a new one, we squished the last one
            return f

    def wantEqual(self, got, wanted):
        if got != wanted:
            print()
            print("wanted: '%s'" % wanted, repr(wanted))
            print("got   : '%s'" % got, repr(got))
            self.fail("did not get expected string")

    def loop(self, obj):
        self.clearOutput()
        d = self.encode(obj)
        d.addCallback(self.shouldDecode)
        return d

    def looptest(self, obj, newvalue=_None):
        if newvalue is _None:
            newvalue = obj
        d = self.loop(obj)
        d.addCallback(self._looptest_1, newvalue)
        return d
    def _looptest_1(self, obj2, newvalue):
        self.assertEqual(obj2, newvalue)
        self.assertEqual(type(obj2), type(newvalue))

def join(*args):
    return b"".join(args)



class BrokenDictUnslicer(DictUnslicer):
    dieInFinish = 0

    def receiveKey(self, key):
        if key == b"die":
            raise Violation("aaagh")
        if key == b"please_die_in_finish":
            self.dieInFinish = 1
        DictUnslicer.receiveKey(self, key)

    def receiveValue(self, value):
        if value == b"die":
            raise Violation("aaaaaaaaargh")
        DictUnslicer.receiveValue(self, value)

    def receiveClose(self):
        if self.dieInFinish:
            raise Violation("dead in receiveClose()")
        DictUnslicer.receiveClose(self)
        return None, None

class ReallyBrokenDictUnslicer(DictUnslicer):
    def start(self, count):
        raise Violation("dead in start")


class DecodeTest(UnbananaTestMixin, unittest.TestCase):
    def setUp(self):
        UnbananaTestMixin.setUp(self)
        self.banana.logReceiveErrors = False
        d ={ ('dict1',): BrokenDictUnslicer,
             ('dict2',): ReallyBrokenDictUnslicer,
             }
        self.banana.rootUnslicer.topRegistries.insert(0, d)
        self.banana.rootUnslicer.openRegistries.insert(0, d)

    def test_simple_list(self):
        "simple list"
        res = self.do([tOPEN(0),'list',1,2,3,"a","b",tCLOSE(0)])
        self.assertEqual(res, [1,2,3,b'a',b'b'])

    def test_aborted_list(self):
        "aborted list"
        f = self.shouldFail([tOPEN(0),'list', 1, tABORT, tCLOSE(0)])
        self.assertTrue(isinstance(f, BananaFailure))
        self.assertTrue(f.check(Violation))
        self.assertEqual(f.value.where, "<RootUnslicer>.[1]")
        self.assertEqual(f.value.args[0], "ABORT received")

    def test_aborted_list2(self):
        "aborted list2"
        f = self.shouldFail([tOPEN(0),'list', 1, tABORT,
                             tOPEN(1),'list', 2, 3, tCLOSE(1),
                             tCLOSE(0)])
        self.assertTrue(isinstance(f, BananaFailure))
        self.assertTrue(f.check(Violation))
        self.assertEqual(f.value.where, "<RootUnslicer>.[1]")
        self.assertEqual(f.value.args[0], "ABORT received")

    def test_aborted_list3(self):
        "aborted list3"
        f = self.shouldFail([tOPEN(0),'list', 1,
                              tOPEN(1),'list', 2, 3, 4,
                               tOPEN(2),'list', 5, 6, tABORT, tCLOSE(2),
                              tCLOSE(1),
                             tCLOSE(0)])
        self.assertTrue(isinstance(f, BananaFailure))
        self.assertTrue(f.check(Violation))
        self.assertEqual(f.value.where, "<RootUnslicer>.[1].[3].[2]")
        self.assertEqual(f.value.args[0], "ABORT received")

    def test_nested_list(self):
        "nested list"
        res = self.do([tOPEN(0),'list',1,2,
                        tOPEN(1),'list',3,4,tCLOSE(1),
                       tCLOSE(0)])
        self.assertEqual(res, [1,2,[3,4]])

    def test_list_with_tuple(self):
        "list with tuple"
        res = self.do([tOPEN(0),'list',1,2,
                        tOPEN(1),'tuple',3,4,tCLOSE(1),
                       tCLOSE(0)])
        self.assertEqual(res, [1,2,(3,4)])

    def test_dict(self):
        "dict"
        res = self.do([tOPEN(0),'dict',"a",1,"b",2,tCLOSE(0)])
        self.assertEqual(res, {b'a':1, b'b':2})

    def test_dict_with_duplicate_keys(self):
        "dict with duplicate keys"
        f = self.shouldDropConnection([tOPEN(0),'dict',
                                       1,"a",1,"b",
                                       tCLOSE(0)])
        self.assertEqual(f.value.where, "<RootUnslicer>.{}")
        self.assertEqual(f.value.args[0], "duplicate key '1'")

    def test_dict_with_list(self):
        "dict with list"
        res = self.do([tOPEN(0),'dict',
                        "a",1,
                        "b", tOPEN(1),'list', 2, 3, tCLOSE(1),
                       tCLOSE(0)])
        self.assertEqual(res, {b'a':1, b'b':[2,3]})

    def test_dict_with_tuple_as_key(self):
        "dict with tuple as key"
        res = self.do([tOPEN(0),'dict',
                        tOPEN(1),'tuple', 1, 2, tCLOSE(1), "a",
                       tCLOSE(0)])
        self.assertEqual(res, {(1,2):b'a'})

    def test_dict_with_mutable_key(self):
        "dict with mutable key"
        f = self.shouldDropConnection([tOPEN(0),'dict',
                                        tOPEN(1),'list', 1, 2, tCLOSE(1), "a",
                                       tCLOSE(0)])
        self.assertEqual(f.value.where, "<RootUnslicer>.{}")
        self.assertEqual(f.value.args[0], "unhashable key '[1, 2]'")

    def test_ref1(self):
        res = self.do([tOPEN(0),'list',
                        tOPEN(1),'list', 1, 2, tCLOSE(1),
                        tOPEN(2),'reference', 1, tCLOSE(2),
                       tCLOSE(0)])
        self.failIfBananaFailure(res)
        self.assertEqual(res, [[1,2], [1,2]])
        self.failUnlessIdentical(res[0], res[1])

    def test_ref2(self):
        res = self.do([tOPEN(0),'list',
                       tOPEN(1),'list', 1, 2, tCLOSE(1),
                       tOPEN(2),'reference', 0, tCLOSE(2),
                       tCLOSE(0)])
        self.failIfBananaFailure(res)
        wanted = [[1,2]]
        wanted.append(wanted)
        # python2.3 is clever and can do
        #  self.failUnlessEqual(res, wanted)
        # python2.4 is not, so we do it by hand
        self.assertEqual(len(res), len(wanted))
        self.assertEqual(res[0], wanted[0])
        self.failUnlessIdentical(res, res[1])

    def test_ref3(self):
        res = self.do([tOPEN(0),'list',
                        tOPEN(1),'tuple', 1, 2, tCLOSE(1),
                        tOPEN(2),'reference', 1, tCLOSE(2),
                       tCLOSE(0)])
        self.failIfBananaFailure(res)
        wanted = [(1,2)]
        wanted.append(wanted[0])
        self.assertEqual(res, wanted)
        self.failUnlessIdentical(res[0], res[1])

    def test_ref4(self):
        res = self.do([tOPEN(0),'list',
                        tOPEN(1),'dict', "a", 1, tCLOSE(1),
                        tOPEN(2),'reference', 1, tCLOSE(2),
                       tCLOSE(0)])
        self.failIfBananaFailure(res)
        wanted = [{b"a":1}]
        wanted.append(wanted[0])
        self.assertEqual(res, wanted)
        self.failUnlessIdentical(res[0], res[1])

    def test_ref5(self):
        # The Droste Effect: a list that contains itself
        res = self.do([tOPEN(0),'list',
                        5,
                        6,
                        tOPEN(1),'reference', 0, tCLOSE(1),
                        7,
                       tCLOSE(0)])
        self.failIfBananaFailure(res)
        wanted = [5,6]
        wanted.append(wanted)
        wanted.append(7)
        #self.failUnlessEqual(res, wanted)
        self.assertEqual(len(res), len(wanted))
        self.assertEqual(res[0:2], wanted[0:2])
        self.failUnlessIdentical(res[2], res)
        self.assertEqual(res[3], wanted[3])

    def test_ref6(self):
        # everybody's favorite "([(ref0" test case. A tuple of a list of a
        # tuple of the original tuple. Such cycles must always have a
        # mutable container in them somewhere, or they couldn't be
        # constructed, but the resulting object involves a lot of deferred
        # results because the mutable list is the *only* object that can
        # be created without dependencies
        res = self.do([tOPEN(0),'tuple',
                        tOPEN(1),'list',
                         tOPEN(2),'tuple',
                          tOPEN(3),'reference', 0, tCLOSE(3),
                         tCLOSE(2),
                        tCLOSE(1),
                       tCLOSE(0)])
        self.failIfBananaFailure(res)
        wanted = ([],)
        wanted[0].append((wanted,))
        #self.failUnlessEqual(res, wanted)
        self.assertTrue(type(res) is tuple)
        self.assertTrue(len(res) == 1)
        self.assertTrue(type(res[0]) is list)
        self.assertTrue(len(res[0]) == 1)
        self.assertTrue(type(res[0][0]) is tuple)
        self.assertTrue(len(res[0][0]) == 1)
        self.failUnlessIdentical(res[0][0][0], res)

        # TODO: need a test where tuple[0] and [1] are deferred, but
        # tuple[0] becomes available before tuple[2] is inserted. Not sure
        # this is possible, but it would improve test coverage in
        # TupleUnslicer

    def test_failed_dict1(self):
        # dies during open because of bad opentype
        f = self.shouldFail([tOPEN(0),'list', 1,
                              tOPEN(1),"bad",
                               "a", 2,
                               "b", 3,
                              tCLOSE(1),
                             tCLOSE(0)])
        self.assertTrue(isinstance(f, BananaFailure))
        self.assertTrue(f.check(Violation))
        self.assertEqual(f.value.where, "<RootUnslicer>.[1]")
        self.assertEqual(f.value.args[0], "unknown OPEN type ('bad',)")

    def test_failed_dict2(self):
        # dies during start
        f = self.shouldFail([tOPEN(0),'list', 1,
                             tOPEN(1),'dict2', "a", 2, "b", 3, tCLOSE(1),
                             tCLOSE(0)])
        self.assertTrue(isinstance(f, BananaFailure))
        self.assertTrue(f.check(Violation))
        self.assertEqual(f.value.where, "<RootUnslicer>.[1].{}")
        self.assertEqual(f.value.args[0], "dead in start")

    def test_failed_dict3(self):
        # dies during key
        f = self.shouldFail([tOPEN(0),'list', 1,
                             tOPEN(1),'dict1', "a", 2, "die", tCLOSE(1),
                             tCLOSE(0)])
        self.assertTrue(isinstance(f, BananaFailure))
        self.assertTrue(f.check(Violation))
        self.assertEqual(f.value.where, "<RootUnslicer>.[1].{}")
        self.assertEqual(f.value.args[0], "aaagh")

        res = self.do([tOPEN(2),'list', 3, 4, tCLOSE(2)])
        self.assertEqual(res, [3,4])

    def test_failed_dict4(self):
        # dies during value
        f = self.shouldFail([tOPEN(0),'list', 1,
                              tOPEN(1),'dict1',
                               "a", 2,
                               3, "die",
                              tCLOSE(1),
                             tCLOSE(0)])
        self.assertTrue(isinstance(f, BananaFailure))
        self.assertTrue(f.check(Violation))
        self.assertEqual(f.value.where, "<RootUnslicer>.[1].{}[3]")
        self.assertEqual(f.value.args[0], "aaaaaaaaargh")

    def test_failed_dict5(self):
        # dies during finish
        f = self.shouldFail([tOPEN(0),'list', 1,
                              tOPEN(1),'dict1',
                               "a", 2,
                               "please_die_in_finish", 3,
                              tCLOSE(1),
                             tCLOSE(0)])
        self.assertTrue(isinstance(f, BananaFailure))
        self.assertTrue(f.check(Violation))
        self.assertEqual(f.value.where, "<RootUnslicer>.[1].{}")
        self.assertEqual(f.value.args[0], "dead in receiveClose()")

class EncodeTest(unittest.TestCase):
    def setUp(self):
        self.banana = TokenBanana()
        self.banana.slicerClass = storage.StorageRootSlicer
        self.banana.unslicerClass = storage.StorageRootUnslicer
        self.banana.connectionMade()
    def do(self, obj):
        return self.banana.testSlice(obj)
    def tearDown(self):
        self.assertTrue(len(self.banana.slicerStack) == 1)
        self.assertTrue(isinstance(self.banana.slicerStack[0][0], RootSlicer))

    def testList(self):
        d = self.do([1,2])
        d.addCallback(self.assertEqual,
                      [tOPEN(0),b'list', 1, 2, tCLOSE(0)])
        return d

    def testTuple(self):
        d = self.do((1,2))
        d.addCallback(self.assertEqual,
                      [tOPEN(0),b'tuple', 1, 2, tCLOSE(0)])
        return d

    def testNestedList(self):
        d = self.do([1,2,[3,4]])
        d.addCallback(self.assertEqual,
                      [tOPEN(0),b'list', 1, 2,
                        tOPEN(1),b'list', 3, 4, tCLOSE(1),
                       tCLOSE(0)])
        return d

    def testNestedList2(self):
        d = self.do([1,2,(3,4,[5, b"hi"])])
        d.addCallback(self.assertEqual,
                      [tOPEN(0),b'list', 1, 2,
                        tOPEN(1),b'tuple', 3, 4,
                         tOPEN(2),b'list', 5, b"hi", tCLOSE(2),
                        tCLOSE(1),
                       tCLOSE(0)])
        return d

    def testDict(self):
        d = self.do({b'a': 1, b'b': 2})
        d.addCallback(lambda res:
                      self.assertTrue(
            res == [tOPEN(0),b'dict', b'a', 1, b'b', 2, tCLOSE(0)] or
            res == [tOPEN(0),b'dict', b'b', 2, b'a', 1, tCLOSE(0)]))
        return d

    def test_ref1(self):
        l = [1,2]
        obj = [l,l]
        d = self.do(obj)
        d.addCallback(self.assertEqual,
                      [tOPEN(0),b'list',
                        tOPEN(1),b'list', 1, 2, tCLOSE(1),
                        tOPEN(2),b'reference', 1, tCLOSE(2),
                       tCLOSE(0)])
        return d

    def test_ref2(self):
        obj = [[1,2]]
        obj.append(obj)
        d = self.do(obj)
        d.addCallback(self.assertEqual,
                      [tOPEN(0),b'list',
                        tOPEN(1),b'list', 1, 2, tCLOSE(1),
                        tOPEN(2),b'reference', 0, tCLOSE(2),
                       tCLOSE(0)])
        return d

    def test_ref3(self):
        obj = [(1,2)]
        obj.append(obj[0])
        d = self.do(obj)
        d.addCallback(self.assertEqual,
                      [tOPEN(0),b'list',
                        tOPEN(1),b'tuple', 1, 2, tCLOSE(1),
                        tOPEN(2),b'reference', 1, tCLOSE(2),
                       tCLOSE(0)])
        return d

    def test_ref4(self):
        obj = [{b"a":1}]
        obj.append(obj[0])
        d = self.do(obj)
        d.addCallback(self.assertEqual,
                      [tOPEN(0),b'list',
                        tOPEN(1),b'dict', b"a", 1, tCLOSE(1),
                        tOPEN(2),b'reference', 1, tCLOSE(2),
                       tCLOSE(0)])
        return d

    def test_ref6(self):
        # everybody's favorite "([(ref0" test case.
        obj = ([],)
        obj[0].append((obj,))
        d = self.do(obj)
        d.addCallback(self.assertEqual,
                      [tOPEN(0),b'tuple',
                        tOPEN(1),b'list',
                         tOPEN(2),b'tuple',
                          tOPEN(3),b'reference', 0, tCLOSE(3),
                         tCLOSE(2),
                        tCLOSE(1),
                       tCLOSE(0)])
        return d

    def test_refdict1(self):
        # a dictionary with a value that isn't available right away
        d0 = {1: b"a"}
        t = (d0,)
        d0[2] = t
        d = self.do(d0)
        d.addCallback(self.assertEqual,
                      [tOPEN(0),b'dict',
                        1, b"a",
                        2, tOPEN(1),b'tuple',
                            tOPEN(2),b'reference', 0, tCLOSE(2),
                           tCLOSE(1),
                       tCLOSE(0)])
        return d


class ErrorfulSlicer(slicer.BaseSlicer):
    def __init__(self, mode, shouldSucceed, ignoreChildDeath=False):
        self.mode = mode
        self.items = [1]
        self.items.append(mode)
        self.items.append(3)
        #if mode not in ('success', 'deferred-good'):
        if not shouldSucceed:
            self.items.append("unreached")
        self.counter = -1
        self.childDied = False
        self.ignoreChildDeath = ignoreChildDeath

    def slice(self, streamable, banana):
        self.streamable = streamable
        if self.mode == "slice":
            raise Violation("slice failed")
        return iter(self)

    def __iter__(self):
        return self

    def __next__(self):
        self.counter += 1
        if not self.items:
            raise StopIteration
        obj = self.items.pop(0)
        if obj == "next":
            raise Violation("next failed")
        if obj == "deferred-good":
            return fireEventually(None)
        if obj == "deferred-bad":
            d = defer.Deferred()
            # the Banana should bail, so don't bother with the timer
            return d
        if obj == "newSlicerFor":
            unserializable = open("unserializable.txt", "w")
            # Hah! Serialize that!
            return unserializable
        if obj == "unreached":
            print("error: slicer.next called after it should have stopped")
        return obj
    next = __next__

    def childAborted(self, v):
        self.childDied = True
        if self.ignoreChildDeath:
            return None
        return v

    def describe(self):
        return "ErrorfulSlicer[%d]" % self.counter

# Slicer creation (schema pre-validation?)
# .slice (called in pushSlicer) ?
# .slice.next raising Violation
# .slice.next returning Deferred when streaming isn't allowed
# .sendToken (non-primitive token, can't happen)
# .newSlicerFor (no ISlicer adapter)
# top.childAborted

class EncodeFailureTest(unittest.TestCase):
    def setUp(self):
        self.banana = TokenBanana()
        self.banana.slicerClass = storage.StorageRootSlicer
        self.banana.unslicerClass = storage.StorageRootUnslicer
        self.banana.connectionMade()

    def tearDown(self):
        return flushEventualQueue()

    def send(self, obj):
        self.banana.tokens = []
        d = self.banana.send(obj)
        d.addCallback(lambda res: self.banana.tokens)
        return d

    def testSuccess1(self):
        # make sure the test slicer works correctly
        s = ErrorfulSlicer(b"success", True)
        d = self.send(s)
        d.addCallback(self.assertEqual,
                      [('OPEN', 0), 1, b'success', 3, ('CLOSE', 0)])
        return d

    def testSuccessStreaming(self):
        # success
        s = ErrorfulSlicer("deferred-good", True)
        d = self.send(s)
        d.addCallback(self.assertEqual,
                      [('OPEN', 0), 1, 3, ('CLOSE', 0)])
        return d

    def test1(self):
        # failure during .slice (called from pushSlicer)
        s = ErrorfulSlicer("slice", False)
        d = self.send(s)
        d.addCallbacks(lambda res: self.fail("this was supposed to fail"),
                       self._test1_1)
        return d
    def _test1_1(self, e):
        e.trap(Violation)
        self.assertEqual(e.value.where, "<RootSlicer>")
        self.assertEqual(e.value.args, ("slice failed",))
        self.assertEqual(self.banana.tokens, [])

    def test2(self):
        # .slice.next raising Violation
        s = ErrorfulSlicer("next", False)
        d = self.send(s)
        d.addCallbacks(lambda res: self.fail("this was supposed to fail"),
                       self._test2_1)
        return d
    def _test2_1(self, e):
        e.trap(Violation)
        self.assertEqual(e.value.where, "<RootSlicer>.ErrorfulSlicer[1]")
        self.assertEqual(e.value.args, ("next failed",))
        self.assertEqual(self.banana.tokens,
                             [('OPEN', 0), 1, ('ABORT',), ('CLOSE', 0)])

    def test3(self):
        # .slice.next returning Deferred when streaming isn't allowed
        self.banana.rootSlicer.allowStreaming(False)
        s = ErrorfulSlicer("deferred-bad", False)
        d = self.send(s)
        d.addCallbacks(lambda res: self.fail("this was supposed to fail"),
                       self._test3_1)
        return d
    def _test3_1(self, e):
        e.trap(Violation)
        self.assertEqual(e.value.where, "<RootSlicer>.ErrorfulSlicer[1]")
        self.assertEqual(e.value.args, ("parent not streamable",))
        self.assertEqual(self.banana.tokens,
                             [('OPEN', 0), 1, ('ABORT',), ('CLOSE', 0)])

    def test4(self):
        # .newSlicerFor (no ISlicer adapter), parent propagates upwards
        s = ErrorfulSlicer("newSlicerFor", False)
        d = self.send(s)
        d.addCallbacks(lambda res: self.fail("this was supposed to fail"),
                       self._test4_1, errbackArgs=(s,))
        return d
    def _test4_1(self, e, s):
        e.trap(Violation)
        self.assertEqual(e.value.where, "<RootSlicer>.ErrorfulSlicer[1]")
        # this used to assert that the error included a description of the
        # object that could not be serialized. The open filehandle we use was
        # presented as "<open file..>" on py2, and "<_io.TextIOWrapper..>" on
        # py3, and it was too difficult to unify these representations, so I
        # just removed the detailed check.
        self.failUnlessSubstring("cannot serialize", e.value.args[0])
        self.assertTrue(s.childDied)
        self.assertEqual(self.banana.tokens,
                             [('OPEN', 0), 1, ('ABORT',), ('CLOSE', 0)])

    def test5(self):
        # .newSlicerFor (no ISlicer adapter), parent ignores
        s = ErrorfulSlicer("newSlicerFor", True, True)
        d = self.send(s)
        d.addCallback(lambda res:
                      self.assertTrue(s.childDied)) # noticed but ignored
        d.addCallback(lambda res:
                      self.assertEqual(self.banana.tokens,
                                           [('OPEN', 0), 1, 3, ('CLOSE', 0)]))
        return d

# receiving side:
#  long header (>64 bytes)
#  checkToken (top.openerCheckToken)
#  checkToken (top.checkToken)
#  typebyte == LIST (oldbanana)
#  bad VOCAB key # TODO
#  too-long vocab key
#  bad FLOAT encoding  # I don't there is such a thing
#  top.receiveClose
#  top.finish
#  top.reportViolation
#  oldtop.finish (in from handleViolation)
#  top.doOpen
#  top.start
# plus all of these when discardCount != 0

class ErrorfulUnslicer(slicer.BaseUnslicer):
    debug = False

    def doOpen(self, opentype):
        if self.mode == "doOpen":
            raise Violation("boom")
        return slicer.BaseUnslicer.doOpen(self, opentype)

    def start(self, count):
        self.mode = self.protocol.mode
        self.ignoreChildDeath = self.protocol.ignoreChildDeath
        if self.debug:
            print("ErrorfulUnslicer.start, mode=%s" % self.mode)
        self.list = []
        if self.mode == "start":
            raise Violation("boom")

    def openerCheckToken(self, typebyte, size, opentype):
        if self.debug:
            print("ErrorfulUnslicer.openerCheckToken(%s)" % tokenNames[typebyte])
        if self.mode == "openerCheckToken":
            raise Violation("boom")
        return slicer.BaseUnslicer.openerCheckToken(self, typebyte,
                                                    size, opentype)
    def checkToken(self, typebyte, size):
        if self.debug:
            print("ErrorfulUnslicer.checkToken(%s)" % tokenNames[typebyte])
        if self.mode == "checkToken":
            raise Violation("boom")
        if self.mode == "checkToken-OPEN" and typebyte == OPEN:
            raise Violation("boom")
        return slicer.BaseUnslicer.checkToken(self, typebyte, size)

    def receiveChild(self, obj, ready_deferred=None):
        if self.debug: print("ErrorfulUnslicer.receiveChild", obj)
        if self.mode == "receiveChild":
            raise Violation("boom")
        self.list.append(obj)

    def reportViolation(self, why):
        if self.ignoreChildDeath:
            return None
        return why

    def receiveClose(self):
        if self.debug: print("ErrorfulUnslicer.receiveClose")
        if self.protocol.mode == "receiveClose":
            raise Violation("boom")
        return self.list, None

    def finish(self):
        if self.debug: print("ErrorfulUnslicer.receiveClose")
        if self.protocol.mode == "finish":
            raise Violation("boom")

    def describe(self):
        return "errorful"

class FailingUnslicer(TupleUnslicer):
    def receiveChild(self, obj, ready_deferred=None):
        if self.protocol.mode != "success":
            raise Violation("foom")
        return TupleUnslicer.receiveChild(self, obj, ready_deferred)
    def describe(self):
        return "failing"

class DecodeFailureTest(TestBananaMixin, unittest.TestCase):
    listStream = join(bOPEN(b"errorful", 0), bINT(1), bINT(2), bCLOSE(0))
    nestedStream = join(bOPEN(b"errorful", 0), bINT(1),
                        bOPEN(b"list", 1), bINT(2), bINT(3), bCLOSE(1),
                        bCLOSE(0))
    nestedStream2 = join(bOPEN(b"failing", 0), bSTR(b"a"),
                          bOPEN(b"errorful", 1), bINT(1),
                           bOPEN(b"list", 2), bINT(2), bINT(3), bCLOSE(2),
                          bCLOSE(1),
                          bSTR(b"b"),
                          bCLOSE(0),
                         )
    abortStream = join(bOPEN(b"errorful", 0), bINT(1),
                        bOPEN(b"list", 1),
                         bINT(2), bABORT(1), bINT(3),
                        bCLOSE(1),
                       bCLOSE(0))

    def setUp(self):
        TestBananaMixin.setUp(self)
        d = {('errorful',): ErrorfulUnslicer,
             ('failing',): FailingUnslicer,
             }
        self.banana.rootUnslicer.topRegistries.insert(0, d)
        self.banana.rootUnslicer.openRegistries.insert(0, d)
        self.banana.ignoreChildDeath = False

    def testSuccess1(self):
        self.banana.mode = "success"
        o = self.shouldDecode(self.listStream)
        self.assertEqual(o, [1,2])
        o = self.shouldDecode(self.nestedStream)
        self.assertEqual(o, [1,[2,3]])
        o = self.shouldDecode(self.nestedStream2)
        self.assertEqual(o, (b"a",[1,[2,3]],b"b"))

    def testLongHeader(self):
        # would be a string but the header is too long
        s = b"\x01" * 66 + b"\x82" + b"stupidly long string"
        f = self.shouldDropConnection(s)
        self.assertTrue(f.value.args[0].startswith("token prefix is limited to 64 bytes"))

    def testLongHeader2(self):
        # bad string while discarding
        s = b"\x01" * 66 + b"\x82" + b"stupidly long string"
        s = bOPEN("errorful",0) + bINT(1) + s + bINT(2) + bCLOSE(0)
        self.banana.mode = "start"
        f = self.shouldDropConnection(s)
        self.assertTrue(f.value.args[0].startswith("token prefix is limited to 64 bytes"))

    def testCheckToken1(self):
        # violation raised in top.openerCheckToken
        self.banana.mode = "openerCheckToken"
        f = self.shouldFail(self.nestedStream)
        self.assertEqual(f.value.where, "<RootUnslicer>.errorful")
        self.assertEqual(f.value.args[0], "boom")
        self.testSuccess1()

    def testCheckToken2(self):
        # violation raised in top.openerCheckToken, but the error is
        # absorbed
        self.banana.mode = "openerCheckToken"
        self.banana.ignoreChildDeath = True
        o = self.shouldDecode(self.nestedStream)
        self.assertEqual(o, [1])
        self.testSuccess1()

    def testCheckToken3(self):
        # violation raised in top.checkToken
        self.banana.mode = "checkToken"
        f = self.shouldFail(self.listStream)
        self.assertEqual(f.value.where, "<RootUnslicer>.errorful")
        self.assertEqual(f.value.args[0], "boom")
        self.testSuccess1()

    def testCheckToken4(self):
        # violation raised in top.checkToken, but only for the OPEN that
        # starts the nested list. The error is absorbed.
        self.banana.mode = "checkToken-OPEN"
        self.banana.ignoreChildDeath = True
        o = self.shouldDecode(self.nestedStream)
        self.assertEqual(o, [1])
        self.testSuccess1()

    def testCheckToken5(self):
        # violation raised in top.checkToken, while discarding
        self.banana.mode = "checkToken"
        #self.banana.debugReceive=True
        f = self.shouldFail(self.nestedStream2)
        self.assertEqual(f.value.where, "<RootUnslicer>.failing")
        self.assertEqual(f.value.args[0], "foom")
        self.testSuccess1()

    def testReceiveChild1(self):
        self.banana.mode = "receiveChild"
        f = self.shouldFail(self.listStream)
        self.assertEqual(f.value.where, "<RootUnslicer>.errorful")
        self.assertEqual(f.value.args[0], "boom")
        self.testSuccess1()

    def testReceiveChild2(self):
        self.banana.mode = "receiveChild"
        f = self.shouldFail(self.nestedStream2)
        self.assertEqual(f.value.where, "<RootUnslicer>.failing")
        self.assertEqual(f.value.args[0], "foom")
        self.testSuccess1()

    def testReceiveChild3(self):
        self.banana.mode = "receiveChild"
        # the ABORT should be ignored, since it is in the middle of a
        # sequence which is being ignored. One possible bug is that the
        # ABORT delivers a second Violation. In this test, we only record
        # the last Violation, so we'll catch that case.
        f = self.shouldFail(self.abortStream)
        self.assertEqual(f.value.where, "<RootUnslicer>.errorful")
        self.assertEqual(f.value.args[0], "boom")
        # (the other Violation would be at 'root', of type 'ABORT received'
        self.testSuccess1()

    def testReceiveClose1(self):
        self.banana.mode = "receiveClose"
        f = self.shouldFail(self.listStream)
        self.assertEqual(f.value.where, "<RootUnslicer>.errorful")
        self.assertEqual(f.value.args[0], "boom")
        self.testSuccess1()

    def testReceiveClose2(self):
        self.banana.mode = "receiveClose"
        f = self.shouldFail(self.nestedStream2)
        self.assertEqual(f.value.where, "<RootUnslicer>.failing")
        self.assertEqual(f.value.args[0], "foom")
        self.testSuccess1()

    def testFinish1(self):
        self.banana.mode = "finish"
        f = self.shouldFail(self.listStream)
        self.assertEqual(f.value.where, "<RootUnslicer>.errorful")
        self.assertEqual(f.value.args[0], "boom")
        self.testSuccess1()

    def testFinish2(self):
        self.banana.mode = "finish"
        f = self.shouldFail(self.nestedStream2)
        self.assertEqual(f.value.where, "<RootUnslicer>.failing")
        self.assertEqual(f.value.args[0], "foom")
        self.testSuccess1()

    def testStart1(self):
        self.banana.mode = "start"
        f = self.shouldFail(self.listStream)
        self.assertEqual(f.value.where, "<RootUnslicer>.errorful")
        self.assertEqual(f.value.args[0], "boom")
        self.testSuccess1()

    def testStart2(self):
        self.banana.mode = "start"
        f = self.shouldFail(self.nestedStream2)
        self.assertEqual(f.value.where, "<RootUnslicer>.failing")
        self.assertEqual(f.value.args[0], "foom")
        self.testSuccess1()

    def testDoOpen1(self):
        self.banana.mode = "doOpen"
        f = self.shouldFail(self.nestedStream)
        self.assertEqual(f.value.where, "<RootUnslicer>.errorful")
        self.assertEqual(f.value.args[0], "boom")
        self.testSuccess1()

    def testDoOpen2(self):
        self.banana.mode = "doOpen"
        f = self.shouldFail(self.nestedStream2)
        self.assertEqual(f.value.where, "<RootUnslicer>.failing")
        self.assertEqual(f.value.args[0], "foom")
        self.testSuccess1()

class ByteStream(TestBananaMixin, unittest.TestCase):

    def test_list(self):
        obj = [1,2]
        expected = join(bOPEN("list", 0),
                         bINT(1), bINT(2),
                        bCLOSE(0),
                        )
        d = self.encode(obj)
        d.addCallback(self.wantEqual, expected)
        return d

    def test_ref6(self):
        # everybody's favorite "([(ref0" test case.
        obj = ([],)
        obj[0].append((obj,))

        expected = join(bOPEN("tuple",0),
                         bOPEN("list",1),
                          bOPEN("tuple",2),
                           bOPEN("reference",3),
                            bINT(0),
                           bCLOSE(3),
                          bCLOSE(2),
                         bCLOSE(1),
                        bCLOSE(0))
        d = self.encode(obj)
        d.addCallback(self.wantEqual, expected)
        return d

class InboundByteStream(TestBananaMixin, unittest.TestCase):

    def check(self, obj, stream):
        # use a new Banana for each check
        self.makeBanana()
        obj2 = self.shouldDecode(stream)
        self.assertEqual(obj, obj2)

    def testInt(self):
        self.check(1, b"\x01\x81")
        self.check(130, b"\x02\x01\x81")
        self.check(-1, b"\x01\x83")
        self.check(-130, b"\x02\x01\x83")
        self.check(0, bINT(0))
        self.check(1, bINT(1))
        self.check(127, bINT(127))
        self.check(-1, bINT(-1))
        self.check(-127, bINT(-127))

    def testLong(self):
        self.check(long_type(258), b"\x02\x85\x01\x02") # TODO: 0x85 for LONGINT??
        self.check(long_type(-258), b"\x02\x86\x01\x02") # TODO: 0x85 for LONGINT??
        self.check(long_type(0), b"\x85")
        self.check(long_type(0), b"\x00\x85")
        self.check(long_type(0), b"\x86")
        self.check(long_type(0), b"\x00\x86")

    def testString(self):
        self.check(b"", b"\x82")
        self.check(b"", b"\x00\x82")
        self.check(b"", b"\x00\x00\x82")
        self.check(b"", b"\x00" * 64 + b"\x82")

        f = self.shouldDropConnection(b"\x00" * 65)
        self.assertEqual(f.value.where, "<RootUnslicer>")
        self.assertTrue(f.value.args[0].startswith("token prefix is limited to 64 bytes"))
        f = self.shouldDropConnection(b"\x00" * 65 + b"\x82")
        self.assertEqual(f.value.where, "<RootUnslicer>")
        self.assertTrue(f.value.args[0].startswith("token prefix is limited to 64 bytes"))

        self.check(b"a", b"\x01\x82a")
        self.check(b"b"*130, b"\x02\x01\x82" + b"b"*130 + b"extra")
        self.check(b"c"*1025, b"\x01\x08\x82" + b"c" * 1025 + b"extra")
        self.check(b"fluuber", bSTR("fluuber"))


    def testList(self):
        self.check([1,2],
                   join(bOPEN('list',1),
                        bINT(1), bINT(2),
                        bCLOSE(1)))
        self.check([1,b"b"],
                   join(bOPEN('list',1), bINT(1),
                        b"\x01\x82b",
                        bCLOSE(1)))
        self.check([1,2,[3,4]],
                   join(bOPEN('list',1), bINT(1), bINT(2),
                         bOPEN('list',2), bINT(3), bINT(4),
                         bCLOSE(2),
                        bCLOSE(1)))

    def testTuple(self):
        self.check((1,2),
                   join(bOPEN('tuple',1), bINT(1), bINT(2),
                        bCLOSE(1)))

    def testDict(self):
        self.check({1:b"a", 2:[b"b",b"c"]},
                   join(bOPEN('dict',1),
                        bINT(1), bSTR("a"),
                        bINT(2), bOPEN('list',2),
                         bSTR("b"), bSTR("c"),
                        bCLOSE(2),
                        bCLOSE(1)))

    def TRUE(self):
        return join(bOPEN("boolean",2), bINT(1), bCLOSE(2))
    def FALSE(self):
        return join(bOPEN("boolean",2), bINT(0), bCLOSE(2))

    def testBool(self):
        self.check(True, self.TRUE())
        self.check(False, self.FALSE())

class InboundByteStream2(TestBananaMixin, unittest.TestCase):

    def setConstraints(self, constraint, childConstraint):
        if constraint:
            constraint = IConstraint(constraint)
        self.banana.receiveStack[-1].constraint = constraint

        if childConstraint:
            childConstraint = IConstraint(childConstraint)
        self.banana.receiveStack[-1].childConstraint = childConstraint

    def conform2(self, stream, obj, constraint=None, childConstraint=None):
        self.setConstraints(constraint, childConstraint)
        obj2 = self.shouldDecode(stream)
        self.assertEqual(obj, obj2)

    def violate2(self, stream, where, constraint=None, childConstraint=None):
        self.setConstraints(constraint, childConstraint)
        f = self.shouldFail(stream)
        self.assertEqual(f.value.where, where)
        self.assertEqual(len(self.banana.receiveStack), 1)

    def testConstrainedInt(self):
        pass # TODO: after implementing new LONGINT token

    def testConstrainedString(self):
        self.conform2(b"\x82", b"",
                      schema.StringConstraint(10))
        self.conform2(b"\x0a\x82" + b"a"*10 + b"extra", b"a"*10,
                      schema.StringConstraint(10))
        self.violate2(b"\x0b\x82" + b"a"*11 + b"extra",
                      "<RootUnslicer>",
                      schema.StringConstraint(10))

    def NOTtestFoo(self):
        if 0:
            a100 = six.int2byte(100) + "\x82" + "a"*100
            b100 = six.int2byte(100) + "\x82" + "b"*100
            self.violate2(join(bOPEN('list',1),
                               bOPEN('list',2), a100, b100, bCLOSE(2),
                               bCLOSE(1)),
                          "<RootUnslicer>.[0].[0]",
                          schema.ListOf(
                schema.ListOf(schema.StringConstraint(99), 2), 2))

        def OPENweird(count, weird):
            return six.int2byte(count) + "\x88" + weird

        self.violate2(join(bOPEN('list',1),
                           bOPEN('list',2),
                           OPENweird(3, bINT(64)),
                           bINT(1), bINT(2), bCLOSE(3),
                           bCLOSE(2),
                           bCLOSE(1)),
                      "<RootUnslicer>.[0].[0]", None)



    def testConstrainedList(self):
        self.conform2(join(bOPEN('list',1), bINT(1), bINT(2),
                           bCLOSE(1)),
                      [1,2],
                      schema.ListOf(int))
        self.violate2(join(bOPEN('list',1), bINT(1), b"\x01\x82b",
                           bCLOSE(1)),
                      "<RootUnslicer>.[1]",
                      schema.ListOf(int))
        self.conform2(join(bOPEN('list',1),
                            bINT(1), bINT(2), bINT(3),
                           bCLOSE(1)),
                      [1,2,3],
                      schema.ListOf(int, maxLength=3))
        self.violate2(join(bOPEN('list',1),
                            bINT(1), bINT(2), bINT(3), bINT(4),
                           bCLOSE(1)),
                      "<RootUnslicer>.[3]",
                      schema.ListOf(int, maxLength=3))
        a100 = six.int2byte(100) + b"\x82" + b"a"*100
        b100 = six.int2byte(100) + b"\x82" + b"b"*100
        self.conform2(join(bOPEN('list',1), a100, b100, bCLOSE(1)),
                      [b"a"*100, b"b"*100],
                      schema.ListOf(schema.StringConstraint(100), 2))
        self.violate2(join(bOPEN('list',1), a100, b100, bCLOSE(1)),
                      "<RootUnslicer>.[0]",
                      schema.ListOf(schema.StringConstraint(99), 2))
        self.violate2(join(bOPEN('list',1), a100, b100, a100, bCLOSE(1)),
                      "<RootUnslicer>.[2]",
                      schema.ListOf(schema.StringConstraint(100), 2))

        self.conform2(join(bOPEN('list',1),
                            bOPEN('list',2),
                             bINT(11), bINT(12),
                            bCLOSE(2),
                            bOPEN('list',3),
                             bINT(21), bINT(22), bINT(23),
                            bCLOSE(3),
                           bCLOSE(1)),
                      [[11,12], [21, 22, 23]],
                      schema.ListOf(schema.ListOf(int, maxLength=3)))

        self.violate2(join(bOPEN('list',1),
                            bOPEN('list',2),
                             bINT(11), bINT(12),
                            bCLOSE(2),
                            bOPEN('list',3),
                             bINT(21), bINT(22), bINT(23),
                            bCLOSE(3),
                           bCLOSE(1)),
                      "<RootUnslicer>.[1].[2]",
                      schema.ListOf(schema.ListOf(int, maxLength=2)))

    def testConstrainedTuple(self):
        self.conform2(join(bOPEN('tuple',1), bINT(1), bINT(2),
                           bCLOSE(1)),
                      (1,2),
                      schema.TupleOf(int, int))
        self.violate2(join(bOPEN('tuple',1),
                           bINT(1), bINT(2), bINT(3),
                           bCLOSE(1)),
                      "<RootUnslicer>.[2]",
                      schema.TupleOf(int, int))
        self.violate2(join(bOPEN('tuple',1),
                           bINT(1), bSTR("not a number"),
                           bCLOSE(1)),
                      "<RootUnslicer>.[1]",
                      schema.TupleOf(int, int))
        self.conform2(join(bOPEN('tuple',1),
                           bINT(1), bSTR("twine"),
                           bCLOSE(1)),
                      (1, b"twine"),
                      schema.TupleOf(int, bytes))
        self.conform2(join(bOPEN('tuple',1),
                           bINT(1),
                            bOPEN('list',2),
                             bINT(1), bINT(2), bINT(3),
                            bCLOSE(2),
                           bCLOSE(1)),
                      (1, [1,2,3]),
                      schema.TupleOf(int, schema.ListOf(int)))
        self.conform2(join(bOPEN('tuple',1),
                           bINT(1),
                            bOPEN('list',2),
                             bOPEN('list',3), bINT(2), bCLOSE(3),
                             bOPEN('list',4), bINT(3), bCLOSE(4),
                            bCLOSE(2),
                           bCLOSE(1)),
                      (1, [[2], [3]]),
                      schema.TupleOf(int, schema.ListOf(schema.ListOf(int))))
        self.violate2(join(bOPEN('tuple',1),
                           bINT(1),
                            bOPEN('list',2),
                             bOPEN('list',3),
                              bSTR("nan"),
                             bCLOSE(3),
                             bOPEN('list',4), bINT(3), bCLOSE(4),
                            bCLOSE(2),
                           bCLOSE(1)),
                      "<RootUnslicer>.[1].[0].[0]",
                      schema.TupleOf(int, schema.ListOf(schema.ListOf(int))))

    def testConstrainedDict(self):
        self.conform2(join(bOPEN('dict',1),
                           bINT(1), bSTR("a"),
                           bINT(2), bSTR("b"),
                           bINT(3), bSTR("c"),
                           bCLOSE(1)),
                      {1:b"a", 2:b"b", 3:b"c"},
                      schema.DictOf(int, bytes))
        self.conform2(join(bOPEN('dict',1),
                           bINT(1), bSTR("a"),
                           bINT(2), bSTR("b"),
                           bINT(3), bSTR("c"),
                           bCLOSE(1)),
                      {1:b"a", 2:b"b", 3:b"c"},
                      schema.DictOf(int, bytes, maxKeys=3))
        self.violate2(join(bOPEN('dict',1),
                           bINT(1), bSTR("a"),
                           bINT(2), bINT(10),
                           bINT(3), bSTR("c"),
                           bCLOSE(1)),
                      "<RootUnslicer>.{}[2]",
                      schema.DictOf(int, bytes))
        self.violate2(join(bOPEN('dict',1),
                           bINT(1), bSTR("a"),
                           bINT(2), bSTR("b"),
                           bINT(3), bSTR("c"),
                           bCLOSE(1)),
                      "<RootUnslicer>.{}",
                      schema.DictOf(int, bytes, maxKeys=2))

    def TRUE(self):
        return join(bOPEN("boolean",2), bINT(1), bCLOSE(2))
    def FALSE(self):
        return join(bOPEN("boolean",2), bINT(0), bCLOSE(2))

    def testConstrainedBool(self):
        self.conform2(self.TRUE(),
                      True,
                      bool)
        self.conform2(self.TRUE(),
                      True,
                      schema.BooleanConstraint())
        self.conform2(self.FALSE(),
                      False,
                      schema.BooleanConstraint())

        # booleans have ints, not strings. To do otherwise is a protocol
        # error, not a schema Violation.
        f = self.shouldDropConnection(join(bOPEN("boolean",1),
                                            bSTR("vrai"),
                                           bCLOSE(1)))
        self.assertEqual(f.value.args[0],
                             "BooleanUnslicer only accepts an INT token")

        # but true/false is a constraint, and is reported with Violation
        self.violate2(self.TRUE(),
                      "<RootUnslicer>.<bool>",
                      schema.BooleanConstraint(False))
        self.violate2(self.FALSE(),
                      "<RootUnslicer>.<bool>",
                      schema.BooleanConstraint(True))


class ThereAndBackAgain(TestBananaMixin, unittest.TestCase):

    def test_int(self):
        d = self.looptest(42)
        d.addCallback(lambda res: self.looptest(-47))
        return d

    def test_bigint(self):
        # some of these are small enough to fit in an INT
        d = self.looptest(int(2**31-1)) # most positive representable INT
        d.addCallback(lambda res: self.looptest(long_type(2**31+0)))
        d.addCallback(lambda res: self.looptest(long_type(2**31+1)))

        d.addCallback(lambda res: self.looptest(long_type(-2**31-1)))
        # the following is the most negative representable INT
        d.addCallback(lambda res: self.looptest(int(-2**31+0)))
        d.addCallback(lambda res: self.looptest(int(-2**31+1)))

        d.addCallback(lambda res: self.looptest(long_type(2**100)))
        d.addCallback(lambda res: self.looptest(long_type(-2**100)))
        d.addCallback(lambda res: self.looptest(long_type(2**1000)))
        d.addCallback(lambda res: self.looptest(long_type(-2**1000)))
        return d

    def test_decimal(self):
        d = self.looptest(Decimal(0))
        d.addCallback(lambda res: self.looptest(Decimal(123)))
        d.addCallback(lambda res: self.looptest(Decimal(-123)))
        d.addCallback(lambda res: self.looptest(Decimal("123")))
        d.addCallback(lambda res: self.looptest(Decimal("-123")))
        d.addCallback(lambda res: self.looptest(Decimal("123.456")))
        d.addCallback(lambda res: self.looptest(Decimal("-123.456")))
        d.addCallback(lambda res: self.looptest(Decimal("0.000003")))
        d.addCallback(lambda res: self.looptest(Decimal("-0.000003")))
        d.addCallback(lambda res: self.looptest(Decimal('Inf')))
        d.addCallback(lambda res: self.looptest(Decimal('-Inf')))
        # NaN is a bit weird: by definition, NaN != NaN. So we need to make
        # sure it serializes, and that str(new) == str(old), but we don't
        # check that new == old.
        d.addCallback(lambda res: self.loop(Decimal('NaN')))
        def _check_NaN(new_NaN):
            self.assertEqual(str(new_NaN), str(Decimal('NaN')))
        d.addCallback(_check_NaN)
        return d

    def test_string(self):
        return self.looptest("biggles")
    def test_unicode(self):
        return self.looptest(u"biggles\u1234")
    def test_list(self):
        return self.looptest([1,2])
    def test_tuple(self):
        return self.looptest((1,2))
    def test_set(self):
        d = self.looptest(set([1,2]))
        d.addCallback(lambda res: self.looptest(frozenset([1,2])))
        return d

    def test_bool(self):
        d = self.looptest(True)
        d.addCallback(lambda res: self.looptest(False))
        return d
    def test_float(self):
        return self.looptest(20.3)
    def test_none(self):
        d = self.loop(None)
        d.addCallback(lambda n2: self.assertTrue(n2 is None))
        return d
    def test_dict(self):
        return self.looptest({'a':1})

    # some stuff from test_newjelly
    def testIdentity(self):
        # test to make sure that objects retain identity properly
        x = []
        y = (x,)
        x.append(y)
        x.append(y)
        self.assertIdentical(x[0], x[1])
        self.assertIdentical(x[0][0], x)
        d = self.encode(x)
        d.addCallback(self.shouldDecode)
        d.addCallback(self._testIdentity_1)
        return d
    def _testIdentity_1(self, z):
        self.assertIdentical(z[0], z[1])
        self.assertIdentical(z[0][0], z)

    def testUnicode(self):
        x = [six.text_type('blah')]
        d = self.loop(x)
        d.addCallback(self._testUnicode_1, x)
        return d
    def _testUnicode_1(self, y, x):
        self.assertEquals(x, y)
        self.assertEquals(type(x[0]), type(y[0]))

    def testStressReferences(self):
        reref = []
        toplevelTuple = ({'list': reref}, reref)
        reref.append(toplevelTuple)
        d = self.loop(toplevelTuple)
        d.addCallback(self._testStressReferences_1)
        return d
    def _testStressReferences_1(self, z):
        self.assertIdentical(z[0]['list'], z[1])
        self.assertIdentical(z[0]['list'][0], z)

    def test_cycles_1(self):
        # a list that contains a tuple that can't be referenced yet
        a = []
        t1 = (a,)
        t2 = (t1,)
        a.append(t2)
        d = self.loop(t1)
        d.addCallback(lambda z: self.assertIdentical(z[0][0][0], z))
        return d

    def test_cycles_2(self):
        # a dict that contains a tuple that can't be referenced yet.
        a = {}
        t1 = (a,)
        t2 = (t1,)
        a['foo'] = t2
        d = self.loop(t1)
        d.addCallback(lambda z: self.assertIdentical(z[0]['foo'][0], z))
        return d

    def test_cycles_3(self):
        # sets seem to be transitively immutable: any mutable contents would
        # be unhashable, and sets can only contain hashable objects.
        # Therefore sets cannot participate in cycles the way that tuples
        # can.

        # a set that contains a tuple that can't be referenced yet. You can't
        # actually create this in python, because you can only create a set
        # out of hashable objects, and sets aren't hashable, and a tuple that
        # contains a set is not hashable.
        a = set()
        t1 = (a,)
        t2 = (t1,)
        a.add(t2)
        d = self.loop(t1)
        d.addCallback(lambda z: self.assertIdentical(list(z[0])[0][0], z))

        # a list that contains a frozenset that can't be referenced yet
        a = []
        t1 = frozenset([a])
        t2 = frozenset([t1])
        a.append(t2)
        d = self.loop(t1)
        d.addCallback(lambda z:
                      self.assertIdentical(list(list(z)[0][0])[0], z))

        # a dict that contains a frozenset that can't be referenced yet.
        a = {}
        t1 = frozenset([a])
        t2 = frozenset([t1])
        a['foo'] = t2
        d = self.loop(t1)
        d.addCallback(lambda z:
                      self.assertIdentical(list(list(z)[0]['foo'])[0], z))

        # a set that contains a frozenset that can't be referenced yet.
        a = set()
        t1 = frozenset([a])
        t2 = frozenset([t1])
        a.add(t2)
        d = self.loop(t1)
        d.addCallback(lambda z: self.assertIdentical(list(list(list(z)[0])[0])[0], z))
        return d
    del test_cycles_3



class VocabTest1(unittest.TestCase):
    def test_incoming1(self):
        b = TokenBanana()
        b.connectionMade()
        vdict = {1: b'list', 2: b'tuple', 3: b'dict'}
        keys = list(vdict.keys())
        keys.sort()
        setVdict = [tOPEN(0),'set-vocab']
        for k in keys:
            setVdict.append(k)
            setVdict.append(vdict[k])
        setVdict.append(tCLOSE(0))
        b.dataReceived(untokenize(setVdict))
        # banana should now know this vocabulary
        self.assertEqual(b.incomingVocabulary, vdict)

    def test_outgoing(self):
        b = TokenBanana()
        b.connectionMade()
        b.tokens = []
        strings = ["list", "tuple", "dict"]
        vdict = {0: b'list', 1: b'tuple', 2: b'dict'}
        keys = list(vdict.keys())
        keys.sort()
        setVdict = [tOPEN(0),b'set-vocab']
        for k in keys:
            setVdict.append(k)
            setVdict.append(vdict[k])
        setVdict.append(tCLOSE(0))
        b.setOutgoingVocabulary(strings)
        vocabTokens = b.tokens
        self.assertEqual(vocabTokens, setVdict)

    def test_table_hashes(self):
        # make sure that we don't change any published vocab tables, and that
        # we don't change the hash algorithm that they use
        hash_v0 = vocab.hashVocabTable(0)
        self.assertEqual(hash_v0, "da39")
        hash_v1 = vocab.hashVocabTable(1)
        self.assertEqual(hash_v1, "bb33")


class VocabTest2(TestBananaMixin, unittest.TestCase):
    def vbOPEN(self, count, opentype):
        opentype = six.ensure_binary(opentype)
        num = self.invdict[opentype]
        return six.int2byte(count) + b"\x88" + six.int2byte(num) + b"\x87"

    def test_loop(self):
        strings = ["list", "tuple", "dict"]
        vdict = {0: b'list', 1: b'tuple', 2: b'dict'}
        self.invdict = dict(list(zip(list(vdict.values()), list(vdict.keys()))))

        self.banana.setOutgoingVocabulary(strings)
        # this next check only happens to work because there is nothing to
        # keep serialization from completing synchronously. If Banana
        # acquires some eventual-sends, this test might need to be rewritten.
        self.assertEqual(self.banana.outgoingVocabulary, self.invdict)
        self.shouldDecode(self.banana.transport.getvalue())
        self.assertEqual(self.banana.incomingVocabulary, vdict)
        self.clearOutput()

        vbOPEN = self.vbOPEN
        expected = b"".join([vbOPEN(1,"list"),
                             vbOPEN(2,"tuple"),
                              vbOPEN(3,"dict"),
                               bSTR('a'), bINT(1),
                              bCLOSE(3),
                             bCLOSE(2),
                            bCLOSE(1)])
        d = self.encode([({b'a':1},)])
        d.addCallback(self.wantEqual, expected)
        return d


class SliceableByItself(slicer.BaseSlicer):
    def __init__(self, value):
        self.value = value
    def slice(self, streamable, banana):
        self.streamable = streamable
        # this is our "instance state"
        yield {b"value": self.value}

class CouldBeSliceable:
    def __init__(self, value):
        self.value = value

class _AndICanHelp(slicer.BaseSlicer):
    def slice(self, streamable, banana):
        self.streamable = streamable
        yield {b"value": self.obj.value}
registerAdapter(_AndICanHelp, CouldBeSliceable, ISlicer)

class Sliceable(unittest.TestCase):
    def setUp(self):
        self.banana = TokenBanana()
        self.banana.connectionMade()
    def do(self, obj):
        return self.banana.testSlice(obj)
    def tearDown(self):
        self.assertTrue(len(self.banana.slicerStack) == 1)
        self.assertTrue(isinstance(self.banana.slicerStack[0][0], RootSlicer))

    def testDirect(self):
        # the object is its own Slicer
        i = SliceableByItself(42)
        d = self.do(i)
        d.addCallback(self.assertEqual,
                      [tOPEN(0),
                       tOPEN(1), b"dict", b"value", 42, tCLOSE(1),
                       tCLOSE(0)])
        return d

    def testAdapter(self):
        # the adapter is the Slicer
        i = CouldBeSliceable(43)
        d = self.do(i)
        d.addCallback(self.assertEqual,
                      [tOPEN(0),
                       tOPEN(1), b"dict", b"value", 43, tCLOSE(1),
                       tCLOSE(0)])
        return d



# TODO: vocab test:
#  send a bunch of strings
#  send an object that stalls
#  send some more strings
#  set the Vocab table to tokenize some of those strings
#  send yet more strings
#  unstall serialization, let everything flow through, verify
