#!/usr/bin/python3

# this testsuite is part of autopkgtest
# autopkgtest is a tool for testing Debian binary packages
#
# autopkgtest is Copyright (C) 2006-2016 Canonical Ltd.
# Author: Martin Pitt <martin.pitt@ubuntu.com>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
#
# See the file CREDITS for a full list of credits information (often
# installed as /usr/share/doc/autopkgtest/CREDITS).

import sys
import os
import re
import subprocess
import unittest
import tempfile
import shutil
import fnmatch
import time
import json
from glob import glob

test_dir = os.path.dirname(os.path.abspath(__file__))
root_dir = os.path.dirname(test_dir)

sys.path.insert(1, test_dir)
import testarchive

# in some corner cases apt-get download might not be available in a build
# environment, so check if this actually works
have_apt = subprocess.call(['apt-get', 'download', 'gir1.2-json-1.0'],
                           stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                           cwd=os.environ.get('TMPDIR', '/tmp')) == 0

have_apt_src = subprocess.call(['apt-cache', 'showsrc', 'coreutils'],
                               stdout=subprocess.PIPE,
                               stderr=subprocess.STDOUT) == 0

have_ubuntu_device_flash = subprocess.call(['which', 'ubuntu-device-flash'],
                                           stdout=subprocess.PIPE,
                                           stderr=subprocess.PIPE) == 0

have_autodep8 = subprocess.call(['which', 'autodep8'], stdout=subprocess.PIPE) == 0
have_git = subprocess.call(['which', 'git'], stdout=subprocess.PIPE) == 0


class AdtTestCase(unittest.TestCase):
    '''Base class with common test setup'''

    def __init__(self, virt_args, *args, **kwargs):
        super(AdtTestCase, self).__init__(*args, **kwargs)
        self.exe = os.path.join(root_dir, 'runner', os.path.basename(__file__))
        self.virt_args = virt_args
        self.orig_home = os.path.expanduser('~')

    def setUp(self):
        self.workdir = tempfile.mkdtemp(prefix='autopkgtest.test.')
        os.chmod(self.workdir, 0o755)
        self.addCleanup(shutil.rmtree, self.workdir)
        self.cwd = os.getcwd()
        self.addCleanup(os.chdir, self.cwd)

        temp_home = os.path.join(self.workdir, 'home')
        shutil.copytree(os.path.join(test_dir, 'home'), temp_home)
        os.chmod(os.path.join(temp_home, '.ssh', 'id_rsa'), 0o600)
        os.mkdir(os.path.join(temp_home, '.cache'))

        # avoid re-downloading ubuntu-device-flash images
        os.symlink(os.path.join(self.orig_home, '.cache', 'ubuntuimages'),
                   os.path.join(temp_home, '.cache', 'ubuntuimages'))

        # also keep per-user LXC containers
        lxc_orig = os.path.join(self.orig_home, '.local', 'share', 'lxc')
        if os.path.isdir(lxc_orig):
            lxc_temp = os.path.join(temp_home, '.local', 'share', 'lxc')
            os.makedirs(os.path.dirname(lxc_temp))
            os.symlink(lxc_orig, lxc_temp)

        # keep LXD client certs and config
        lxc_orig = os.path.join(self.orig_home, '.config', 'lxc')
        lxc_temp = os.path.join(temp_home, '.config', 'lxc')
        os.makedirs(os.path.dirname(lxc_temp))
        os.symlink(lxc_orig, lxc_temp)

        os.environ['HOME'] = temp_home

    def build_src(self, test_control, test_scripts, pkgname='testpkg'):
        '''Create source package tree with given tests.

         @test_control: contents of debian/tests/control
         @test_scripts: map of test name (in debian/tests/) to file contents

        Return path to the source tree.
        '''
        srcdir = os.path.join(self.workdir, pkgname)
        shutil.copytree(os.path.join(test_dir, 'testpkg'), srcdir, symlinks=True)
        if test_control:
            dtdir = os.path.join(srcdir, 'debian', 'tests')
            os.mkdir(dtdir)
            with open(os.path.join(dtdir, 'control'), 'w', encoding='UTF-8') as f:
                f.write(test_control)
            for name, contents in test_scripts.items():
                with open(os.path.join(dtdir, name), 'w', encoding='UTF-8') as f:
                    f.write(contents)

        return srcdir

    def build_dsc(self, test_control, test_scripts):
        '''Create source package dsc with given tests.

         @test_control: contents of debian/tests/control
         @test_scripts: map of test name (in debian/tests/) to file contents

        Return path to the dsc.
        '''
        srcdir = self.build_src(test_control, test_scripts)
        dbp = subprocess.Popen(['dpkg-buildpackage', '-S', '-us', '-uc'],
                               stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                               cwd=srcdir)
        out, err = dbp.communicate()
        self.assertEqual(dbp.returncode, 0, err)
        return os.path.join(os.path.dirname(srcdir), 'testpkg_1.dsc')

    def runtest(self, args, virt_args=None, env=None):
        '''Run adt-run with given arguments with configured virt runner.

         @args: command line args of adt-run, excluding "adt-run" itself;
                "--- virt" will be appended automatically (called from the
                source tree)

        Return a tuple (exit_code, stdout, stderr).
        '''
        p = subprocess.Popen([self.exe] + args +
                             ['---'] + (virt_args or self.virt_args),
                             stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                             env=env)
        (out, err) = p.communicate()
        return (p.returncode, out.decode('UTF-8'), err.decode('UTF-8'))


class DebTestsAll:
    '''Common deb tests for all runners'''

    def test_tree_norestrictions_nobuild_success(self):
        '''source tree, no build, no restrictions, test success'''

        p = self.build_src('Tests: pass\nDepends: coreutils\n',
                           {'pass': '#!/bin/sh\necho I am fine\n'})

        (code, out, err) = self.runtest(['-d', '--no-built-binaries', '--unbuilt-tree=' + p])
        # print('----- out ----\n%s\n----- err ----\n%s\n----' % (out, err))
        # test should succeed
        self.assertEqual(code, 0, err)
        self.assertRegex(out, 'pass\s+PASS', out)

        self.assertIn('processing dependency coreutils', err)
        self.assertNotIn('@@@@@@ test bed setup', err)
        # should show test stdout
        self.assertRegex(out, '(^|\n)I am fine\n')
        # should show summary
        self.assertRegex(err, '@@@ summary\npass\s+PASS\nadt-run: DBG')
        # should log kernel version
        self.assertRegex(err, 'testbed running kernel: Linux \d')
        # should not have any test stderr
        self.assertNotIn(' stderr ', err)
        # should not build package
        self.assertNotIn('dh build', err)
        # should log package version
        self.assertIn('testing package testpkg version 1\n', err)

    def test_tree_build_needed_success(self):
        '''source tree, build-needed restriction, test success'''

        p = self.build_src('Tests: pass\nDepends: coreutils\nRestrictions: build-needed\n',
                           {'pass': '#!/bin/sh -e\n./test_built | grep -q "built script OK"\n'
                            './test_abspath | grep -q "built script OK"\necho GOOD'})
        # add Build-Depends-Indep: and a build profile package
        subprocess.check_call(['sed', '-i', '/^Build-Depends:/ a\Build-Depends-Indep: dpkg-dev, nonexisting <cross>',
                               os.path.join(p, 'debian', 'control')])

        (code, out, err) = self.runtest(['--no-built-binaries', '--unbuilt-tree=' + p])
        # test should succeed
        self.assertEqual(code, 0, out + err)
        self.assertRegex(out, 'pass\s+PASS', out)

        # should build package
        self.assertIn('dh build', err)

        # should show test stdout
        self.assertRegex(out, '(^|\n)GOOD\n')
        # should not have any test stderr
        self.assertNotIn(' stderr ', err)

    def test_isolation(self):
        '''isolation restrictions'''

        p = self.build_src('Tests: ic\nDepends:\nRestrictions: isolation-container\n\n'
                           'Tests: im\nDepends:\nRestrictions: isolation-machine\n',
                           {'ic': '#!/bin/sh\necho container ok',
                            'im': '#!/bin/sh\necho machine ok'})

        (code, out, err) = self.runtest(['-B', '--built-tree=' + p])

        if self.virt_args[0] in ['qemu', 'null']:
            self.assertEqual(code, 0, out + err)
            self.assertRegex(out, 'im\s+PASS', out)
            self.assertIn('\nmachine ok\n', out)
        else:
            self.assertEqual(code, 2, out + err)
            self.assertRegex(out, 'im\s+SKIP .*machine', out)
            self.assertNotIn('machine ok', out)

        if self.virt_args[0] in ['chroot', 'schroot']:
            self.assertRegex(out, 'ic\s+SKIP .*container', out)
            self.assertNotIn('container ok', out)
        else:
            self.assertRegex(out, 'ic\s+PASS', out)
            self.assertIn('container ok\n', out)

    def test_breaks_testbed(self):
        '''breaks-testbed restriction'''

        p = self.build_src('Tests: zap boom\nDepends:\nRestrictions: needs-root breaks-testbed',
                           {'zap': '#!/bin/sh\ntouch /zap\n'
                            '[ -d "$HOME" ] || echo "HOME not set" >&2\n'
                            'apt-get install -y aspell-doc\n',
                            'boom': '#!/bin/sh -e\n[ ! -e /zap ]; [ ! -e /usr/share/doc/aspell-doc ]; touch /boom'})

        (code, out, err) = self.runtest(['--no-built-binaries', '--unbuilt-tree=' + p, '-d'])

        if self.virt_args[0] in ['null', 'chroot', 'schroot']:
            # test should be skipped as these runners doesn't provide revert-full-system
            self.assertEqual(code, 2, err)
            self.assertRegex(out, 'zap\s+SKIP Test breaks testbed')
            # FIXME: we should see this too!
            # self.assertRegex(out, 'boom\s+SKIP Test breaks testbed')
            self.assertNotIn(out, 'no tests')
        else:
            # both tests should run; the second one (boom) should not see the
            # effect of the first one (/zap existing and installing aspell-doc)
            self.assertEqual(code, 0, err)
            self.assertRegex(out, 'zap\s+PASS')
            self.assertRegex(out, 'boom\s+PASS')
            self.assertIn('Unpacking aspell-doc', out)


class DebTestsFailureModes:
    '''Common deb tests for handling various failure modes

    These are not sensitive to the virt runner, thus are ony run with fast
    backends (null, chroot) so that the testsuite does not take longer than
    necessary.
    '''
    def test_tree_norestrictions_nobuild_fail_on_exit(self):
        '''source tree, no build, no restrictions, test fails with non-zero'''

        p = self.build_src('Tests: nz\nDepends: coreutils\n',
                           {'nz': '#!/bin/sh\necho I am sick\nexit 7'})

        (code, out, err) = self.runtest(['--no-built-binaries', '--built-tree=' + p])
        # test should fail
        self.assertEqual(code, 4)
        self.assertRegex(out, 'nz\s+FAIL non-zero exit status 7')

        # should show test stdout
        self.assertRegex(out, '(^|\n)I am sick\n')
        # should not have any test stderr
        self.assertNotIn(' stderr ', err)
        # should not build package
        self.assertNotIn('dh build', err)
        # should show summary
        self.assertRegex(err, '@@@ summary\nnz\s+FAIL non-zero exit status 7\n$')

    def test_tree_norestrictions_nobuild_fail_on_stderr(self):
        '''source tree, no build, no restrictions, test fails with stderr'''

        p = self.build_src('Tests: se\nDepends: coreutils\n',
                           {'se': '#!/bin/sh\necho I am sick >&2\n'})

        (code, out, err) = self.runtest(['--no-built-binaries', '--built-tree=' + p])
        # test should fail
        self.assertEqual(code, 4)
        self.assertRegex(out, '(^|\n)se\s+FAIL stderr: I am sick\n')

        # should show test stderr
        self.assertRegex(err, 'stderr [ -]+\nI am sick', err)

    def test_tree_allow_stderr_nobuild_fail_on_exit(self):
        '''source tree, no build, allow-stderr, test fails with non-zero'''

        p = self.build_src('Tests: nz\nDepends: coreutils\nRestrictions: allow-stderr',
                           {'nz': '#!/bin/sh\necho I am fine >&2\necho babble\nexit 7'})

        (code, out, err) = self.runtest(['--no-built-binaries', '--built-tree=' + p])
        # test should fail
        self.assertEqual(code, 4)
        self.assertRegex(out, 'nz\s+FAIL non-zero exit status 7')

        # should show test stdout/err inline
        self.assertRegex(out, '(^|\n)babble\n')
        self.assertRegex(err, '---+\nI am fine\nadt-run', err)
        # but not complain about stderr
        self.assertNotIn(' stderr ', err)

    def test_tree_norestrictions_nobuild_fail_on_stderr_and_exit(self):
        '''source tree, no build, no restrictions, test fails with stderr+exit'''

        p = self.build_src('Tests: senz\nDepends:\n',
                           {'senz': '#!/bin/sh\necho I am sick >&2\nexit 7\n'})

        (code, out, err) = self.runtest(['--no-built-binaries', '--built-tree=' + p])
        # test should fail
        self.assertEqual(code, 4, err)
        self.assertRegex(out, 'senz\s+FAIL non-zero exit status 7')

        # should show test stderr, but inline
        self.assertRegex(err, 'stderr [ -]+\nI am sick', err)

    def test_tree_allow_stderr_nobuild_success(self):
        '''source tree, no build, allow-stderr, test success'''

        p = self.build_src('Tests: pass\nDepends: coreutils\nRestrictions: allow-stderr',
                           {'pass': '#!/bin/sh\necho I am fine >&2\necho babble'})

        (code, out, err) = self.runtest(['--no-built-binaries', '--built-tree=' + p])
        self.assertEqual(code, 0, err)
        self.assertRegex(out, 'pass\s+PASS', out)

        # should show test stdout/err
        self.assertIn('babble\n', out)
        self.assertRegex(err, '-+\nI am fine\nadt-run \[[0-9: -]+\]: test pass: --', err)
        # but not complain about stderr
        self.assertNotIn(' stderr ', err)

        # should log package version
        self.assertIn('testing package testpkg version 1\n', err)

        # no restricted dependencies functionality as they are already installed
        self.assertNotIn('will only work for some packages', err)


class NullRunner(AdtTestCase, DebTestsAll, DebTestsFailureModes):
    def __init__(self, *args, **kwargs):
        super(NullRunner, self).__init__(['null'], *args, **kwargs)

    def test_tree_output_dir(self):
        '''source tree, explicit --output-dir

        This also covers using upper-case lettes in test names.
        '''
        p = self.build_src('Tests: sP sF\nDepends: coreutils\n\n'
                           'Tests: bP\nDepends: coreutils\nRestrictions: build-needed',
                           {'sP': '#!/bin/sh\n./test_static\n',
                            'sF': '#!/bin/sh\necho kaputt >&2',
                            'bP': '#!/bin/sh\n./test_built'})

        outdir = os.path.join(self.workdir, 'out')
        # check pre-existing, but empty dir
        os.mkdir(outdir)

        (code, out, err) = self.runtest(['--no-built-binaries',
                                         '--unbuilt-tree=' + p,
                                         '--env=COLOR=blue',
                                         '--env=SOMETHING=foo bar',
                                         '--output-dir=' + outdir])

        # test results
        self.assertEqual(code, 4, err)
        self.assertRegex(out, 'sP\s+PASS', out)
        self.assertRegex(out, 'sF\s+FAIL stderr: kaputt', out)
        self.assertRegex(out, 'bP\s+PASS', out)

        # should show test stdout and stderr
        self.assertRegex(out, '(^|\n)static script OK\n')
        self.assertIn('\nbuilt script OK\n', out)
        self.assertRegex(err, 'stderr [ -]+\nkaputt', err)

        # should show summary at the end
        self.assertRegex(err, '@@@ summary\nsP\s+PASS\nsF\s+FAIL stderr: kaputt\nbP\s+PASS\n$')

        # should build package
        self.assertIn('dh build', err)

        # check outdir test stdout/err
        with open(os.path.join(outdir, 'sP-stdout')) as f:
            self.assertEqual(f.read(), 'static script OK\n')
        self.assertFalse(os.path.exists(
            os.path.join(outdir, 'sP-stderr')))
        with open(os.path.join(outdir, 'bP-stdout')) as f:
            self.assertEqual(f.read(), 'built script OK\n')
        self.assertFalse(os.path.exists(
            os.path.join(outdir, 'bP-stderr')))
        self.assertFalse(os.path.exists(
            os.path.join(outdir, 'sF-stdout')))
        with open(os.path.join(outdir, 'sF-stderr')) as f:
            self.assertEqual(f.read(), 'kaputt\n')

        # check outdir log
        with open(os.path.join(outdir, 'log')) as f:
            contents = f.read()
        self.assertIn('build needed for tests', contents)
        self.assertIn('dh build', contents)
        self.assertRegex(contents, 'sF\s+FAIL stderr: kaputt')
        self.assertIn('testing package testpkg version 1\n', contents)
        self.assertRegex(err, '@@@ summary\nsP\s+PASS\nsF\s+FAIL stderr: kaputt\nbP\s+PASS\n$')

        # check summary
        with open(os.path.join(outdir, 'summary')) as f:
            self.assertRegex(f.read(), '^sP\s+PASS\nsF\s+FAIL stderr: kaputt\nbP\s+PASS$')

        # check test package version
        with open(os.path.join(outdir, 'testpkg-version')) as f:
            contents = f.read()
        self.assertEqual(contents, 'testpkg 1\n')

        # check recorded package lists
        with open(os.path.join(outdir, 'testbed-packages')) as f:
            contents = f.read()
            lines = contents.splitlines()
            self.assertGreater(len(lines), 10)
            self.assertRegex(lines[0], '^[0-9a-z.-]+\t[0-9a-z.~-]+')
            self.assertRegex(lines[1], '^[0-9a-z.-]+\t[0-9a-z.~-]+')
            self.assertIn('bash\t', contents)

        # check testinfo
        with open(os.path.join(outdir, 'testinfo.json')) as f:
            info = json.load(f)
        u = os.uname()
        self.assertEqual(info['kernel_version'],
                         '%s %s %s' % (u.sysname, u.release, u.version.strip()))
        self.assertNotIn('test_kernel_versions', info)
        self.assertEqual(info['custom_environment'],
                         ['COLOR=blue', 'SOMETHING=foo bar'])
        self.assertEqual(info['virt_server'], 'autopkgtest-virt-null')
        self.assertEqual(info['nproc'],
                         subprocess.check_output(['nproc'], universal_newlines=True).strip())
        if os.uname()[4] in ['x86_64', 'i686']:
            self.assertIn('cpu_model', info)
            self.assertIn('cpu_flags', info)

        # test don't pull in any additional dependencies
        for t in ['sP', 'sF', 'bP']:
            self.assertEqual(os.path.getsize(os.path.join(
                outdir, '%s-packages' % t)), 0)

        # check for cruft in outdir
        # --no-built-binaries, we don't expect any debs
        files = [i for i in os.listdir(outdir)
                 if not fnmatch.fnmatch(i, '[sb]*-std*') and
                 not fnmatch.fnmatch(i, '[sb]*-packages')]
        self.assertEqual(set(files), set(['log', 'summary', 'testpkg-version',
                                          'testbed-packages', 'testinfo.json']))

    def test_tree_output_dir_nonempty(self):
        '''existing and non-empty --output-dir'''

        p = self.build_src('Tests: p\nDepends:\n', {'p': '#!/bin/sh\ntrue\n'})

        outdir = os.path.join(self.workdir, 'out')
        os.mkdir(outdir)
        # put some cruft into it
        with open(os.path.join(outdir, 'cruft.txt'), 'w') as f:
            f.write('hello world')

        (code, out, err) = self.runtest(['--no-built-binaries',
                                         '--unbuilt-tree=' + p,
                                         '--output-dir=' + outdir])

        # test results
        self.assertEqual(code, 20, err)
        self.assertEqual(out, '')
        self.assertRegex(err, '--output-dir.*not empty')

        # file is still there
        with open(os.path.join(outdir, 'cruft.txt')) as f:
            self.assertEqual(f.read(), 'hello world')
        # nothing else got written there
        self.assertEqual(os.listdir(outdir), ['cruft.txt'])

    def test_tree_output_dir_in_test_tree(self):
        '''source tree, --output-dir in tests tree

        NB that this is a pathological case, but easy to run into.
        '''
        p = self.build_src('Tests: s\nDepends:\n',
                           {'s': '#!/bin/sh\necho OK'})

        outdir = os.path.join(p, 'out')

        (code, out, err) = self.runtest(['-B', p + '/', '--output-dir=' + outdir])

        # test results
        self.assertEqual(code, 0, err)
        self.assertRegex(out, 's\s+PASS', out)

        # should show test stdout and stderr
        self.assertRegex(out, '(^|\n)OK\n')

        # check outdir test stdout/err
        with open(os.path.join(outdir, 's-stdout')) as f:
            self.assertEqual(f.read(), 'OK\n')
        self.assertFalse(os.path.exists(
            os.path.join(outdir, 's-stderr')))

        # check outdir log
        with open(os.path.join(outdir, 'log')) as f:
            contents = f.read()
        self.assertRegex(contents, 's\s+PASS')

    def test_tree_apply_patches(self):
        '''source tree, 3.0 (quilt) patches get applied'''

        p = self.build_src('Tests: pass\nDepends: coreutils\nRestrictions: build-needed\n',
                           {'pass': '#!/bin/sh -e\ndebian/testpkg/usr/bin/test_built\n'
                            'debian/testpkg/usr/bin/test_static\n'})

        # add patch
        patchdir = os.path.join(p, 'debian', 'patches')
        os.mkdir(patchdir)
        with open(os.path.join(patchdir, '01_hack.patch'), 'w') as f:
            f.write('''--- testpkg.orig/test_static
+++ testpkg/test_static
@@ -1,2 +1,2 @@
 #!/bin/sh
-echo "static script OK"
+echo "static patched script OK"
''')
        with open(os.path.join(patchdir, 'series'), 'w') as f:
            f.write('01_hack.patch')

        # turn into 3.0 (quilt) source
        dsrcdir = os.path.join(p, 'debian', 'source')
        os.mkdir(dsrcdir)
        with open(os.path.join(dsrcdir, 'format'), 'w') as f:
            f.write('3.0 (quilt)\n')

        # run tests, should apply unapplied patches
        (code, out, err) = self.runtest(['-B', '-d', '--unbuilt-tree=' + p])

        # test should succeed
        self.assertRegex(out, 'pass\s+PASS', out)
        self.assertEqual(code, 0, err)

        # should have patched source
        self.assertRegex(err, 'dpkg-source:.*01_hack.patch')
        self.assertIn('built script OK\nstatic patched script OK\n', out)

        # should not have any test stderr
        self.assertNotIn(' stderr ', err)

        # should build package
        self.assertIn('dh build', err)

    def test_dsc_norestrictions_nobuild_success(self):
        '''dsc, no build, no restrictions, test success'''

        p = self.build_dsc('Tests: pass\nDepends: coreutils\n',
                           {'pass': '#!/bin/sh\necho I am fine\n'})

        (code, out, err) = self.runtest(['-d', '--no-built-binaries', p])
        # test should succeed
        self.assertEqual(code, 0, err)
        self.assertIn('coreutils', err)
        self.assertRegex(out, 'pass\s+PASS', out)

        self.assertIn('processing dependency coreutils', err)
        # should show test stdout
        self.assertRegex(out, '(^|\n)I am fine\n')
        # should not have any test stderr
        self.assertNotIn(' stderr ', err)

        # should log package version
        self.assertIn('testing package testpkg version 1\n', err)

    def test_dev_stdouterr_access(self):
        '''write to /dev/stdout and /dev/stderr in a test'''

        p = self.build_src('Tests: pass\nDepends:\nRestrictions: allow-stderr',
                           {'pass': '#!/bin/sh\necho I am fine >/dev/stdout\n'
                            'echo SomeDebug >/dev/stderr\n'})

        (code, out, err) = self.runtest(['--no-built-binaries', '--built-tree=' + p])
        self.assertEqual(code, 0, err)
        self.assertRegex(out, 'pass\s+PASS', out)

        # should show test stdout/err
        self.assertIn('I am fine\n', out)
        # should show test stderr
        self.assertIn('\nSomeDebug\n', err)

    def test_summary(self):
        '''--summary option'''

        p = self.build_src('Tests: good bad\nDepends:\n',
                           {'good': '#!/bin/sh\necho happy\n',
                            'bad': '#!/bin/sh\nexit 1'})

        summary = os.path.join(self.workdir, 'summary.log')

        (code, out, err) = self.runtest(['--no-built-binaries',
                                         '--unbuilt-tree=' + p,
                                         '--summary=' + summary])

        # test results
        self.assertEqual(code, 4, err)
        self.assertRegex(out, 'good\s+PASS', out)
        self.assertRegex(out, 'bad\s+FAIL non-zero exit status 1', out)

        # check summary file
        with open(summary) as f:
            self.assertEqual(f.read(), '''good                 PASS
bad                  FAIL non-zero exit status 1
''')

    def test_timeout(self):
        '''handling test timeout'''

        p = self.build_dsc('Tests: totest\nDepends:\n',
                           {'totest': '#!/bin/sh\necho start\nsleep 10\necho after_sleep\n'})

        (code, out, err) = self.runtest(['--no-built-binaries', '--timeout-test=3', p])
        # test should time out
        self.assertEqual(code, 4, err)
        self.assertIn("timed out on command", err)
        self.assertRegex(out, 'totest\s+FAIL timed out', out)

        # should show test stdout
        self.assertIn('start', out)
        # but not let the test finish
        self.assertNotIn('after_sleep', out)

    def test_timeout_no_output(self):
        '''handling test timeout for test without any output'''

        # use pid-specific name here to avoid clashes during multiple parallel runs
        procname = 'mysleep%i' % os.getpid()

        p = self.build_dsc('Tests: totest\nDepends:\n',
                           {'totest': '#!/bin/bash\nexec -a %s sleep 15\n' % procname})

        time_start = time.time()
        (code, out, err) = self.runtest(['--no-built-binaries', '--timeout-test=3', p])
        duration = time.time() - time_start

        # test should time out
        self.assertEqual(code, 4, err)
        self.assertIn("timed out on command", err)
        self.assertRegex(out, 'totest\s+FAIL timed out', out)
        self.assertLess(duration, 10, err)

        # should not leave test process running
        self.assertEqual(subprocess.getoutput('pidof ' + procname), '')

    def test_timeout_long_test(self):
        '''long-running test with custom timeouts

        This verifies that the right timeout is being used for tests.
        '''
        p = self.build_dsc('Tests: p\nDepends:\nRestrictions: build-needed',
                           {'p': '#!/bin/sh\necho start\nsleep 5\n./test_built\n'})

        (code, out, err) = self.runtest(['-B', '--timeout-test=6',
                                         '--timeout-build=20', p])
        # test should not time out
        self.assertEqual(code, 0, err)
        self.assertNotIn('timed out', err)

        # should show test stdout
        self.assertIn('start\n', out)
        self.assertIn('built script OK\n', out)

    def test_timeout_long_build(self):
        '''long-running build with custom timeouts

        This verifies that the right timeout is being used for builds.
        '''
        p = self.build_src('Tests: p\nDepends:\nRestrictions: build-needed',
                           {'p': '#!/bin/sh\necho start\n\n./test_built\n'})

        # make build take 4s
        subprocess.check_call(['sed', '-i', '/^build:/ s/$/\\n\\tsleep 4/',
                               os.path.join(p, 'Makefile')])

        (code, out, err) = self.runtest(['--timeout-test=1', '--timeout-build=30',
                                         '-B', '--unbuilt-tree', p])
        # should build package
        self.assertIn('dh build', err)

        # test should not time out
        self.assertEqual(code, 0, err)
        self.assertNotIn('timed out', err)

        # should show test stdout
        self.assertIn('start\n', out)
        self.assertIn('built script OK\n', out)

    def test_timeout_long_build_fail(self):
        '''long-running build times out

        This verifies that the right timeout is being used for builds.
        '''
        p = self.build_src('Tests: p\nDepends:\nRestrictions: build-needed',
                           {'p': '#!/bin/sh\necho start\n\n./test_built\n'})

        # make build take 4s
        subprocess.check_call(['sed', '-i', '/^build:/ s/$/\\n\\tsleep 4/',
                               os.path.join(p, 'Makefile')])

        (code, out, err) = self.runtest(['--timeout-test=1', '--timeout-build=6',
                                         '-B', '--unbuilt-tree', p])
        # should start building package
        self.assertIn('dh build', err)

        # build should time out
        self.assertEqual(code, 16, err)
        self.assertIn('timed out', err)

        # should not start tests
        self.assertNotIn('start\n', out)

    def test_logfile_success(self):
        '''--log-file option, success'''

        p = self.build_src('Tests: pass\nDepends: coreutils\n',
                           {'pass': '#!/bin/sh\necho I am fine\n'})

        logfile = os.path.join(self.workdir, 'test.log')
        (code, out, err) = self.runtest(['-d', '--no-built-binaries',
                                         '--unbuilt-tree=' + p,
                                         '--log-file=' + logfile])
        # test should succeed
        self.assertEqual(code, 0, err)

        with open(logfile) as f:
            log = f.read()
        self.assertIn('coreutils', err)
        self.assertIn('coreutils', log)
        self.assertRegex(out, 'pass\s+PASS')
        self.assertRegex(log, 'pass\s+PASS')

        self.assertIn('processing dependency coreutils', err)
        self.assertIn('processing dependency coreutils', log)
        # should show test stdout
        self.assertRegex(out, '(^|\n)I am fine\n')
        self.assertRegex(log, '(^|\n)I am fine\n')
        # should not have any test stderr
        self.assertNotIn(' stderr ', err)
        self.assertNotIn(' stderr ', log)

        # should not build package
        self.assertNotIn('dh build', err)

    def test_logfile_failure(self):
        '''--log-file option, failure'''

        p = self.build_src('Tests: nz\nDepends: coreutils\nRestrictions: build-needed\n',
                           {'nz': '#!/bin/sh\n./test_built\necho I am sick >&2\nexit 7'})

        logfile = os.path.join(self.workdir, 'test.log')
        (code, out, err) = self.runtest(['-d', '--no-built-binaries',
                                         '--unbuilt-tree=' + p,
                                         '--log-file=' + logfile])
        with open(logfile) as f:
            log = f.read()

        # test should fail
        self.assertEqual(code, 4)
        self.assertRegex(out, 'nz\s+FAIL non-zero exit status 7')
        self.assertRegex(log, 'nz\s+FAIL non-zero exit status 7')

        self.assertIn('processing dependency coreutils', err)
        self.assertIn('processing dependency coreutils', log)

        # should build package
        self.assertIn('dh build', err)
        self.assertIn('dh build', log)

        # should show test stdout
        self.assertRegex(out, '(^|\n)built script OK\n')
        self.assertRegex(log, '(^|\n)built script OK\n')
        # should show test stderr
        self.assertRegex(err, 'stderr [ -]+\nI am sick\n')
        self.assertRegex(log, 'stderr [ -]+\nI am sick\n')

    def test_unicode(self):
        '''Unicode test output'''

        p = self.build_src('Tests: se\nDepends:\n',
                           {'se': '#!/bin/sh\necho ‘a♩’; echo fancy ‴ʎɔuɐɟ″!>&2'})

        summary = os.path.join(self.workdir, 'summary.log')

        (code, out, err) = self.runtest(['--no-built-binaries',
                                         '--unbuilt-tree=' + p,
                                         '--summary=' + summary])
        self.assertEqual(code, 4, err)
        self.assertRegex(out, 'se\s+FAIL stderr: fancy ‴ʎɔuɐɟ″!\n')

        # should show test stdout/stderr
        self.assertIn('‘a♩’\n', out)
        self.assertRegex(err, 'stderr [ -]+\nfancy ‴ʎɔuɐɟ″!\n')

        with open(summary, encoding='UTF-8') as f:
            self.assertEqual(f.read(), 'se                   FAIL stderr: fancy ‴ʎɔuɐɟ″!\n')

    def test_no_tests_dir(self):
        '''package without debian/tests/'''

        p = self.build_src(None, None)
        (code, out, err) = self.runtest(['--unbuilt-tree=' + p])
        self.assertEqual(code, 8, err)
        self.assertRegex(out, 'SKIP no tests in this package', out)

    @unittest.skipIf(os.getuid() == 0, 'needs to run as user')
    @unittest.skipIf(os.path.exists('/usr/bin/dotty'),
                     'needs graphviz uninstalled')
    @unittest.skipUnless(have_apt, 'needs apt-get download working')
    def test_tmp_install(self):
        '''temp dir unpack of test dependencies'''

        p = self.build_src('Tests: t\nDepends: graphviz, gir1.2-json-1.0 (>= 0.14), python3-gi, a|b,',
                           {'t': '#!/bin/sh\ndotty -V 2>&1 || true\n'
                                 'python3 -c "import gi; gi.require_version(\'Json\', \'1.0\'); from gi.repository import Json; print(Json)"'})

        (code, out, err) = self.runtest(['-B', p + '//'])
        self.assertEqual(code, 0, err)
        self.assertRegex(out, 't\s+PASS', out)

        # should show test stdout
        self.assertIn('dotty version ', out)
        try:
            from gi.repository import Json
            Json  # pyflakes
            # already installed on the system
            self.assertRegex(out, '(Dynamic|Introspection)Module.*Json.* from .*/usr/lib/.*girepository')
        except ImportError:
            # should use from local unpack dir
            self.assertRegex(out, '(Dynamic|Introspection)Module.*Json.* from .*tmp.*/deps/usr/lib')
        # no stderr
        self.assertNotIn(' stderr ', err)

        # downloads dependencies
        self.assertIn('libcgraph', err)
        self.assertIn('libcgraph', err)

        # warn about restricted functionality
        self.assertRegex(err, 'WARNING.*cannot be handled.* a | b')
        self.assertRegex(err, 'WARNING.*will only work for some packages')

    @unittest.skipIf(os.getuid() == 0, 'needs to run as user')
    @unittest.skipIf(os.path.exists('/usr/share/perl5/Test/Requires.pm'),
                     'needs libtest-requires-perl uninstalled')
    @unittest.skipIf(os.path.exists('/usr/share/doc/libconvert-uulib-perl'),
                     'needs libconvert-uulib-perl uninstalled')
    @unittest.skipUnless(have_apt, 'needs apt-get download working')
    def test_tmp_install_perl(self):
        '''temp dir unpack of Perl dependencies'''

        # one arch: all, one binary
        p = self.build_src('Tests: t\nDepends: libtest-requires-perl, libconvert-uulib-perl',
                           {'t': '#!/usr/bin/perl\nuse Test::Requires;\nuse Convert::UUlib;\n'})

        (code, out, err) = self.runtest(['-B', p + '//'])
        self.assertEqual(code, 0, err)
        self.assertRegex(out, 't\s+PASS', out)

    @unittest.skipIf(os.getuid() == 0, 'needs to run as user')
    @unittest.skipIf(os.path.exists('/usr/lib/python3/dist-packages/wand/'),
                     'needs python3-wand uninstalled')
    @unittest.skipUnless(have_apt, 'needs apt-get download working')
    @unittest.skipUnless(subprocess.call(['apt-cache', 'show', 'python3-wand'],
                                         stdout=subprocess.PIPE,
                                         stderr=subprocess.STDOUT) == 0,
                         'needs python3-wand package')
    def test_tmp_install_imagemagick(self):
        '''temp dir unpack of imagemagick dependencies'''

        p = self.build_src('Tests: t\nDepends: python3-wand',
                           {'t': '#!/usr/bin/env python3\nfrom wand.image import Image\n'})

        (code, out, err) = self.runtest(['-d', '-B', p + '//'])
        self.assertEqual(code, 0, err)
        self.assertRegex(out, 't\s+PASS', out)

        # no stderr
        self.assertNotIn(' stderr ', err)

    @unittest.skipIf(os.getuid() == 0, 'needs to run as user')
    def test_tmp_install_nonexisting_pkg(self):
        '''temp dir unpack of nonexisting test dependency'''

        p = self.build_src('Tests: t\nDepends: nosuchpackage',
                           {'t': '#!/bin/sh\nfalse'})

        (code, out, err) = self.runtest(['-B', p + '//'])
        self.assertEqual(code, 12, err)

        self.assertRegex(err, 'E: .*nosuchpackage')
        self.assertIn('Test dependencies are unsatisfiable', err)

    @unittest.skipIf(os.getuid() == 0, 'needs to run as user')
    @unittest.skipIf(os.path.exists('/usr/bin/dotty'),
                     'needs graphviz uninstalled')
    def test_tmp_install_no_such_version(self):
        '''temp dir unpack of test dependency with unsatisfiable version'''

        p = self.build_src('Tests: t\nDepends: graphviz (>= 4:999)',
                           {'t': '#!/bin/sh\nfalse'})

        (code, out, err) = self.runtest(['-B', p + '//'])
        self.assertEqual(code, 12, err)

        self.assertIn('test dependency graphviz (>= 4:999) is unsatisfiable: available version ', err)

    def test_test_command(self):
        '''Test-Command: instead of Tests:'''

        p = self.build_src('Test-Command: echo "Some Stdout"\nDepends:\n\n'
                           'Test-Command: echo "Some StdErr" >&2; sleep 0.5; echo done\nDepends:\n'
                           'Restrictions: allow-stderr\n\n'
                           'Test-Command: echo "More Stderr" >&2; echo hello > $AUTOPKGTEST_ARTIFACTS/world.txt\nDepends:\n',
                           {})

        outdir = os.path.join(self.workdir, 'out')
        os.mkdir(outdir)

        (code, out, err) = self.runtest(['-B', '--unbuilt-tree=' + p,
                                         '-o', outdir, '-d'])
        # two pass, one fails
        self.assertEqual(code, 4, err)
        self.assertRegex(out, 'done\n')
        self.assertRegex(out, 'command1\s+PASS')
        self.assertRegex(out, 'command2\s+PASS')
        self.assertRegex(out, 'command3\s+FAIL stderr: More Stderr')
        self.assertRegex(err, 'stderr [ -]+\nMore Stderr\n')
        # shows commands
        self.assertIn('test command1: echo "Some Stdout"\n', err)
        self.assertIn('test command2: echo "Some StdErr" >&2; sleep 0.5; echo done\n', err)
        self.assertIn('test command3: echo "More Stderr" >&2; echo hello > $AUTOPKGTEST_ARTIFACTS/world.txt\n', err)

        # check artifacts
        with open(os.path.join(outdir, 'command1-stdout')) as f:
            self.assertEqual(f.read(), 'Some Stdout\n')
        with open(os.path.join(outdir, 'command2-stdout')) as f:
            self.assertEqual(f.read(), 'done\n')
        with open(os.path.join(outdir, 'command2-stderr')) as f:
            self.assertEqual(f.read(), 'Some StdErr\n')
        with open(os.path.join(outdir, 'command3-stderr')) as f:
            self.assertEqual(f.read(), 'More Stderr\n')
        with open(os.path.join(outdir, 'artifacts', 'world.txt')) as f:
            self.assertEqual(f.read(), 'hello\n')

    @unittest.skipUnless(have_apt_src, 'this test needs local deb-src apt sources')
    def test_apt_source_nonexisting(self):
        '''apt-source for nonexisting package'''

        (code, out, err) = self.runtest(['no.such-package'])

        self.assertEqual(code, 12, err)
        self.assertRegex(err, 'no.such-package')
        self.assertNotIn('PASS', out)
        self.assertNotIn('[----', out)

    def test_undeletable_files(self):
        '''source tree has undeletable files'''

        p = self.build_src('Tests: t1 t2\nDepends:\n',
                           {'t1': '#!/bin/sh\necho eins\n',
                            't2': '#!/bin/sh\necho zwei\n'})
        os.mkdir(os.path.join(p, 'data'))
        with open(os.path.join(p, 'data', 'data.txt'), 'w') as f:
            f.write('data\n')
        os.chmod(os.path.join(p, 'data'), 0o555)

        (code, out, err) = self.runtest(['--built-tree=' + p])
        # test should succeed
        self.assertEqual(code, 0, err)
        self.assertRegex(out, 't1\s+PASS', out)
        self.assertRegex(out, 't2\s+PASS', out)

        # should show test stdout
        self.assertRegex(out, '^eins\n')
        self.assertRegex(out, '\nzwei\n')
        # should not have any test stderr
        self.assertNotIn(' stderr ', err)

        # chown it back so that we can clean up
        os.chmod(os.path.join(p, 'data'), 0o755)

    def test_signal(self):
        '''tests keep default signal handler'''

        p = self.build_src('Tests: int\nDepends:',
                           {'int': '''#!/usr/bin/perl -wl
$| = 1; # unbuffer output
my $pid = fork;
if ($pid) { # parent
    sleep 1;
    print "P: killing";
    kill INT => $pid;
    print "P: waiting";
    wait;
    print "P: done";
} else { # child
    sleep;
    print "C: survived!";
}
'''})

        (code, out, err) = self.runtest(['-B', '--timeout-test=20', p + '//'])
        self.assertEqual(code, 0, err)
        self.assertRegex(out, 'int\s+PASS')

        self.assertIn('P: done\n', out)
        self.assertNotIn('survived', out)

    def test_tree_debian_tests_only(self):
        '''tree with only debian/tests/'''

        srcdir = os.path.join(self.workdir, 'testpkg')
        testdir = os.path.join(srcdir, 'debian', 'tests')
        os.makedirs(testdir)
        with open(os.path.join(testdir, 'control'), 'w') as f:
            f.write('Test-Command: echo IamFine\nDepends:\n')

        (code, out, err) = self.runtest(['-d', '-B', srcdir + '/'])
        # test should succeed
        self.assertEqual(code, 0, err)
        self.assertRegex(out, 'command1\s+PASS', out)

        # should show test stdout
        self.assertRegex(out, '(^|\n)IamFine\n')
        # should not have any test stderr
        self.assertNotIn(' stderr ', err)

    def test_testname(self):
        '''Run only one specified test'''

        p = self.build_src('Tests: one two three\nDepends:',
                           {'one': '#!/bin/sh\necho 1_ONE',
                            'two': '#!/bin/sh\necho 2_TWO',
                            'three': '#!/bin/sh\necho 3_THREE'})

        (code, out, err) = self.runtest(['-B', '--testname', 'two',
                                         '--unbuilt-tree=' + p])

        self.assertEqual(code, 0, err)
        self.assertRegex(out, 'two\s+PASS', out)
        self.assertNotIn('one', out)
        self.assertNotIn('three', out)

        self.assertIn('2_TWO', out)
        self.assertNotIn('1_ONE', out)
        self.assertNotIn('3_THREE', out)

    def test_testname_noexist(self):
        '''Run only one specified test which does not exist'''

        p = self.build_src('Tests: one two three\nDepends:',
                           {'one': '#!/bin/sh\necho 1_ONE',
                            'two': '#!/bin/sh\necho 2_TWO',
                            'three': '#!/bin/sh\necho 3_THREE'})

        (code, out, err) = self.runtest(['-B', '--testname', 'four',
                                         '--unbuilt-tree=' + p])

        self.assertEqual(code, 8, err)
        for w in ['one', 'two', 'three']:
            self.assertNotIn(w, out)

    def test_command_not_found(self):
        '''command-not-found failure is test failure'''

        p = self.build_src('Tests: f\nDepends:\n', {'f': '#!/bin/sh -e\nno_such_command'})

        (code, out, err) = self.runtest(['-d', '-B', p + '//'])
        self.assertEqual(code, 4, err)
        self.assertRegex(out, 'f\s+FAIL non-zero exit status 127', out)
        # this isn't a failure of the auxverb
        self.assertNotIn('testbed auxverb failed with', err)

    def test_test_not_found(self):
        '''Tests: specifies a nonexisting test'''

        p = self.build_src('Tests: foo\nDepends:\n', {'bar': '#!/bin/sh -e\nno_such_command'})

        (code, out, err) = self.runtest(['-B', p + '//'])
        self.assertEqual(code, 12, err)
        self.assertRegex(out, 'blame: .*/testpkg', out)
        self.assertIn('badpkg: debian/tests/foo does not exist', out)
        self.assertIn('debian/tests/foo does not exist', err)
        self.assertNotIn('testbed auxverb failed with', err)
        self.assertNotIn('bar', out)
        self.assertNotIn('bar', err)

    def test_invalid_depends(self):
        '''Depends: has invalid syntax'''

        p = self.build_src('Tests: foo\nDepends: foo bar baz\n', {'foo': '#!/bin/sh -e\nfalse'})

        outdir = os.path.join(p, 'out')
        (code, out, err) = self.runtest(['-B', p + '//', '--output-dir=' + outdir])
        self.assertEqual(code, 12, err)
        self.assertRegex(out, 'blame: .*/testpkg//', out)
        self.assertRegex(out, 'badpkg:.*test foo.*Depends field contains an invalid dependency.*foo bar baz')
        self.assertRegex(err, 'erroneous package:.*test foo.*Depends field contains an invalid dependency.*foo bar baz')

        with open(os.path.join(outdir, 'testinfo.json')) as f:
            info = json.load(f)
        self.assertIn(os.uname().sysname, info['kernel_version'])
        self.assertEqual(info['virt_server'], 'autopkgtest-virt-null')

    def test_versioned_depends(self):
        '''Depends: with versions'''

        # ensure that build profile sedding does not affect versioned dependencies
        p = self.build_src('Test-Command: true\nDepends: base-files, coreutils (<< 99:1), bash (>= 0.1), coreutils (>=0.1~)\n', {})

        (code, out, err) = self.runtest(['-d', '-B', p + '//'])
        self.assertEqual(code, 0, err)
        self.assertIn('install-deps: architecture resolved: base-files, coreutils (<< 99:1), bash (>= 0.1), coreutils (>= 0.1~)', err)

    def test_env_passing(self):
        '''Pass environment variable to test'''

        p = self.build_src('Tests: pass\nDepends: coreutils\n',
                           {'pass': '#!/bin/sh -eu\necho Hey ${NAME}! favourite color: ${FAV_COLOR}.\n'})

        (code, out, err) = self.runtest(
            ['-d', '--env=FAV_COLOR=blue', '--env', 'NAME=Joe', '-B', '--unbuilt-tree=' + p])
        # test should succeed
        self.assertEqual(code, 0, err)
        self.assertRegex(out, 'pass\s+PASS', out)

        self.assertIn('processing dependency coreutils', err)
        # should show test stdout
        self.assertRegex(out, '(^|\n)Hey Joe! favourite color: blue.\n')
        # should not have any test stderr
        self.assertNotIn(' stderr ', err)

    @unittest.skipUnless(have_git, 'git not installed')
    def test_git_source(self):
        '''Check out source from git (default branch)'''

        p = self.build_src('Tests: pass\nDepends:\n',
                           {'pass': '#!/bin/sh\necho I am fine\n'})

        subprocess.check_call(['git', 'config', '--global', 'user.email', 'autopkgtest@example.com'])
        subprocess.check_call(['git', 'config', '--global', 'user.name', 'Autopkgtest'])
        subprocess.check_call(['git', 'init', '--quiet'], cwd=p)
        subprocess.check_call(['git', 'add', '.'], cwd=p)
        subprocess.check_call(['git', 'commit', '--quiet', '-mfirst'], cwd=p)

        (code, out, err) = self.runtest(['-B', '--git-source', 'file://' + p])
        # test should succeed
        self.assertEqual(code, 0, err)
        self.assertRegex(out, 'pass\s+PASS', out)

        # should show test stdout
        self.assertRegex(out, '(^|\n)I am fine\n')
        # should show summary
        self.assertRegex(err, '@@@ summary\npass\s+PASS')

    @unittest.skipUnless(have_git, 'git not installed')
    def test_git_source_branch(self):
        '''Check out source from git (specified branch)'''

        p = self.build_src('Tests: pass\nDepends:\n',
                           {'pass': '#!/bin/sh -e\nls -l branch.txt; echo I am fine\n'})
        subprocess.check_call(['git', 'config', '--global', 'user.email', 'autopkgtest@example.com'])
        subprocess.check_call(['git', 'config', '--global', 'user.name', 'Autopkgtest'])
        subprocess.check_call(['git', 'init', '--quiet'], cwd=p)
        subprocess.check_call(['git', 'add', '.'], cwd=p)
        subprocess.check_call(['git', 'commit', '--quiet', '-mfirst'], cwd=p)
        subprocess.check_call(['git', 'branch', 'wip'], cwd=p)
        subprocess.check_call(['git', 'checkout', '--quiet', 'wip'], cwd=p)
        with open(os.path.join(p, 'branch.txt'), 'w') as f:
            f.write('hello')
        subprocess.check_call(['git', 'add', 'branch.txt'], cwd=p)
        subprocess.check_call(['git', 'commit', '--quiet', '-mdobranch'], cwd=p)
        subprocess.check_call(['git', 'checkout', '--quiet', 'master'], cwd=p)

        (code, out, err) = self.runtest(['-B', '--git-source', 'file://%s#wip' % p])
        # test should succeed
        self.assertEqual(code, 0, err)
        self.assertRegex(out, 'pass\s+PASS', out)

        # should show test stdout
        self.assertRegex(out, '(^|\n)I am fine\n')
        # should show summary
        self.assertRegex(err, '@@@ summary\npass\s+PASS')

    def test_tmpdir_env(self):
        '''Respects $TMPDIR'''

        p = self.build_src('Tests: p\nDepends:\n',
                           {'p': '#!/bin/sh\necho $AUTOPKGTEST_TMP\n[ "${AUTOPKGTEST_TMP#/var/tmp}" != "$AUTOPKGTEST_TMP" ]'})

        env = os.environ.copy()
        env['TMPDIR'] = '/var/tmp'
        (code, out, err) = self.runtest(['--no-built-binaries',
                                         '--unbuilt-tree=' + p],
                                        env=env)

        # test results
        self.assertEqual(code, 0, err)
        self.assertRegex(out, '/var/tmp/autopkgtest.*')

    def test_build_parallel(self):
        '''--build-parallel option'''

        p = self.build_src('Tests: pass\nDepends:\nRestrictions: build-needed\n',
                           {'pass': '#!/bin/sh -e\necho DBO=$DEB_BUILD_OPTIONS\n'})

        (code, out, err) = self.runtest(['--build-parallel=17', '--no-built-binaries', '--unbuilt-tree=' + p])
        # test should succeed
        self.assertEqual(code, 0, out + err)
        self.assertRegex(out, 'pass\s+PASS', out)

        # should build package
        self.assertIn('dh build', err)

        # should have expected parallel= option
        self.assertIn('DBO=parallel=17\n', out)
        # should not have any test stderr
        self.assertNotIn(' stderr ', err)

    def test_multiple_actions(self):
        '''multiple actions'''

        p = self.build_src('Tests: p\nDepends:\n',
                           {'p': '#!/bin/sh\necho I am fine\n'})
        q = self.build_src('Tests: q\nDepends:\n',
                           {'q': '#!/bin/sh\necho I am good\n'},
                           pkgname='testpkg2')

        (code, out, err) = self.runtest(['-d', p + '/', q + '/'])
        # tests should succeed
        self.assertEqual(code, 0, err)
        self.assertRegex(out, 'p\s+PASS', out)
        self.assertRegex(out, 'q\s+PASS', out)

        self.assertRegex(out, '(^|\n)I am fine\n')
        self.assertRegex(out, '(^|\n)I am good\n')
        self.assertRegex(err, 'testbed running kernel: Linux \d')
        # should not have any test stderr
        self.assertNotIn(' stderr ', err)


@unittest.skipIf(os.getuid() > 0,
                 'NullRunnerRoot tests need to run as root')
class NullRunnerRoot(AdtTestCase):
    def __init__(self, *args, **kwargs):
        super(NullRunnerRoot, self).__init__(['null'], *args, **kwargs)

    def test_tmpdir_for_other_users(self):
        '''$TMPDIR is accessible to non-root users'''

        prev_mask = os.umask(0o077)
        self.addCleanup(os.umask, prev_mask)
        p = self.build_src('Tests: t\nDepends: coreutils\nRestrictions: needs-root\n',
                           {'t': '''#!/bin/sh -e
echo hello > ${TMPDIR:=/tmp}/rootowned.txt
su -s /bin/sh -c "echo hello > $TMPDIR/world.txt" nobody
if su -s /bin/sh -c "echo break > $TMPDIR/rootowned.txt" nobody 2>/dev/null; then
    exit 1
fi
rm $TMPDIR/rootowned.txt $TMPDIR/world.txt
'''})

        (code, out, err) = self.runtest(['--no-built-binaries', '--built-tree=' + p])
        # test should succeed
        self.assertRegex(out, 't\s+PASS', out)
        self.assertEqual(code, 0, err)

        # should not have any test stderr
        self.assertNotIn(' stderr ', err)

    def test_user(self):
        '''--user option'''

        p = self.build_src('Tests: t\nDepends:\n',
                           {'t': '''#!/bin/sh -e
                                    echo world > ${TMPDIR:=/tmp}/hello.txt
                                    cat $TMPDIR/hello.txt
                                    rm $TMPDIR/hello.txt
                                    stat -c %U .
                                    stat -c %U debian
                                    stat -c %U Makefile
                                    echo "USER: $USER"
                                    whoami'''})

        (code, out, err) = self.runtest(['--no-built-binaries',
                                         '--unbuilt-tree=' + p,
                                         '--user=nobody'])
        self.assertEqual(code, 0, err)

        # test should succeed
        self.assertRegex(out, 't\s+PASS', out)

        # has output from cat and whoami
        self.assertIn('world\nnobody\nnobody\nnobody\nUSER: nobody\nnobody\n', out)

    def test_default_locale(self):
        '''tests have default locale C.UTF-8'''

        p = self.build_src('Tests: r\nDepends:\nRestrictions: needs-root\n\n'
                           'Tests: u\nDepends:\n',
                           {'r': '#!/bin/sh -e\necho root $LANG\nlocale',
                            'u': '#!/bin/sh -e\necho user $LANG\nlocale'})

        # invalid LC_* should not be copied, only $LANG set
        env = os.environ.copy()
        env['LC_PAPER'] = 'fo_BAR.UTF-9'
        (code, out, err) = self.runtest(['-B', '--unbuilt-tree=' + p], env=env)
        self.assertEqual(code, 0, err)

        # test should succeed
        self.assertRegex(out, 'r\s+PASS', out)
        self.assertRegex(out, 'u\s+PASS', out)

        # has expected locales
        self.assertIn('root C.UTF-8\n', out)
        self.assertIn('user C.UTF-8\n', out)

    def test_specify_locale(self):
        '''tests have specified locale'''

        p = self.build_src('Tests: r\nDepends:\nRestrictions: needs-root\n\n'
                           'Tests: u\nDepends:\n',
                           {'r': '#!/bin/sh -e\necho root $LANG',
                            'u': '#!/bin/sh -e\necho user $LANG'})

        (code, out, err) = self.runtest(['-B', '--unbuilt-tree=' + p,
                                         '--set-lang=ab_CD.UTF-8'])
        self.assertEqual(code, 0, err)

        # test should succeed
        self.assertRegex(out, 'r\s+PASS', out)
        self.assertRegex(out, 'u\s+PASS', out)

        # has expected locales
        self.assertIn('root ab_CD.UTF-8\n', out)
        self.assertIn('user ab_CD.UTF-8\n', out)


@unittest.skipIf('cowdancer' in os.environ.get('LD_PRELOAD', ''),
                 'chroot tests do not work under cowdancer')
@unittest.skipIf(os.getuid() > 0,
                 'chroot runner needs to run as root')
class ChrootRunner(AdtTestCase, DebTestsAll, DebTestsFailureModes):
    def __init__(self, *args, **kwargs):
        super(ChrootRunner, self).__init__(['chroot', '/uninited'], *args, **kwargs)

    def setUp(self):
        super(ChrootRunner, self).setUp()

        def install_file(path):
            destdir = self.chroot + '/' + os.path.dirname(path)
            if not os.path.exists(destdir):
                os.makedirs(destdir)
            if os.path.isfile(path):
                shutil.copy(path, destdir)
            else:
                subprocess.check_call(['cp', '-a', path, destdir])

        def install_elf(path):
            install_file(path)
            out = subprocess.check_output(['ldd', path], universal_newlines=True)
            libs = set()
            for lib in re.finditer('/[^ ]+', out):
                libs.add(lib.group(0))
            for lib in libs:
                install_file(lib)

        # build a mini-chroot
        self.chroot = os.path.join(self.workdir, 'chroot')

        install_file('/usr/bin/which')
        install_elf('/bin/bash')
        install_elf('/bin/sh')
        install_elf('/bin/ls')
        install_elf('/bin/cat')
        install_elf('/bin/rm')
        install_elf('/bin/cp')
        install_elf('/bin/mkdir')
        install_elf('/bin/chmod')
        install_elf('/bin/chown')
        install_elf('/bin/mktemp')
        install_elf('/bin/tar')
        install_elf('/bin/sleep')
        install_elf('/bin/sed')
        install_elf('/bin/readlink')
        install_elf('/bin/grep')
        install_elf('/bin/uname')
        install_elf('/usr/bin/test')
        install_elf('/usr/bin/awk')
        install_elf('/usr/bin/head')
        install_elf('/usr/bin/env')
        install_elf('/usr/bin/tee')
        install_elf('/usr/bin/touch')
        install_elf('/usr/bin/nproc')
        install_file('/usr/lib/locale/C.UTF-8')

        # necessary bind mounts
        dev_dir = os.path.join(self.chroot, 'dev')
        os.mkdir(dev_dir)
        subprocess.check_call(['mount', '-o', 'bind', '/dev', dev_dir])
        proc_dir = os.path.join(self.chroot, 'proc')
        os.mkdir(proc_dir)
        subprocess.check_call(['mount', '-o', 'bind', '/proc', proc_dir])

        # some fakes
        for cmd in ['dpkg', 'dpkg-query', 'dpkg-source', 'apt-get', 'apt-key', 'apt-cache']:
            p = os.path.join(self.chroot, 'usr', 'bin', cmd)
            with open(p, 'w') as f:
                f.write('#!/bin/sh\n')
                if cmd == 'dpkg':
                    f.write('if [ "$1" = "--print-architecture" ]; then echo powerpc; exit; fi\n')
                f.write('echo "fake-%s: $@"\n' % cmd)
                if cmd == 'apt-get':
                    f.write('if [ "$1" = source ]; then cp -r /aptget-src $4-1; fi\n')
                if cmd == 'apt-cache':
                    f.write('if [ "$1" = showsrc ]; then printf "Package-List:\\n $3 deb utils optional arch=any\\nFormat: 1.0\\n"; fi\n')
            os.chmod(p, 0o755)
        with open(os.path.join(self.chroot, 'bin', 'su'), 'w') as f:
            f.write('#!/bin/sh\nshift 4\nexec bash -c "$@"\n')
            os.fchmod(f.fileno(), 0o755)
        with open(os.path.join(self.chroot, 'usr', 'bin', 'id'), 'w') as f:
            f.write('#!/bin/sh\necho root\n')
            os.fchmod(f.fileno(), 0o755)

        p = os.path.join(self.chroot, 'tmp')
        os.mkdir(p)
        os.chmod(p, 0o177)
        os.makedirs(os.path.join(self.chroot, 'etc', 'apt', 'sources.list.d'))

        self.virt_args[-1] = self.chroot

    def tearDown(self):
        # these sometimes fail on EBUSY
        subprocess.call('for p in %(c)s/dev %(c)s/proc; do while mountpoint $p >/dev/null; do umount $p; sleep 0.1; done; done' %
                        {'c': self.chroot}, shell=True)
        subprocess.call('for i in `seq 10`; do rm -rf %s && break; sleep 0.5; done' % self.chroot, shell=True)

    def test_fancy_deps(self):
        '''wrapped and versioned test dependencies'''

        p = self.build_src('''Tests: pass
Depends: fancypkg,
         coreutils | vanilla (>= 10),
         chocolate,
Restrictions: needs-root
''', {'pass': '#!/bin/sh\ntrue'})

        (code, out, err) = self.runtest(['-d', '-B', '--unbuilt-tree=' + p])
        self.assertEqual(code, 0, err)
        self.assertRegex(out, 'pass\s+PASS')

        self.assertIn('processing dependency fancypkg\n', err)
        self.assertIn('processing dependency coreutils | vanilla (>= 10)\n', err)
        self.assertIn('processing dependency chocolate\n', err)

    def test_build_deps(self):
        '''test depends on build dependencies'''

        p = self.build_src('''Tests: pass
Depends: @, testdep1, @builddeps@, testdep2,
# blabla ☺
 testdep3
Restrictions: needs-root
''', {'pass': '#!/bin/sh\ntrue'})

        # add extra build dependencies to testpkg
        subprocess.check_call(['sed', '-i', '/^Build-Depends:/ s/:.*/: bdep1 , bdep2,\\n bdep3, # moo\\n#comment\\n bdep4\\n'
                                            'Build-Depends-Indep: bdep5/',
                               os.path.join(p, 'debian', 'control')])

        # run this under C locale to test that UTF-8 debian/control is still
        # handled correctly
        (code, out, err) = self.runtest(['-d', '-B', '--unbuilt-tree=' + p],
                                        env={'LC_ALL': 'C', 'PATH': os.environ['PATH']})
        self.assertEqual(code, 0, err)
        self.assertRegex(out, 'pass\s+PASS')

        self.assertIn('synthesised dependency testpkg (>= 0~)\n', err)
        self.assertIn('processing dependency testdep1\n', err)
        self.assertIn('synthesised dependency bdep1\n', err)
        self.assertIn('synthesised dependency bdep2\n', err)
        self.assertIn('synthesised dependency bdep3\n', err)
        self.assertIn('synthesised dependency bdep4\n', err)
        self.assertIn('synthesised dependency bdep5\n', err)
        self.assertIn('synthesised dependency build-essential\n', err)
        self.assertIn('processing dependency testdep2\n', err)

    def test_build_deps_profiles(self):
        '''test depends on build dependencies with build profiles'''

        p = self.build_src('''Tests: pass
Depends: @builddeps@
Restrictions: needs-root
''', {'pass': '#!/bin/sh\ntrue'})

        # add extra build dependencies to testpkg
        subprocess.check_call(['sed', '-i', '/^Build-Depends:/ s/:.*/: bdepyes <!nocheck>, bdepno <stage1> <cross>/',
                               os.path.join(p, 'debian', 'control')])

        (code, out, err) = self.runtest(['-d', '-B', '--unbuilt-tree=' + p])
        self.assertEqual(code, 0, err)
        self.assertRegex(out, 'pass\s+PASS')

        self.assertIn('synthesised dependency bdepyes\n', err)
        self.assertIn('synthesised dependency build-essential\n', err)

        dpkg_deps_ver = subprocess.check_output(['perl', '-MDpkg::Deps', '-e', 'print $Dpkg::Deps::VERSION'],
                                                universal_newlines=True)
        if dpkg_deps_ver >= '1.04':
            self.assertNotIn('bdepno', err)

    @unittest.skip('chroot runner tests do not support building')
    def test_tree_build_needed_success(self):
        pass

    def test_artifacts(self):
        '''tests producing additional artifacts'''

        p = self.build_src('Tests: a1 a2 a3 a4\nDepends:\nRestrictions: needs-root\n',
                           {'a1': '#!/bin/sh -e\n[ -d "$AUTOPKGTEST_ARTIFACTS" ]\n'
                                  'echo old > $AUTOPKGTEST_ARTIFACTS/health.txt\n',
                            'a2': '#!/bin/sh -e\n[ -d "$AUTOPKGTEST_ARTIFACTS" ]\n'
                                  'echo I am fine > $AUTOPKGTEST_ARTIFACTS/health.txt\n',
                            'a3': '#!/bin/sh -e\n[ -d "$AUTOPKGTEST_ARTIFACTS" ]\n'
                                  'mkdir $AUTOPKGTEST_ARTIFACTS/logs\n'
                                  'echo world > $AUTOPKGTEST_ARTIFACTS/logs/hello.txt\n',
                            'a4': '#!/bin/sh -e\n[ -d "$AUTOPKGTEST_ARTIFACTS" ]\n'
                                  'mkdir $AUTOPKGTEST_ARTIFACTS/logs\n'
                                  'echo 42 > $AUTOPKGTEST_ARTIFACTS/logs/answer.txt\n'})

        outdir = os.path.join(self.workdir, 'out')

        (code, out, err) = self.runtest(['--no-built-binaries',
                                         '--unbuilt-tree=' + p,
                                         '--output-dir=' + outdir])

        # test should succeed
        self.assertEqual(code, 0, err)
        self.assertRegex(out, 'a1\s+PASS', out)
        self.assertRegex(out, 'a2\s+PASS', out)
        self.assertRegex(out, 'a3\s+PASS', out)
        self.assertRegex(out, 'a4\s+PASS', out)

        # check for cruft in output dir
        files = [i for i in os.listdir(outdir)
                 if not fnmatch.fnmatch(i, 'a*-std*') and
                 not fnmatch.fnmatch(i, 'a*-packages')]
        self.assertEqual(set(files),
                         set(['log', 'artifacts', 'testpkg-version',
                              'testbed-packages', 'summary', 'testinfo.json']))

        # check artifact; a2 should overwrite a1's health.txt
        with open(os.path.join(outdir, 'artifacts', 'health.txt')) as f:
            self.assertEqual(f.read(), 'I am fine\n')
        with open(os.path.join(outdir, 'artifacts', 'logs', 'hello.txt')) as f:
            self.assertEqual(f.read(), 'world\n')
        with open(os.path.join(outdir, 'artifacts', 'logs', 'answer.txt')) as f:
            self.assertEqual(f.read(), '42\n')

    def test_slash_in_test_name(self):
        '''test names must not contain /'''

        p = self.build_src('Tests: pass subdir/p\nDepends:\nRestrictions: needs-root',
                           {'pass': '#!/bin/sh\ntrue'})

        (code, out, err) = self.runtest(['--no-built-binaries', '--unbuilt-tree=' + p])
        # invalid test gets skipped
        self.assertEqual(code, 2, err)
        self.assertRegex(out, 'subdir/p\s+SKIP test name may not contain /.*')

        # valid test still gets run
        self.assertFalse(re.match('pass\s+SKIP', out), out)
        self.assertRegex(out, 'pass\s+PASS', out)

    def test_unicode(self):
        '''Unicode test output'''

        p = self.build_src('Tests: se\nDepends:\nRestrictions: needs-root\n',
                           {'se': '#!/bin/sh\necho ‘a♩’; echo fancy ‴ʎɔuɐɟ″!>&2'})

        summary = os.path.join(self.workdir, 'summary.log')

        (code, out, err) = self.runtest(['--no-built-binaries',
                                         '--unbuilt-tree=' + p,
                                         '--summary=' + summary])
        self.assertEqual(code, 4, err)
        self.assertRegex(out, 'se\s+FAIL stderr: fancy ‴ʎɔuɐɟ″!\n')

        # should show test stdout/stderr
        self.assertRegex(out, '‘a♩’\n')
        self.assertRegex(err, '\nfancy ‴ʎɔuɐɟ″!\n')

        with open(summary, encoding='UTF-8') as f:
            self.assertRegex(f.read(), 'se\s+FAIL stderr: fancy ‴ʎɔuɐɟ″!\n')

    def test_apt_source_no_restrictions(self):
        '''apt source, no build, no restrictions'''

        p = self.build_src('Tests: pass\nDepends:\nRestrictions: needs-root\n',
                           {'pass': '#!/bin/sh\necho I am fine\n'})
        # copy that into the chroot where fake apt-get source can find it
        shutil.copytree(p, os.path.join(self.chroot, 'aptget-src'))

        (code, out, err) = self.runtest(['-d', 'testpkg'])
        # test should succeed
        self.assertEqual(code, 0, err)
        self.assertRegex(out, 'pass\s+PASS', out)

        # should show test stdout
        self.assertRegex(out, '(^|\n)I am fine\n')
        # should not have any test stderr
        self.assertNotIn(' stderr ', err)

        # should not build package
        self.assertNotIn('dh build', err)

    def test_apt_source_no_package_list(self):
        '''old apt source without Package-List'''

        with open(os.path.join(self.chroot, 'usr', 'bin', 'apt-cache'), 'w') as f:
            f.write('#!/bin/sh\n')
            f.write('if [ "$1" = showsrc ]; then printf "Format: 1.0\\nBinary: $3\\n"; fi\n')

        p = self.build_src('Tests: pass\nDepends:\nRestrictions: needs-root\n',
                           {'pass': '#!/bin/sh\necho I am fine\n'})
        # copy that into the chroot where fake apt-get source can find it
        shutil.copytree(p, os.path.join(self.chroot, 'aptget-src'))

        (code, out, err) = self.runtest(['-d', 'testpkg'])
        # test should succeed
        self.assertEqual(code, 0, err)
        self.assertRegex(out, 'pass\s+PASS', out)

    def test_setup_commands_string(self):
        '''--setup-commands with command string'''

        p = self.build_src('Tests: pass\nDepends:\nRestrictions: needs-root\n',
                           {'pass': '#!/bin/sh -e\n[ -x /bin/cp_cp ]; cat /setup.log\n'})

        # not expecting a normal user for chroot; but filter out any
        # $AUTOPKGTEST_NORMAL_USER from *our* environment
        env = os.environ.copy()
        try:
            del env['AUTOPKGTEST_NORMAL_USER']
        except KeyError:
            pass
        (code, out, err) = self.runtest(['-B', '--unbuilt-tree=' + p,
                                         '--setup-commands', '[ -z $AUTOPKGTEST_NORMAL_USER ];'
                                         'sleep 3; cp /bin/cp /bin/cp_cp; '
                                         'echo setup_success > /setup.log',
                                         '--setup-commands', 'cp /bin/cp /bin/cp_cp',
                                         '--timeout-short=1', '--timeout-copy=1'], env=env)
        # test should succeed
        self.assertEqual(code, 0, err)
        self.assertRegex(out, 'pass\s+PASS', out)

        # should show test stdout
        self.assertRegex(out, '^setup_success\n')
        # should not have any test stderr
        self.assertNotIn(' stderr ', err)

        self.assertIn('@@@@@@ test bed setup', err)

    def test_setup_commands_file(self):
        '''--setup-commands with command file'''

        p = self.build_src('Tests: pass\nDepends:\nRestrictions: needs-root\n',
                           {'pass': '#!/bin/sh -e\n[ -x /bin/cp_cp ]; cat /setup.log\ncat /s2.log'})

        cmds = os.path.join(self.workdir, 'setup.sh')
        with open(cmds, 'w') as f:
            f.write('cp /bin/cp /bin/cp_cp\necho setup_success > /setup.log\n')
            f.flush()
        cmds2 = os.path.join(self.workdir, 'setup2.sh')
        with open(cmds2, 'w') as f:
            f.write('echo setup2_success > /s2.log\n')
            f.flush()

        (code, out, err) = self.runtest(['-B', '-d', '--unbuilt-tree=' + p,
                                         '--setup-commands', cmds,
                                         '--setup-commands', cmds2])
        # test should succeed
        self.assertEqual(code, 0, err)
        self.assertRegex(out, 'pass\s+PASS', out)

        # should show test stdout
        self.assertRegex(out, '^setup_success\n')
        # should not have any test stderr
        self.assertNotIn(' stderr ', err)

        self.assertIn('@@@@@@ test bed setup', err)

    def test_copy(self):
        '''--copy'''

        p = self.build_src('Tests: pass\nDepends:\nRestrictions: needs-root\n',
                           {'pass': '#!/bin/sh -e\ncp -a /mytestdata/* $AUTOPKGTEST_ARTIFACTS\necho test_ok'})

        outdir = os.path.join(self.workdir, 'out')
        os.mkdir(outdir)

        (code, out, err) = self.runtest(['-B', '--unbuilt-tree=' + p, '-o', outdir,
                                         '--copy', '%s:/mytestdata/Makefile' % os.path.join(root_dir, 'Makefile'),
                                         '--copy', '%s:/mytestdata/tree' % os.path.join(test_dir, 'testpkg')])
        # test should succeed
        self.assertEqual(code, 0, err)
        self.assertRegex(out, 'pass\s+PASS', out)
        self.assertRegex(out, '^test_ok\n')

        # has copied the file correctly
        f = os.path.join(outdir, 'artifacts', 'Makefile')
        self.assertTrue(os.path.isfile(f))
        with open(f) as fcopy:
            with open(os.path.join(root_dir, 'Makefile')) as forig:
                self.assertEqual(forig.read(), fcopy.read())

        # has copied the dir correctly
        d = os.path.join(outdir, 'artifacts', 'tree')
        self.assertTrue(os.path.isdir(d))
        subprocess.check_call(['diff', '-Nur', os.path.join(test_dir, 'testpkg'), d])

    def test_apt_pocket(self):
        '''--apt-pocket'''

        p = self.build_src('Tests: pass\nDepends:\nRestrictions: needs-root\n',
                           {'pass': '#!/bin/sh -e\ntest -e /etc/apt/sources.list.d/proposed.list'})

        apt_dir = os.path.join(self.chroot, 'etc', 'apt')
        with open(os.path.join(apt_dir, 'sources.list'), 'w') as f:
            f.write('''# comment
deb http://foo.ubuntu.com/ fluffy-updates main non-free
deb-src http://foo.ubuntu.com/ fluffy-updates main non-free
deb http://foo.ubuntu.com/ fluffy main non-free
deb-src http://foo.ubuntu.com/ fluffy main non-free
deb http://bar.debian.org/ fluffy extras
deb-src http://bar.debian.org/ fluffy extras
# third-party repo
deb http://something.else.net/ fluffy addons
# options
deb [trusted=yes arch=6510] http://foo.ubuntu.com/ fluffy main 6510
''')
        with open(os.path.join(apt_dir, 'sources.list.d', 'restricted.list'), 'w') as f:
            f.write('deb http://foo.ubuntu.com/ fluffy restricted\n')

        (code, out, err) = self.runtest(['-B', '-d', '--unbuilt-tree=' + p,
                                         '--apt-pocket', 'proposed'])

        # test should succeed
        self.assertEqual(code, 0, err)
        self.assertRegex(out, 'pass\s+PASS', out)

        # verify proposed.list
        with open(os.path.join(apt_dir, 'sources.list.d', 'proposed.list')) as f:
            self.assertEqual(f.read(), '''deb http://foo.ubuntu.com/ fluffy-proposed main non-free
deb-src http://foo.ubuntu.com/ fluffy-proposed main non-free
deb http://bar.debian.org/ fluffy-proposed extras
deb-src http://bar.debian.org/ fluffy-proposed extras
deb [trusted=yes arch=6510] http://foo.ubuntu.com/ fluffy-proposed main 6510
deb http://foo.ubuntu.com/ fluffy-proposed restricted
''')
        # should not have any apt pinning
        self.assertFalse(os.path.exists(os.path.join(apt_dir, 'preferences.d')))

    def test_apt_pocket_selected(self):
        '''--apt-pocket with package list'''

        p = self.build_src('Tests: pass\nDepends:\nRestrictions: needs-root\n',
                           {'pass': '#!/bin/sh -e\ntest -e /etc/apt/sources.list.d/proposed.list'})

        apt_dir = os.path.join(self.chroot, 'etc', 'apt')
        with open(os.path.join(apt_dir, 'sources.list'), 'w') as f:
            f.write('''# comment
deb http://foo.ubuntu.com/ fluffy-updates main non-free
deb-src http://foo.ubuntu.com/ fluffy-updates main non-free
deb http://foo.ubuntu.com/ fluffy main non-free
deb-src http://foo.ubuntu.com/ fluffy main non-free
deb [trusted=yes arch=6510] http://foo.ubuntu.com/ fluffy main 6510
''')

        (code, out, err) = self.runtest(['-B', '-d', '--unbuilt-tree=' + p,
                                         '--apt-pocket', 'proposed=foo,bar'])

        # test should succeed
        self.assertEqual(code, 0, err)
        self.assertRegex(out, 'pass\s+PASS', out)

        # verify proposed.list
        with open(os.path.join(apt_dir, 'sources.list.d', 'proposed.list')) as f:
            self.assertEqual(f.read(), '''deb http://foo.ubuntu.com/ fluffy-proposed main non-free
deb-src http://foo.ubuntu.com/ fluffy-proposed main non-free
deb [trusted=yes arch=6510] http://foo.ubuntu.com/ fluffy-proposed main 6510
''')

        # should set up apt pinning
        self.assertEqual(os.listdir(os.path.join(apt_dir, 'preferences.d')),
                         ['autopkgtest-fluffy-proposed'])
        with open(os.path.join(apt_dir, 'preferences.d', 'autopkgtest-fluffy-proposed')) as f:
            self.assertEqual(f.read(), '''Package: foo bar
Pin: release a=fluffy-proposed
Pin-Priority: 990

Package: *
Pin: release a=fluffy
Pin-Priority: 900

Package: *
Pin: release a=fluffy-updates
Pin-Priority: 900

Package: *
Pin: release a=fluffy-proposed
Pin-Priority: 800
''')

    def test_tree_garbage(self):
        '''copied source tree contains only expected files'''

        p = self.build_src('Tests: g\nDepends:\nRestrictions: needs-root\n',
                           {'g': '#!/bin/sh\npwd\nLC_ALL=C ls .\n'})

        (code, out, err) = self.runtest(['-B', '--unbuilt-tree=' + p])
        self.assertEqual(code, 0, err)
        self.assertRegex(out, 'g\s+PASS', out)

        self.assertRegex(out, '/autopkgtest.*/real-tree\n'
                         'Makefile\ndebian\ntest_static\n')
        # should not have any test stderr
        self.assertNotIn(' stderr ', err)

    def test_no_tests_dir(self):
        '''package without debian/tests/'''

        p = self.build_src(None, None)
        (code, out, err) = self.runtest(['-B', '--unbuilt-tree=' + p])
        self.assertEqual(code, 8, err)
        self.assertRegex(out, 'SKIP no tests in this package', out)

    def test_override_control(self):
        '''custom control file path with --override-control'''

        p = self.build_src('Tests: NOTME\nDepends: nonexisting\nRestrictions: breaks-testbed\n',
                           {'pass': '#!/bin/sh\necho I am fine\n'})

        custom_control = os.path.join(self.workdir, 'myctrl')
        with open(custom_control, 'w') as f:
            f.write('Tests: pass\nDepends:\nRestrictions: needs-root')

        (code, out, err) = self.runtest(['-B', '--override-control=' + custom_control, p + '//'])
        # test should succeed
        self.assertEqual(code, 0, err)
        self.assertRegex(out, 'pass\s+PASS', out)

        # should show test stdout
        self.assertRegex(out, '^I am fine\n')
        # should not have any test stderr
        self.assertNotIn(' stderr ', err)

        # should not try anything with the bad test
        self.assertNotIn('NOTME', out)
        self.assertNotIn('NOTME', err)

    def test_nonexisting_override_control(self):
        '''nonexisting --override-control'''

        p = self.build_src('Tests: NOTME\nDepends: nonexisting\n', {})
        (code, out, err) = self.runtest(['--override-control=/nonexisting', p + '/'])
        self.assertEqual(code, 20, err)
        self.assertIn('/nonexisting', err)

        # should not try anything with the original control file
        self.assertNotIn('NOTME', out)
        self.assertNotIn('NOTME', err)

    def test_relative_chroot_path(self):
        '''relative chroot path'''

        p = self.build_src('Tests: pass\nDepends:\nRestrictions: needs-root\n',
                           {'pass': '#!/bin/sh\necho I am fine\n'})

        os.chdir(self.workdir)
        (code, out, err) = self.runtest(['--no-built-binaries', '--unbuilt-tree=' + p],
                                        virt_args=['chroot', os.path.basename(self.chroot)])
        # test should succeed
        self.assertEqual(code, 0, err)
        self.assertRegex(out, 'pass\s+PASS', out)
        # should show test stdout
        self.assertRegex(out, '^I am fine\n')

    def test_cli_args_from_file(self):
        '''read CLI arguments from file'''

        p = self.build_src('Tests: pass\nDepends:\nRestrictions: needs-root\n',
                           {'pass': '#!/bin/sh\necho I am fine\n'})

        argfile = os.path.join(self.workdir, 'myopts')
        with open(argfile, 'w') as f:
            f.write(' -d \n%s' % self.chroot)

        (code, out, err) = self.runtest(['--no-built-binaries', '--unbuilt-tree=' + p],
                                        virt_args=['chroot', '@' + argfile])

        # test should succeed
        self.assertEqual(code, 0, err)
        self.assertRegex(out, 'pass\s+PASS', out)

        # should have debug messages
        self.assertIn('chroot: DBG:', err)

    def test_no_dpkg_query(self):
        '''testbed does not have dpkg-query'''

        p = self.build_src('Tests: t\nDepends:\nRestrictions: needs-root\n',
                           {'t': '#!/bin/sh -e\necho ok\n'})
        outdir = os.path.join(self.workdir, 'out')
        (code, out, err) = self.runtest(['--setup-commands', 'rm /usr/bin/dpkg-query',
                                         '-d', '--built-tree=' + p,
                                         '--output-dir=' + outdir])

        # test should succeed
        self.assertEqual(code, 0, err)
        self.assertRegex(out, 't\s+PASS', out)

        # check outdir
        with open(os.path.join(outdir, 't-stdout')) as f:
            self.assertEqual(f.read(), 'ok\n')
        with open(os.path.join(outdir, 'log')) as f:
            contents = f.read()
        self.assertRegex(contents, 't\s+PASS\n')

        # don't create empty/bogus files
        self.assertNotIn('testbed-packages', os.listdir(outdir))


class DebTestsVirtFS(DebTestsAll):
    '''Common tests for runners with file system virtualization'''

    def test_tree_norestrictions_nobuild_success(self):
        '''source tree, no build, no restrictions, test success'''

        # also test some fancy dependencies
        p = self.build_src('Tests: p1 p2\n'
                           'Depends: coreutils, aspell-doc [linux-any], apt-doc:native, unknown [nosucharch], bash-doc [!c64], bogus [!linux-any]\n'
                           'Restrictions: needs-root\n\n'
                           'Tests: n\n'
                           'Depends:\n'
                           'Restrictions: needs-root\n',
                           {'p1': '#!/bin/sh -e\nif dpkg -s autopkgtest-satdep 2>/dev/null; then exit 1; fi; '
                            'apt-get --purge -y autoremove; '
                            'ls /usr/share/doc/aspell-doc/copyright;'
                            'echo I am fine\n',
                            'p2': '#!/bin/sh\necho I am also fine; echo "SOMEVAR: >$SOMEVAR<"; echo "PATH: $PATH"\n',
                            'n': '#!/bin/sh -eu\nid $AUTOPKGTEST_NORMAL_USER'})

        (code, out, err) = self.runtest(['-d', '--no-built-binaries', '--unbuilt-tree=' + p],
                                        env={'SOMEVAR': 'junk',
                                             'HOME': os.environ['HOME'],
                                             'PATH': '/something/broken:' + os.environ.get('PATH', '')})

        # test should succeed
        self.assertEqual(code, 0, err)
        self.assertRegex(out, 'p1\s+PASS', out)
        self.assertRegex(out, 'p2\s+PASS', out)
        self.assertRegex(out, 'n\s+PASS', out)

        # $SOMEVAR does not leak into container
        self.assertIn('\nSOMEVAR: ><\n', out)
        # modified $PATH does not leak into container
        self.assertNotIn('/something/broken', out)

        # should have $AUTOPKGTEST_NORMAL_USER
        self.assertIn('uid=', out)

        # handles expected packages
        self.assertIn('processing dependency coreutils', err)
        self.assertIn('Unpacking aspell-doc', out)
        self.assertIn('Unpacking apt-doc', out)
        self.assertIn('Unpacking bash-doc', out)
        self.assertNotIn('unknown', out)
        self.assertNotIn('bogus', out)
        self.assertNotIn('Removing aspell-doc', out)

        # should show test stdout
        self.assertRegex(out, '(^|\n)I am fine\n')
        self.assertIn('\nI am also fine\n', out)
        # should not have any test stderr
        self.assertNotIn(' stderr ', err)

        # should not build package
        self.assertNotIn('dh build', err)

        # should not show boot messages
        self.assertNotIn(' login:', err)

    def test_tree_built_binaries(self):
        '''source tree, install built binaries'''

        # no "build-needed" restriction here, it should be built because we
        # need to install the package as a dependency (@)
        p = self.build_src('Tests: pass\n',
                           {'pass': '#!/bin/sh -e\n/usr/bin/test_built | grep -q "built script OK"\necho GOOD'})

        # add extra dependency to testpkg, to ensure autopkgtest does not stop
        # and ask for confirmation in apt-get
        subprocess.check_call(['sed', '-i', '/^Depends:/ s/$/, aspell-doc/',
                               os.path.join(p, 'debian', 'control')])

        (code, out, err) = self.runtest(['--unbuilt-tree=' + p])

        # test should succeed
        self.assertEqual(code, 0, err)
        self.assertRegex(out, 'pass\s+PASS', out)

        # should build and install package
        self.assertIn('dh build', err)
        self.assertIn('binaries/./testpkg.deb', out)
        self.assertRegex(out, '\.\.\./aspell-doc_\d.*_all.deb')

        # should show test stdout
        self.assertRegex(out, '(^|\n)GOOD\n', out)
        # should not have any test stderr
        self.assertNotIn(' stderr ', err)

    def test_dsc_build_needed_success(self):
        '''dsc, build-needed restriction, test success'''

        p = self.build_dsc('Tests: pass\nDepends:\nRestrictions: build-needed\n',
                           {'pass': '#!/bin/sh -e\n./test_built | grep -q "built script OK"; '
                            'echo XXX $USER $(whoami) YYY'})

        (code, out, err) = self.runtest(['--no-built-binaries', p])
        # test should succeed
        self.assertRegex(out, 'pass\s+PASS', out)
        self.assertEqual(code, 0, err)

        # should not have any test stderr
        self.assertNotIn(' stderr ', err)

        # should build package
        self.assertIn('dh build', err)

        # test should run as user
        lines = [l for l in out.splitlines() if l.startswith('XXX')]
        self.assertEqual(len(lines), 1, lines)
        fields = lines[0].split()
        self.assertEqual(len(fields), 4)
        self.assertNotEqual(fields[1], 'root')
        self.assertEqual(fields[1], fields[2])
        self.assertEqual(fields[3], 'YYY')

    def test_setup_commands(self):
        '''--setup-commands'''

        p = self.build_src('Tests: t1 t2\nDepends:\n',
                           {'t1': '#!/bin/sh -e\ncat /setup.log; echo t1ok\n',
                            't2': '#!/bin/sh -e\ncat /setup.log; echo t2ok\n'})

        # we expect a normal user for LXC
        (code, out, err) = self.runtest(['-B', '-d', p + '//', '--setup-commands',
                                         '[ -n "$AUTOPKGTEST_NORMAL_USER" ]; getent passwd $AUTOPKGTEST_NORMAL_USER;'
                                         'echo setupok >> /setup.log'])

        # test should succeed
        self.assertEqual(code, 0, err)
        self.assertRegex(out, 't1\s+PASS', out)
        self.assertRegex(out, 't2\s+PASS', out)

        # should show test stdout
        self.assertRegex(out, '^\w+:x:.*\nsetupok\nt1ok\n')
        self.assertIn('\nsetupok\nt2ok\n', out)
        # should not have any test stderr
        self.assertNotIn(' stderr ', err)

        # should not have rebooted for these commands
        self.assertIn('@@@@@@ test bed setup', err)
        self.assertNotIn('rebooting testbed', err)

    def test_setup_commands_suppress_reboot(self):
        '''--setup-commands with suppressed reboot'''

        p = self.build_src('Tests: t\nDepends:\n',
                           {'t': '#!/bin/sh -e\ntest -e /run/autopkgtest_no_reboot.stamp'})

        (code, out, err) = self.runtest(['-B', '-d', p + '//', '--setup-commands',
                                         'apt-get install --reinstall --no-install-recommends -y cron; touch /run/autopkgtest_no_reboot.stamp'])

        # test should succeed
        self.assertEqual(code, 0, err)
        self.assertRegex(out, 't\s+PASS', out)

        # should not have any test stderr
        self.assertNotIn(' stderr ', err)

        # should not have rebooted
        self.assertIn('@@@@@@ test bed setup', err)
        self.assertNotIn('rebooting testbed', err)

    def test_apt_source(self):
        '''apt source'''

        (code, out, err) = self.runtest(['-d', 'gdk-pixbuf'])
        # test should succeed
        self.assertEqual(code, 0, err)
        self.assertRegex(out, 'Unpacking libgdk-pixbuf.*-dev')
        self.assertRegex(out, 'build\s+PASS', out)

        # should show test stdout
        self.assertIn('OK\n', out)
        # should not have any test stderr
        self.assertNotIn(' stderr ', err)

    def test_user(self):
        '''Run tests as different user'''

        p = self.build_src('Tests: r\nDepends:\nRestrictions: rw-build-tree, needs-root\n\n'
                           'Tests: t\nDepends: aspell-doc\nRestrictions: rw-build-tree\n',
                           {'r': '''#!/bin/sh -e
                                    mkdir xsrc
                                    echo r00t > xsrc/root.txt
                                    ''',
                            't': '''#!/bin/sh -e
                                    echo world > $AUTOPKGTEST_TMP/hello.txt
                                    cat $AUTOPKGTEST_TMP/hello.txt
                                    mkdir -p $AUTOPKGTEST_TMP/one/subdir
                                    echo deep > $AUTOPKGTEST_TMP/one/subdir/subdir.txt
                                    cat $AUTOPKGTEST_TMP/one/subdir/subdir.txt
                                    rm -r $AUTOPKGTEST_TMP/one
                                    echo dir_is_rw > rw_flag
                                    echo user >> xsrc/root.txt
                                    cat rw_flag
                                    echo hello > $AUTOPKGTEST_ARTIFACTS/world.txt
                                    chmod 600 $AUTOPKGTEST_ARTIFACTS/world.txt
                                    echo xxtmpdir > ${TMPDIR:=/tmp}/xxtmpdir.txt
                                    cat $TMPDIR/xxtmpdir.txt
                                    whoami'''})

        outdir = os.path.join(self.workdir, 'out')
        (code, out, err) = self.runtest(['-B', p + '//', '--user=nobody',
                                         '--output-dir', outdir])
        self.assertEqual(code, 0, err)

        # test should succeed
        self.assertRegex(out, 't\s+PASS', out)

        # has output from cats and whoami
        self.assertIn('world\ndeep\ndir_is_rw\nxxtmpdir\nnobody\n', out)

        # should install dependencies
        self.assertIn('Unpacking aspell-doc', out)

        # can write artifacts
        with open(os.path.join(outdir, 'artifacts', 'world.txt')) as f:
            self.assertEqual(f.read(), 'hello\n')

    def test_user_needs_root(self):
        '''--user option with needs-root restriction'''

        p = self.build_src('Tests: t\nDepends: aspell-doc\nRestrictions: needs-root',
                           {'t': '''#!/bin/sh -e
                                    echo world > ${TMPDIR:=/tmp}/hello.txt
                                    cat $TMPDIR/hello.txt
                                    rm $TMPDIR/hello.txt
                                    whoami
                                    echo "USER: $USER"
                                 '''})

        (code, out, err) = self.runtest(['--no-built-binaries',
                                         '--unbuilt-tree=' + p,
                                         '--user=nobody'])
        self.assertEqual(code, 0, err)

        # should install dependencies
        self.assertIn('Unpacking aspell-doc', out)

        # test should succeed
        self.assertRegex(out, 't\s+PASS', out)

        # has output from cat and whoami
        self.assertRegex(out, '(^|\n)world\nroot\nUSER: root\n')

    def test_copy_timeout(self):
        '''handling copying timeout'''

        p = self.build_src('Tests: to\nDepends:\n',
                           {'to': '#!/bin/sh\ntruncate -s 10G $AUTOPKGTEST_ARTIFACTS/huge\n'})

        outdir = os.path.join(self.workdir, 'out')
        (code, out, err) = self.runtest(['--output-dir=' + outdir,
                                         '--timeout-copy=1',
                                         '-B', p + '//'])

        # test should time out
        self.assertEqual(code, 16, err)
        self.assertRegex(out, 'to\s+PASS', out)

        # should have copied parts of the file
        huge_size = os.path.getsize(os.path.join(outdir, 'artifacts', 'huge'))
        self.assertGreater(huge_size, 100000)
        self.assertLess(huge_size, 10000000000)

        # we don't want the raw exception
        self.assertNotIn("Timeout", err)
        self.assertNotIn("Traceback", err)
        # but a proper error message, with cleanup handlers
        self.assertIn("got `timeout', expected `ok...'", err)

    def test_copy_performance(self):
        '''copying files between host and testbed is fast and reliable'''

        p = self.build_src('Tests: t\nDepends:\n',
                           {'t': '#!/bin/sh\ncp -r . $AUTOPKGTEST_ARTIFACTS/tree\n'})
        # create ~ 100 MB of fake source tree (~ 1000 files with 100 kB each)
        block = b'0123456789' * 10000
        for dirnum in range(33):
            d = os.path.join(p, str(dirnum))
            os.mkdir(d)
            for filenum in range(33):
                with open(os.path.join(d, 'src%i' % filenum), 'wb') as f:
                    f.write(block)

        # we expect at least 20 MB/s, thus give it 6 seconds
        outdir = os.path.join(self.workdir, 'out')
        (code, out, err) = self.runtest(['--output-dir=' + outdir,
                                         '--timeout-copy=6',
                                         '-B', p + '//'])

        # test should succeed
        self.assertEqual(code, 0, out + err)
        self.assertRegex(out, 't\s+PASS', out)

        # check integrity of the copy
        subprocess.check_call(['diff', '-Nur', p,
                               os.path.join(outdir, 'artifacts', 'tree')])


class DebTestsVirtContainer(DebTestsVirtFS):
    '''Common tests for runners with container virtualization'''

    def test_logind(self):
        '''test has logind session'''

        p = self.build_src('Tests: logind\nDepends:\n',
                           {'logind': '#!/bin/sh -e\n[ -n "$XDG_SESSION_ID" ] || { echo "no XDG_SESSION_ID" >&2; exit 1; }; '
                            'O=$(loginctl show-session $XDG_SESSION_ID);'
                            'echo "$O" | grep -q "^Active=yes" || echo "$O" >&2; '
                            'echo "$O" | grep -q "^Name=$USER" || echo "$O" >&2; '})

        # testbed might not have libpam-systemd
        (code, out, err) = self.runtest(['--setup-commands', 'apt-get install -y libpam-systemd',
                                         '--no-built-binaries', '--unbuilt-tree=' + p])

        # test should succeed
        self.assertEqual(code, 0, err)
        self.assertRegex(out, 'logind\s+PASS', out)

    def test_no_locale_conf(self):
        '''No /etc/default/locale'''

        p = self.build_src('Test-Command: locale; echo TestOK\nDepends:\n', {})

        # we expect a normal user for LXC
        (code, out, err) = self.runtest(['-B', p + '//', '--setup-commands',
                                         'rm -f /etc/default/locale /etc/environment'])

        # test should succeed
        self.assertEqual(code, 0, err)
        self.assertRegex(out, 'command1\s+PASS')
        self.assertRegex(out, 'LC_MESSAGES="C.UTF-8"')
        self.assertRegex(out, 'TestOK')
        # should not have any test stderr
        self.assertNotIn(' stderr ', err)

    def test_timeout(self):
        '''handling test timeout'''

        p = self.build_dsc('Tests: totest\nDepends:\n',
                           {'totest': '#!/bin/sh\necho start\nsleep 10\necho after_sleep\n'})

        (code, out, err) = self.runtest(['-B', '-d', '--timeout-test=3', p])
        # test should time out
        self.assertEqual(code, 4, err)
        self.assertIn("timed out on command", err)
        self.assertRegex(out, 'totest\s+FAIL timed out', out)

        # should show test stdout
        self.assertIn('start', out)
        # but not let the test finish
        self.assertNotIn('after_sleep', out)

    def test_reboot(self):
        '''test that reboots'''

        p = self.build_src('Tests: s r\nDepends:\nRestrictions: needs-root',
                           {'s': '#!/bin/sh\necho simple_done',
                            'r': '''#!/bin/sh -e
        case "$AUTOPKGTEST_REBOOT_MARK" in
          mark1) echo "test in mark1"; echo one > $AUTOPKGTEST_ARTIFACTS/mark1.txt; /tmp/autopkgtest-reboot mark2 ;;
          mark2) echo "test in mark2"; echo two > $AUTOPKGTEST_ARTIFACTS/mark2.txt ;;
          "") echo "test beginning"; echo zero > $AUTOPKGTEST_ARTIFACTS/begin.txt; /tmp/autopkgtest-reboot mark1 ;;
          *) echo "unknown AUTOPKGTEST_REBOOT_MARK value $AUTOPKGTEST_REBOOT_MARK" >&2; exit 1 ;;
        esac
        echo "test end"'''})

        outdir = os.path.join(self.workdir, 'out')
        os.mkdir(outdir)

        # special-case LXC runner, --ephemeral does not support reboot with LXC 1.x
        if not shutil.which('lxc-copy'):
            virt_args = [a for a in self.virt_args if a != '--ephemeral']
        else:
            virt_args = self.virt_args
        (code, out, err) = self.runtest(['--output-dir=' + outdir, '-B', p + '//'],
                                        virt_args=virt_args)

        # test should succeed
        self.assertEqual(code, 0, err)
        self.assertRegex(out, 's\s+PASS', out)
        self.assertRegex(out, 'r\s+PASS', out)

        # should have all three phases
        self.assertIn('simple_done\n', out)
        self.assertIn('\ntest beginning\ntest in mark1\ntest in mark2\ntest end\n', out)
        self.assertIn('test process requested reboot with marker mark1', err)
        self.assertIn('test process requested reboot with marker mark2', err)

        self.assertRegex(err, 'testbed running kernel: Linux \d')
        self.assertNotIn('running kernel changed', err)

        # should not have any test stderr
        self.assertNotIn(' stderr ', err)

    def test_reboot_prepare(self):
        '''test that reboots by itself'''

        p = self.build_src('Tests: s r\nDepends:\nRestrictions: needs-root',
                           {'s': '#!/bin/sh\necho simple_done',
                            'r': '''#!/bin/sh -e
        case "$AUTOPKGTEST_REBOOT_MARK" in
          "")
              echo "test beginning"
              echo zero > $AUTOPKGTEST_ARTIFACTS/begin.txt
              /tmp/autopkgtest-reboot-prepare mark1
              touch /var/post-prepare-1
              reboot ;;
          mark1)
              echo "test in mark1"
              echo one > $AUTOPKGTEST_ARTIFACTS/mark1.txt
              /tmp/autopkgtest-reboot-prepare mark2
              touch /var/post-prepare-2
              printf '#!/bin/sh\necho Linux 1.2.3-testy foobar' > /usr/local/bin/uname
              chmod 755 /usr/local/bin/uname
              reboot ;;
          mark2)
              echo "test in mark2"
              echo two > $AUTOPKGTEST_ARTIFACTS/mark2.txt
              ls /var/post-prepare-1 /var/post-prepare-2 ;;
          *)
              echo "unknown AUTOPKGTEST_REBOOT_MARK value $AUTOPKGTEST_REBOOT_MARK" >&2
              exit 1 ;;
        esac'''})

        outdir = os.path.join(self.workdir, 'out')
        os.mkdir(outdir)

        # special-case LXC runner, --ephemeral does not support reboot with LXC 1.x
        if not shutil.which('lxc-copy'):
            virt_args = [a for a in self.virt_args if a != '--ephemeral']
        else:
            virt_args = self.virt_args
        (code, out, err) = self.runtest(['--output-dir=' + outdir, '-B', p +
                                         '//'], virt_args)

        # test should succeed
        self.assertEqual(code, 0, err)
        self.assertRegex(out, 's\s+PASS', out)
        self.assertRegex(out, 'r\s+PASS', out)

        # should have all three phases
        self.assertIn('simple_done\n', out)
        self.assertIn('\ntest beginning\ntest in mark1\n'
                      'test in mark2\n/var/post-prepare-1\n/var/post-prepare-2\n', out)
        self.assertIn('test process requested preparation for reboot with marker mark1', err)
        self.assertIn('test process requested preparation for reboot with marker mark2', err)

        self.assertIn('running kernel changed: Linux 1.2.3-testy foobar (current test: r, last reboot marker: mark2)\n', err)
        self.assertNotIn('running kernel changed: Linux 1.2.3-testy foobar (current test s', err)

        # should not have any test stderr
        self.assertNotIn(' stderr ', err)

        # check artifacts
        with open(os.path.join(outdir, 'artifacts', 'begin.txt')) as f:
            self.assertEqual(f.read(), 'zero\n')
        with open(os.path.join(outdir, 'artifacts', 'mark1.txt')) as f:
            self.assertEqual(f.read(), 'one\n')
        with open(os.path.join(outdir, 'artifacts', 'mark2.txt')) as f:
            self.assertEqual(f.read(), 'two\n')
        with open(os.path.join(outdir, 'testinfo.json')) as f:
            info = json.load(f)
        self.assertEqual(info['test_kernel_versions'],
                         [['r', 'mark2', 'Linux 1.2.3-testy foobar']])
        # this check only works in LXC where the host kernel is used, not in Qemu
        if 'Qemu' not in self.__class__.__name__:
            u = os.uname()
            self.assertEqual(info['kernel_version'],
                             '%s %s %s' % (u.sysname, u.release, u.version.strip()))

    def test_exit_255(self):
        '''test exits with code 255

        This should not be considered a failure of the auxverb.
        '''
        p = self.build_src('Tests: t255\nDepends:\n', {'t255': '#!/bin/sh\nexit 255'})

        (code, out, err) = self.runtest(['-B', p + '//'])
        self.assertEqual(code, 4, err)
        self.assertRegex(out, 't255\s+FAIL non-zero exit status 253')
        self.assertNotIn('testbed auxverb failed with', err)

    def test_background_process(self):
        '''leftover background process in test'''

        # leak sleep processes during build and test
        p = self.build_src('Tests: bg\nDepends:\nRestrictions: build-needed',
                           {'bg': '#!/bin/bash -e\nexec -a bg_daemon sleep 30 &\n'
                                  'echo TEST DONE\n'})

        with open(os.path.join(p, 'debian', 'rules'), 'a') as f:
            f.write('\n\noverride_dh_auto_build:\n\tsleep 60 &\n'
                    '\tdh_auto_build\n')

        (code, out, err) = self.runtest(
            ['-d', '-B', p + '//',
             '--timeout-build=10', '--timeout-test=5'])
        self.assertEqual(code, 0, err)
        self.assertRegex(out, 'bg\s+PASS', out)
        self.assertIn('TEST DONE\n', out)

    def test_setup_commands_reboot(self):
        '''--setup-commands with reboot'''

        p = self.build_src('Tests: t\nDepends:\n\nTests: r\nDepends:\nRestrictions: needs-root, build-needed\n',
                           {'t': '#!/bin/sh -e\n[ -d "$HOME" ] || echo "HOME not set" >&2\n',
                            'r': '#!/bin/sh -e\n[ -d "$HOME" ] || echo "HOME not set" >&2\n'
                                 '[ "$USER" = "root" ] || echo "USER invalid: $USER" >&2\n'
                                 './test_built\n./test_abspath\n'})

        # special-case LXC runner, --ephemeral does not support reboot
        virt_args = [a for a in self.virt_args if a != '--ephemeral']
        (code, out, err) = self.runtest(['-B', '-d', p + '//', '--setup-commands',
                                         'apt-get install --reinstall -y cron'],
                                        virt_args=virt_args)

        # test should succeed
        self.assertEqual(code, 0, err)
        self.assertRegex(out, 't\s+PASS', out)
        self.assertRegex(out, 'r\s+PASS', out)

        # built sources should succeed
        self.assertIn('built script OK\nbuilt script OK\n', out)

        # should not have any test stderr
        self.assertNotIn(' stderr ', err)

        # should have rebooted for these commands
        self.assertIn('@@@@@@ test bed setup', err)
        self.assertIn('rebooting testbed', err)


@unittest.skipUnless('AUTOPKGTEST_TEST_LXC' in os.environ,
                     'Set $AUTOPKGTEST_TEST_LXC to an existing container')
class LxcRunner(AdtTestCase, DebTestsVirtContainer):
    def __init__(self, *args, **kwargs):
        self.container_name = os.environ.get('AUTOPKGTEST_TEST_LXC')
        if self.container_name:
            vargs = ['lxc', '--ephemeral', self.container_name]
            # check for unprivileged containers
            if os.getuid() > 0 and subprocess.call(
                    ['lxc-info', '-n', self.container_name],
                    stdout=subprocess.PIPE, stderr=subprocess.STDOUT) != 0:
                vargs.insert(1, '--sudo')
        else:
            vargs = []  # tests will be skipped anyway, but we still must be instantiable

        super().__init__(vargs, *args, **kwargs)


@unittest.skipUnless('AUTOPKGTEST_TEST_LXD' in os.environ,
                     'Set $AUTOPKGTEST_TEST_LXD to an existing LXD image')
class LxdRunner(AdtTestCase, DebTestsVirtContainer):
    def __init__(self, *args, **kwargs):
        vargs = ['lxd', os.environ.get('AUTOPKGTEST_TEST_LXD')]
        super().__init__(vargs, *args, **kwargs)


@unittest.skipUnless('AUTOPKGTEST_TEST_QEMU' in os.environ,
                     'Set $AUTOPKGTEST_TEST_QEMU to an existing autopkgtest QEMU image')
class QemuRunner(AdtTestCase, DebTestsVirtContainer):
    def __init__(self, *args, **kwargs):
        super(QemuRunner, self).__init__(['qemu', os.environ.get('AUTOPKGTEST_TEST_QEMU')], *args, **kwargs)

    def test_options_and_baseimage(self):
        '''autopkgtest-virt-qemu options and /dev/baseimage'''

        p = self.build_src('Tests: t\nDepends:',
                           {'t': '''echo CPUNUM; nproc
                                 mkdir -p $AUTOPKGTEST_TMP/one/subdir
                                 echo deep > $AUTOPKGTEST_TMP/one/subdir/subdir.txt
                                 cat $AUTOPKGTEST_TMP/one/subdir/subdir.txt
                                 whoami
                                 test -L /dev/baseimage || echo "no /dev/baseimage link"
                                 blkid -p /dev/baseimage'''})

        (code, out, err) = self.runtest(['-B', '-d', p + '//'],
                                        self.virt_args + ['--cpus=2', '--show-boot', '--user=nobody'])

        # test should succeed
        self.assertEqual(code, 0, err)
        self.assertRegex(out, 't\s+PASS', out)

        # expected number of CPUs, cat, and expected user
        self.assertIn('CPUNUM\n2\ndeep\nnobody\n', out)

        # should show boot console output
        self.assertIn(' login:', err)

    def test_reboot_ro(self):
        '''test that reboots, r/o system'''

        p = self.build_src('Tests: s r\nDepends:\nRestrictions: needs-root breaks-testbed',
                           {'s': '#!/bin/sh\necho simple_done',
                            'r': '''#!/bin/sh -e
        case "$AUTOPKGTEST_REBOOT_MARK" in
          mark1) echo "test in mark1"; echo one > $AUTOPKGTEST_ARTIFACTS/mark1.txt; /tmp/autopkgtest-reboot mark2 ;;
          mark2) echo "test in mark2"; echo two > $AUTOPKGTEST_ARTIFACTS/mark2.txt ;;
          "") echo "test beginning"; echo zero > $AUTOPKGTEST_ARTIFACTS/begin.txt; /tmp/autopkgtest-reboot mark1 ;;
          *) echo "unknown AUTOPKGTEST_REBOOT_MARK value $AUTOPKGTEST_REBOOT_MARK" >&2; exit 1 ;;
        esac
        echo "test end"'''})

        outdir = os.path.join(self.workdir, 'out')
        os.mkdir(outdir)

        ro_sbin = 'M=$(mktemp --directory /run/ro-sbin.XXXXX); ' \
                  'mount -t tmpfs tmpfs $M; ' \
                  'cp -a /sbin $M; cp -a /var/cache $M; ' \
                  'mount -o remount,ro $M; ' \
                  'mount -o bind,ro $M/sbin /sbin; ' \
                  'mount -o bind,ro $M/cache /var/cache; '

        (code, out, err) = self.runtest(['--output-dir=' + outdir, '-B', p + '//',
                                         '--setup-commands', ro_sbin])

        # test should succeed
        self.assertEqual(code, 0, err)
        self.assertRegex(out, 's\s+PASS', out)
        self.assertRegex(out, 'r\s+PASS', out)

        # should have all three phases
        self.assertIn('simple_done\n', out)
        self.assertIn('\ntest beginning\ntest in mark1\ntest in mark2\ntest end\n', out)
        self.assertIn('test process requested reboot with marker mark1', err)
        self.assertIn('test process requested reboot with marker mark2', err)

        # should not have any test stderr
        self.assertNotIn(' stderr ', err)

        # check artifacts
        with open(os.path.join(outdir, 'artifacts', 'begin.txt')) as f:
            self.assertEqual(f.read(), 'zero\n')
        with open(os.path.join(outdir, 'artifacts', 'mark1.txt')) as f:
            self.assertEqual(f.read(), 'one\n')
        with open(os.path.join(outdir, 'artifacts', 'mark2.txt')) as f:
            self.assertEqual(f.read(), 'two\n')

        # check outdir test stdout/err
        with open(os.path.join(outdir, 'r-stdout')) as f:
            self.assertEqual(f.read(), 'test beginning\ntest in mark1\ntest in mark2\ntest end\n')
        self.assertFalse(os.path.exists(
            os.path.join(outdir, 'r-stderr')))

        # should not have any errors from tar or otherwise
        self.assertNotIn('Traceback', err)
        self.assertNotIn('tar:', err)


@unittest.skipUnless('AUTOPKGTEST_TEST_SCHROOT' in os.environ,
                     'Set $AUTOPKGTEST_TEST_SCHROOT to an existing schroot')
class SchrootRunner(AdtTestCase, DebTestsVirtFS, DebTestsFailureModes):
    def __init__(self, *args, **kwargs):
        super(SchrootRunner, self).__init__(['schroot', os.environ.get('AUTOPKGTEST_TEST_SCHROOT')],
                                            *args, **kwargs)

    def test_binary_unbuilt_tree(self):
        '''--binary for test, unbuilt tree'''

        p = self.build_src('Tests: pass\n',
                           {'pass': '#!/bin/sh -e\n/usr/bin/test_built'})

        # build the package
        subprocess.check_call(['dpkg-buildpackage', '-b', '-us', '-uc', '-tc'],
                              stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                              cwd=p)
        deb = os.path.join(os.path.dirname(p), 'testpkg_1_all.deb')
        self.assertTrue(os.path.exists(deb))
        # destroy debian/rules, to ensure it does not try to build the package
        # again
        with open(os.path.join(p, 'debian', 'rules'), 'w') as f:
            f.write('bwhahahahano!')

        (code, out, err) = self.runtest(['-d', '-B', '--binary', deb, '--unbuilt-tree=' + p])

        # should install package
        self.assertNotIn('dh build', err)
        self.assertIn('Unpacking testpkg', out)
        self.assertIn('binaries/./testpkg.deb', out)

        # test should succeed
        self.assertRegex(out, 'pass\s+PASS', out)
        self.assertEqual(code, 0, err)

        # should show test stdout
        self.assertRegex(out, '(^|\n)built script OK\n')
        # should not have any test stderr
        self.assertNotIn(' stderr ', err)

        # our deb should still be there
        self.assertTrue(os.path.exists(deb))

    def test_binary_built_tree(self):
        '''--binary for test, built tree'''

        # we test both for the installed package as well as the built tree
        p = self.build_src('Tests: pass\n',
                           {'pass': '#!/bin/sh -e\n/usr/bin/test_built\n./test_built'})

        # build the package
        subprocess.check_call(['dpkg-buildpackage', '-b', '-us', '-uc'],
                              stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                              cwd=p)
        deb = os.path.join(os.path.dirname(p), 'testpkg_1_all.deb')
        self.assertTrue(os.path.exists(deb))
        # destroy debian/rules, to ensure it does not try to build the package
        # again
        with open(os.path.join(p, 'debian', 'rules'), 'w') as f:
            f.write('bwhahahahano!')

        # --no-built-binaries should be implied here
        (code, out, err) = self.runtest(['-d', deb, '--built-tree=' + p])

        # should install package
        self.assertNotIn('dh build', err)
        self.assertIn('Unpacking testpkg', out)
        self.assertIn('binaries/./testpkg.deb', out)

        # test should succeed
        self.assertRegex(out, 'pass\s+PASS', out + err)
        self.assertEqual(code, 0, out + err)

        # should show test stdout
        self.assertRegex(out, '(^|\n)built script OK\nbuilt script OK')
        # should not have any test stderr
        self.assertNotIn(' stderr ', err)

        # our deb should still be there
        self.assertTrue(os.path.exists(deb))

    def test_changes(self):
        '''source/binary .changes'''

        p = self.build_src('Tests: pass\n',
                           {'pass': '#!/bin/sh -e\n/usr/bin/test_built'})

        # build the package
        subprocess.check_call(['dpkg-buildpackage', '-us', '-uc'],
                              stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                              cwd=p)
        changes = glob('%s/testpkg_1_*.changes' % os.path.dirname(p))
        self.assertEqual(len(changes), 1)
        changes = changes[0]
        (code, out, err) = self.runtest([changes])

        # should not rebuild
        self.assertNotIn('dh build', err)

        # should install package
        self.assertIn('Unpacking testpkg', out)
        self.assertIn('binaries/./testpkg.deb', out)

        # test should succeed
        self.assertRegex(out, 'pass\s+PASS', out)
        self.assertEqual(code, 0, err)

        # should show test stdout
        self.assertRegex(out, '(^|\n)built script OK\n')
        # should not have any test stderr
        self.assertNotIn(' stderr ', err)

        # our .changes should still be there
        self.assertTrue(os.path.exists(changes))

    def test_logfile(self):
        '''--log-file option'''

        p = self.build_src('Tests: nz\nDepends: coreutils\nRestrictions: build-needed\n',
                           {'nz': '#!/bin/sh\n./test_built\necho I am sick >&2\nexit 7'})

        logfile = os.path.join(self.workdir, 'test.log')
        (code, out, err) = self.runtest(['-d', '--no-built-binaries',
                                         '--unbuilt-tree=' + p,
                                         '--log-file=' + logfile])
        with open(logfile) as f:
            log = f.read()

        # test should fail
        self.assertEqual(code, 4, err)
        self.assertRegex(out, 'nz\s+FAIL non-zero exit status 7')
        self.assertRegex(log, 'nz\s+FAIL non-zero exit status 7')

        self.assertIn('processing dependency coreutils', err)
        self.assertIn('processing dependency coreutils', log)

        # should build package
        self.assertIn('dh build', err)
        self.assertIn('dh build', log)

        # should show test stdout
        self.assertRegex(out, '(^|\n)built script OK\n')
        self.assertRegex(log, '(^|\n)built script OK\n')
        # should show test stderr
        self.assertRegex(err, 'stderr [ -]+\nI am sick\n')
        self.assertRegex(log, 'stderr [ -]+\nI am sick\n')

    def test_tree_output_dir(self):
        '''source tree, explicit --output-dir'''

        p = self.build_src('Tests: ok\n\nTests: broken\nDepends: @, aspell-doc',
                           {'ok': '#!/bin/sh\n/usr/bin/test_built',
                            'broken': '#!/bin/sh\necho kaputt >&2'})

        outdir = os.path.join(self.workdir, 'out')

        (code, out, err) = self.runtest(['--unbuilt-tree=' + p,
                                         '--output-dir=' + outdir])

        # test results
        self.assertEqual(code, 4, err)
        self.assertRegex(out, 'ok\s+PASS', out)
        self.assertRegex(out, 'broken\s+FAIL stderr: kaputt', out)

        # should show test stdout and stderr
        self.assertRegex(out, '(^|\n)built script OK\n')
        self.assertRegex(err, 'stderr [ -]+\nkaputt', err)

        # should build package
        self.assertIn('dh build', err)

        # check outdir test stdout/err
        with open(os.path.join(outdir, 'ok-stdout')) as f:
            self.assertEqual(f.read(), 'built script OK\n')
        self.assertFalse(os.path.exists(
            os.path.join(outdir, 'ok-stderr')))
        self.assertFalse(os.path.exists(
            os.path.join(outdir, 'broken-stdout')))
        with open(os.path.join(outdir, 'broken-stderr')) as f:
            self.assertEqual(f.read(), 'kaputt\n')

        # check outdir log
        with open(os.path.join(outdir, 'log')) as f:
            contents = f.read()
        self.assertIn('dh build', contents)
        self.assertRegex(contents, 'broken\s+FAIL stderr: kaputt')

        # check summary
        with open(os.path.join(outdir, 'summary')) as f:
            self.assertRegex(f.read(), '^ok\s+PASS\nbroken\s+FAIL stderr: kaputt$')

        # check test package version
        with open(os.path.join(outdir, 'testpkg-version')) as f:
            contents = f.read()
        self.assertEqual(contents, 'testpkg 1\n')

        # check recorded package lists
        with open(os.path.join(outdir, 'testbed-packages')) as f:
            contents = f.read()
            lines = contents.splitlines()
            self.assertGreater(len(lines), 10)
            self.assertRegex(lines[0], '^[0-9a-z.-]+\t[0-9a-z.~-]+')
            self.assertRegex(lines[1], '^[0-9a-z.-]+\t[0-9a-z.~-]+')
            self.assertIn('bash\t', contents)

        # check recorded package list
        for t in ['ok', 'broken']:
            with open(os.path.join(outdir, '%s-packages' % t)) as f:
                contents = f.read()
                self.assertIn('testpkg\t1\n', contents)
                if t == 'broken':
                    self.assertIn('aspell-doc\t', contents)
                else:
                    self.assertNotIn('aspell-doc\t', contents)
                self.assertNotIn('autopkgtest-satdep', contents)

        # check binaries
        bins = os.listdir(os.path.join(outdir, 'binaries'))
        self.assertEqual(set(bins),
                         set(['Release', 'Packages', 'testpkg.deb']))

        # check testinfo
        with open(os.path.join(outdir, 'testinfo.json')) as f:
            info = json.load(f)
        u = os.uname()
        self.assertEqual(info['kernel_version'],
                         '%s %s %s' % (u.sysname, u.release, u.version.strip()))
        self.assertNotIn('test_kernel_versions', info)
        self.assertEqual(info['virt_server'], 'schroot ' + self.virt_args[1])

        # check for cruft in outdir
        self.assertEqual(set(os.listdir(outdir)),
                         set(['log', 'summary', 'binaries', 'testpkg-version',
                              'testbed-packages', 'testinfo.json',
                              'ok-packages', 'ok-stdout', 'broken-packages',
                              'broken-stderr']))

    def test_needs_recommends(self):
        '''needs-recommends restriction'''

        # check that testpkg's Recommends: apt-doc gets installed
        p = self.build_src('Tests: pass\nRestrictions: needs-recommends',
                           {'pass': '#!/bin/sh -e\n/usr/bin/test_built; '
                            'test -e /usr/share/doc/apt-doc/copyright ||'
                            'echo "apt-doc not installed!" >&2'})

        (code, out, err) = self.runtest(['--unbuilt-tree=' + p])
        self.assertEqual(code, 0, out + err)
        self.assertIn('Unpacking apt-doc', out)
        self.assertIn('built script OK', out)

    def test_needs_recommends_last(self):
        '''needs-recommends restriction in later of two tests'''

        # check that testpkg's Recommends: gets installed
        # even if there's a preceding test with the same dependencies
        # but without needs-recommends
        p = self.build_src('Tests: nr\n\nTests: wr\nRestrictions: needs-recommends',
                           {'nr': '#!/bin/sh -e\n'
                            'test ! -e /usr/share/doc/apt-doc/copyright ||'
                            'echo "apt-doc unexpectedly installed" >&2',
                            'wr': '#!/bin/sh -e\n'
                            'test -e /usr/share/doc/apt-doc/copyright ||'
                            'echo "apt-doc not installed!" >&2'})

        (code, out, err) = self.runtest(['--unbuilt-tree=' + p])
        self.assertEqual(code, 0, out + err)

    def test_needs_recommends_first(self):
        '''needs-recommends restriction in former of two tests'''

        # check that testpkg's Recommends: isn't installed without
        # needs-recommends even if there's a preceding test with the
        # same dependencies but with needs-recommends
        p = self.build_src('Tests: wr\nRestrictions: needs-recommends\n\nTests: nr\n',
                           {'wr': '#!/bin/sh -e\n'
                            'test -e /usr/share/doc/apt-doc/copyright ||'
                            'echo "apt-doc not installed!" >&2',
                            'nr': '#!/bin/sh -e\n'
                            'test ! -e /usr/share/doc/apt-doc/copyright ||'
                            'echo "apt-doc unexpectedly installed!" >&2'})

        (code, out, err) = self.runtest(['--unbuilt-tree=' + p])
        self.assertEqual(code, 0, out + err)

    def test_apt_autodep8(self):
        '''apt, autodep8'''

        (code, out, err) = self.runtest(['libtest-carp-perl'])
        if not have_autodep8:
            self.assertEqual(code, 8, err)
            self.assertIn('SKIP no tests in this package', out)
            return

        # test should succeed
        self.assertEqual(code, 0, err)
        self.assertRegex(out, 'command1\s+PASS', out)

    @unittest.skipUnless(have_autodep8,
                         'autodep8 not installed')
    def test_apt_no_autocontrol(self):
        '''apt, disable automatic test generation'''

        (code, out, err) = self.runtest(['--no-auto-control', 'libtest-carp-perl'])
        self.assertEqual(code, 8, err)
        self.assertIn('SKIP no tests in this package', out)

    def test_broken_test_deps(self):
        '''unsatisfiable test dependencies'''

        p = self.build_src('Tests: p\nDepends: unknown, libc6 (>= 99:99)\n',
                           {'p': '#!/bin/sh -e\ntrue'})

        (code, out, err) = self.runtest(['--no-built-binaries', '--unbuilt-tree=' + p])
        self.assertEqual(code, 12, err)
        self.assertRegex(out, '[dD]epends.*unknown', err)
        self.assertRegex(err, 'Test dependencies are unsatisfiable', err)
        self.assertRegex(out, 'blame: .*/testpkg', out)
        self.assertRegex(out, 'Test dependencies are unsatisfiable', out)

    def test_install_failure(self):
        '''test dependency install failure'''

        p = self.build_src('Tests: p\nDepends: @\n',
                           {'p': '#!/bin/sh -e\ntrue'})
        with open(os.path.join(p, 'debian', 'testpkg.postinst'), 'w') as f:
            f.write('#!/bin/sh -e\necho "BROKEN POSTINST"\nexit 1')

        (code, out, err) = self.runtest(['--unbuilt-tree=' + p])
        self.assertEqual(code, 12, err)
        self.assertIn('BROKEN POSTINST', out)
        self.assertRegex(out, 'blame: .*/testpkg', out)
        self.assertRegex(out, 'badpkg: Test dependencies are unsatisfiable', out)

    def do_apt_pocket_test(self, pocket_args, ver_checks,
                           add_deb_src=False, expect_success=True,
                           extra_control=''):
        '''Common test code for --apt-pocket test'''

        '''--apt-pocket with selected packages from -proposed'''

        ar = testarchive.Archive(series='testy', component='main')
        ar.add_sources('flavor', ['libflavor0', 'mint'], '1')
        ar.create_deb('libflavor0', '1')
        ar.create_deb('vanilla', '1', dependencies={'Depends': 'libflavor0'})
        ar.create_deb('chocolate', '1', dependencies={'Depends': 'libflavor0'})
        ar.create_deb('mint', '1')
        ar.create_deb('honey', '1')

        ap = testarchive.Archive(path=ar.path, pooldir='pool-proposed',
                                 series='testy-proposed', component='main')
        ap.add_sources('flavor', ['libflavor0', 'mint'], '2')
        ap.create_deb('libflavor0', '2')
        ap.create_deb('vanilla', '2', dependencies={'Depends': 'libflavor0 (>= 2)'})
        ap.create_deb('chocolate', '2', dependencies={'Depends': 'libflavor0'})
        ap.create_deb('mint', '2')
        ap.create_deb('honey', '2')

        p = self.build_src('Tests: t\nDepends: vanilla, chocolate, mint, honey\nRestrictions: allow-stderr\n\n' + extra_control,
                           {'t': '''#!/bin/sh -ex\nV=$(dpkg-query --show --showformat='${Version}' vanilla); '''
                                 '''C=$(dpkg-query --show --showformat='${Version}' chocolate); '''
                                 '''M=$(dpkg-query --show --showformat='${Version}' mint); '''
                                 '''H=$(dpkg-query --show --showformat='${Version}' honey); '''
                                 '''L=$(dpkg-query --show --showformat='${Version}' libflavor0); '''
                                 '''apt-cache policy vanilla chocolate mint honey libflavor0; ''' +
                                 ver_checks})

        apt_sources = os.path.join(self.workdir, 'sources.list')
        with open(apt_sources, 'w') as f:
            f.write('deb [trusted=yes] file:///tmp/testarchive/ testy main\n')
            if add_deb_src:
                f.write('deb-src [trusted=yes] file:///tmp/testarchive/ testy main\n')

        (code, out, err) = self.runtest(['--copy', '%s:/tmp/testarchive' % ar.path,
                                         '--copy', '%s:/etc/apt/sources.list' % apt_sources] +
                                        pocket_args +
                                        ['--apt-upgrade',
                                         '--debug', '-B', '--unbuilt-tree=' + p])
        if expect_success:
            self.assertEqual(code, 0, err)
        return (code, out, err)

    def test_apt_pocket_all(self):
        '''--apt-pocket with all packages from -proposed'''

        self.do_apt_pocket_test(['--apt-pocket=proposed'],
                                '[ "$V" = 2 ]; [ "$C" = 2 ]; [ "$M" = 2 ]; [ "$H" = 2 ]; [ "$L" = 2 ]; '
                                '[ ! -e /etc/apt/preferences.d/* ]')

    def test_apt_pocket_lib(self):
        '''--apt-pocket selecting library dep from -proposed'''

        self.do_apt_pocket_test(['--apt-pocket=proposed=libflavor0'],
                                '[ "$V" = 1 ]; [ "$C" = 1 ]; [ "$M" = 1 ]; [ "$H" = 1 ]; [ "$L" = 2 ]')

    def test_apt_pocket_selected(self):
        '''--apt-pocket selecting two packages from -proposed'''

        self.do_apt_pocket_test(['--apt-pocket=proposed=chocolate,mint,foo-udeb'],
                                '[ "$V" = 1 ]; [ "$C" = 2 ]; [ "$M" = 2 ]; [ "$H" = 1 ]; [ "$L" = 1 ]')

    def test_apt_pocket_src(self):
        '''--apt-pocket selecting all binaries from a source'''

        self.do_apt_pocket_test(['--apt-pocket=proposed=src:flavor'],
                                '[ "$V" = 1 ]; [ "$C" = 1 ]; [ "$M" = 2 ]; [ "$H" = 1 ]; [ "$L" = 2 ]',
                                add_deb_src=True)

    def test_apt_pocket_pkg_with_proposed_dep(self):
        '''--apt-pocket selecting package from -proposed with dep from -proposed'''

        (code, out, err) = self.do_apt_pocket_test(
            ['--apt-pocket=proposed=vanilla'],
            # FIXME: we really want this, but stopped working with apt 1.1's resolver
            # '[ "$V" = 2 ]; [ "$C" = 1 ]; [ "$M" = 1 ]; [ "$H" = 1 ]; [ "$L" = 2 ]')
            '[ "$V" = 2 ]; [ "$C" = 2 ]; [ "$M" = 2 ]; [ "$H" = 2 ]; [ "$L" = 2 ]')
        # FIXME: our current apt pinning does not resolve this properly, so
        # this will re-try with the entirety of -proposed
        self.assertIn('WARNING: Test dependencies are unsatisfiable with using apt pinning.', err)
        self.assertIn('Retrying with using all packages from proposed\n', err)

    def test_apt_pocket_multi_nonexisting_dep(self):
        '''--apt-pocket with multiple tests and a nonexisting dependency'''

        # include resets of the testbed here to ensure that the setup and
        # error handling work several times in a row in good and bad cases
        (code, out, err) = self.do_apt_pocket_test(
            ['--apt-pocket=proposed=chocolate'],
            '[ "$V" = 1 ]; [ "$C" = 2 ]; [ "$M" = 1 ]; [ "$H" = 1 ]; [ "$L" = 1 ]',
            expect_success=False,
            extra_control='''Test-Command: dpkg-query --show chocolate | grep 2; dpkg-query --show vanilla | grep 1
Depends: vanilla, chocolate

Test-Command: echo "not me" >&2; false
Depends: nonexisting'''
        )
        self.assertEqual(code, 12, err)
        self.assertRegex(out, 't\s+PASS', out)
        self.assertRegex(out, 'command1\s+PASS', out)
        self.assertNotIn(out, 'command2')
        self.assertIn('WARNING: Test dependencies are unsatisfiable with using apt pinning.', err)
        self.assertIn('ERROR: erroneous package: Test dependencies are unsatisfiable', err)

    def test_apt_pocket_nbs(self):
        '''--apt-pocket with only NBS binaries in unstable'''

        ar = testarchive.Archive(series='testy', component='main')
        ar.add_sources('vanilla', ['vanilla'], '1')
        ar.create_deb('vanilla', '1', dependencies={'Depends': 'libflavor0'})

        ap = testarchive.Archive(path=ar.path, pooldir='pool-proposed',
                                 series='testy-proposed', component='main')
        # version 3 source which is FTBFS (no binaries)
        ap.add_sources('flavor', ['libflavor0', 'mint'], '3')
        # version 2 binaries still stick around
        ap.create_deb('libflavor0', '2', srcpkg='flavor')
        ap.create_deb('mint', '2', srcpkg='flavor')

        apt_sources = os.path.join(self.workdir, 'sources.list')
        with open(apt_sources, 'w') as f:
            f.write('deb [trusted=yes] file:///tmp/testarchive/ testy main\n')
            f.write('deb-src [trusted=yes] file:///tmp/testarchive/ testy main\n')

        (code, out, err) = self.runtest(['--copy', '%s:/tmp/testarchive' % ar.path,
                                         '--copy', '%s:/etc/apt/sources.list' % apt_sources,
                                         '--apt-pocket=proposed',
                                         '--apt-upgrade',
                                         '--debug', 'flavor'])
        self.assertEqual(code, 12, err)

    def test_git_source_build(self):
        '''Check out and build source from git'''

        (code, out, err) = self.runtest(
            ['--env=DEB_BUILD_OPTIONS=nocheck',
             '--git-source', 'git://anonscm.debian.org/autopkgtest/autopkgtest.git'])
        # test should succeed, but skip the Lxd test (if present)
        self.assertIn(code, [0, 2], err)
        self.assertRegex(err, 'adt-run\s+PASS')
        self.assertRegex(err, 'installed\s+PASS')
        if code == 2:
            self.assertRegex(err, 'lxd\s+SKIP')


@unittest.skipUnless('AUTOPKGTEST_TEST_SCHROOT_CLICK' in os.environ,
                     'Set $AUTOPKGTEST_TEST_SCHROOT_CLICK to an existing schroot')
class SchrootClickRunner(AdtTestCase):
    def __init__(self, *args, **kwargs):
        super(SchrootClickRunner, self).__init__(
            ['schroot', os.environ.get('AUTOPKGTEST_TEST_SCHROOT_CLICK')], *args, **kwargs)
        self.click = os.path.join(test_dir, 'testclick_0.1_all.click')
        self.click_src = os.path.join(test_dir, 'testclick')
        self.setup_cmd = 'apt-get install -y --no-install-recommends click ubuntu-sdk-libs; adduser --no-create-home --system clickpkg; chown -R clickpkg:root /opt/click.ubuntu.com'

    def test_click_local_source_implicit(self):
        '''click package with local source, implicit arg type'''

        outdir = os.path.join(self.workdir, 'out')
        os.mkdir(outdir)
        sumfile = os.path.join(self.workdir, 'summary')

        (code, out, err) = self.runtest(
            ['-d', '-o', outdir, '--summary-file', sumfile,
             '--setup-commands', self.setup_cmd,
             self.click_src, self.click])
        self.assertEqual(code, 4, err)

        # one test (broken) should fail, the others succeed
        with open(sumfile) as f:
            summary = f.read()
        self.assertEqual(summary, '''broken               FAIL stderr: Bad Things!
inst                 PASS
serr                 PASS
shell                PASS
simple               PASS
''')

        # shell test should give expected output
        with open(os.path.join(outdir, 'shell-stdout')) as f:
            self.assertIn('root:x:0', f.read())
        with open(os.path.join(outdir, 'broken-stderr')) as f:
            self.assertEqual(f.read(), 'Bad Things!\n')

        # inst test has additional dependency
        with open(os.path.join(outdir, 'inst-packages')) as f:
            contents = f.read()
        self.assertTrue(contents.startswith('python3-evdev'), contents)

    def test_click_local_source_explicit(self):
        '''click package with local source, explicit arg type'''

        sumfile = os.path.join(self.workdir, 'summary')

        (code, out, err) = self.runtest(
            ['-d', '--summary-file', sumfile,
             '--setup-commands', self.setup_cmd,
             '--click-source', self.click_src, '--click', self.click])
        self.assertEqual(code, 4, err)

        # one test (broken) should fail, the others succeed
        with open(sumfile) as f:
            summary = f.read()
        self.assertEqual(summary, '''broken               FAIL stderr: Bad Things!
inst                 PASS
serr                 PASS
shell                PASS
simple               PASS
''')

    def test_click_local_source_tmp_install(self):
        '''click package with local source, temp dir deps'''

        sumfile = os.path.join(self.workdir, 'summary')

        (code, out, err) = self.runtest(
            ['-d', '--summary-file', sumfile,
             '--setup-commands', self.setup_cmd,
             '--setup-commands', os.path.join(root_dir, 'setup-commands', 'ro-apt'),
             self.click_src, self.click])
        self.assertEqual(code, 4, err)

        # one test (broken) should fail, the others succeed
        with open(sumfile) as f:
            summary = f.read()
        self.assertEqual(summary, '''broken               FAIL stderr: Bad Things!
inst                 PASS
serr                 PASS
shell                PASS
simple               PASS
''')

        self.assertIn('python3-evdev', err)

        # warn about restricted functionality
        self.assertIn('will only work for some packages', err)

    def test_click_preinstalled_all_users(self):
        '''already installed click package for all users, temp dir deps'''

        sumfile = os.path.join(self.workdir, 'summary')

        (code, out, err) = self.runtest(
            ['-d', '--summary-file', sumfile,
             '--copy', '%s:/root/testclick.click' % self.click,
             '--setup-commands', self.setup_cmd,
             '--setup-commands', 'click install --all-users /root/testclick.click',
             '--setup-commands', os.path.join(root_dir, 'setup-commands', 'ro-apt'),
             self.click_src, '--click', 'testclick'])
        self.assertEqual(code, 4, err)

        # one test (broken) should fail, the others succeed
        with open(sumfile) as f:
            summary = f.read()
        self.assertEqual(summary, '''broken               FAIL stderr: Bad Things!
inst                 PASS
serr                 PASS
shell                PASS
simple               PASS
''')

    def test_click_preinstalled_user(self):
        '''already installed click package for one user, temp dir deps'''

        sumfile = os.path.join(self.workdir, 'summary')

        (code, out, err) = self.runtest(
            ['-d', '--summary-file', sumfile,
             '--copy', '%s:/root/testclick.click' % self.click,
             '--setup-commands', self.setup_cmd,
             '--setup-commands', 'click install --user $AUTOPKGTEST_NORMAL_USER /root/testclick.click',
             '--setup-commands', os.path.join(root_dir, 'setup-commands', 'ro-apt'),
             self.click_src, '--click', 'testclick'])
        self.assertEqual(code, 4, err)

        # one test (broken) should fail, the others succeed
        with open(sumfile) as f:
            summary = f.read()
        self.assertEqual(summary, '''broken               FAIL stderr: Bad Things!
inst                 PASS
serr                 PASS
shell                PASS
simple               PASS
''')

    def test_override_control(self):
        '''custom manifest with --override-control'''

        custom_manifest = os.path.join(self.workdir, 'myctrl')
        with open(custom_manifest, 'w') as f:
            f.write('''{"name": "testclick",
   "x-test": { "simple": "tests/simple" }
}''')

        sumfile = os.path.join(self.workdir, 'summary')

        (code, out, err) = self.runtest(
            ['--summary-file', sumfile,
             '--setup-commands', self.setup_cmd,
             '--override-control', custom_manifest,
             '--click-source', self.click_src, '--click', self.click])
        self.assertEqual(code, 0, err)

        with open(sumfile) as f:
            summary = f.read()
        self.assertEqual(summary, 'simple               PASS\n')

    def test_setup_ubuntu_touch_session(self):
        '''setup-commands/ubuntu-touch-session'''

        p = self.build_src(
            'Tests: t\nDepends: x11-utils',
            {'t': '#!/bin/sh -e\nOUT=$(xprop -root); echo "$OUT" | grep -q _XKB_RULES\n'
                  'JOBS=$(/sbin/initctl  --user list)\n'
                  'echo "$JOBS" | grep -q "dbus start/running"\n'
                  'echo "$JOBS" | grep -q "click-user-hooks"\n'
                  # in environments without /dev/shm/ mounted (like schroots)
                  # this spits out "Error opening shm /lttng-ust-wait-*"
                  'ubuntu-app-list 2>/dev/null\n'
             })

        (code, out, err) = self.runtest(
            ['-B', p + '//', '--setup-commands',
             os.path.join(root_dir, 'setup-commands', 'ubuntu-touch-session')])
        self.assertEqual(code, 0, err)
        self.assertRegex(out, 't\s+PASS', out)


@unittest.skipUnless('AUTOPKGTEST_TEST_LXD' in os.environ,
                     'Set $AUTOPKGTEST_TEST_LXD to an existing container')
class SshRunnerNoScript(AdtTestCase):
    def __init__(self, *args, **kwargs):
        super(SshRunnerNoScript, self).__init__(['invalid'], *args, **kwargs)
        self.info = {}

    def start_container(self, install_key=True, sudo=False, sudo_nopwd=False):
        '''Set up container with SSH'''

        cmd = [os.path.join(test_dir, 'ssh-setup-lxd')]
        if install_key:
            cmd.append('-k')
        if sudo_nopwd:
            cmd.append('-S')
        elif sudo:
            cmd.append('-s')
        cmd += ['open', os.environ.get('AUTOPKGTEST_TEST_LXD')]
        out = subprocess.check_output(cmd, universal_newlines=True)
        for l in out.splitlines():
            (k, v) = l.split('=', 1)
            self.info[k] = v
        self.virt_args = ['ssh', '-d', '-H', self.info['hostname'], '-l', self.info['login'], '-i', self.info['identity']]

    def tearDown(self):
        if self.info:
            subprocess.call([os.path.join(test_dir, 'ssh-setup-lxd'), 'cleanup'] +
                            self.info['extraopts'].split())
            self.info = {}

    def test_no_root(self):
        '''no root'''

        self.start_container()

        p = self.build_src('Tests: pass\nDepends:\nRestrictions: allow-stderr\n\n'
                           'Tests: rootonly\nDepends:\nRestrictions: needs-root',
                           {'pass': '#!/bin/sh\necho I am fine >&2\necho babble\nwhoami',
                            'rootonly': '#!/bin/sh\necho NOTME>&2; exit 1'})

        (code, out, err) = self.runtest(['-d', '-B', p + '//'])
        self.assertEqual(code, 2, err)
        self.assertRegex(out, 'pass\s+PASS', out)
        self.assertRegex(out, 'rootonly\s+SKIP.*needs root', out)

        # should show test stdout/err
        self.assertIn('babble\nautopkgtest\n', out)
        self.assertIn('\nI am fine\n', err)
        self.assertRegex(err, 'adt-run \[[0-9: -]+\]: test pass: --')

        # no root
        self.assertIn('testbed capabilities: [', err)
        self.assertNotRegex(err, 'testbed capabilities:.*root-on-testbed')

    def test_with_root(self):
        '''with root'''

        self.start_container(sudo=True)

        p = self.build_src('Tests: pass\nDepends:\nRestrictions: needs-root',
                           {'pass': '#!/bin/sh\necho hello\nwhoami'})

        (code, out, err) = self.runtest(['-d', '-B', p + '//'],
                                        self.virt_args + ['-P', self.info['password']])
        self.assertEqual(code, 0, out + err)
        self.assertRegex(out, 'pass\s+PASS', out)

        self.assertIn('hello\nroot\n', out)

        # has root
        self.assertRegex(err, 'testbed capabilities: \[.*root-on-testbed')

    def test_passwordless_sudo(self):
        '''sudo without password'''

        self.start_container(sudo_nopwd=True)

        p = self.build_src('Tests: pass\nDepends:\nRestrictions: needs-root',
                           {'pass': '#!/bin/sh\necho hello\nwhoami'})

        (code, out, err) = self.runtest(['-d', '-B', p + '//'])
        self.assertEqual(code, 0, out + err)
        self.assertRegex(out, 'pass\s+PASS', out)

        self.assertIn('hello\nroot\n', out)

        # has root
        self.assertRegex(err, 'testbed capabilities: \[.*root-on-testbed')

    @unittest.skip('ssh password auth is not implemented')
    def test_password(self):
        '''no root, password auth'''

        self.start_container(install_key=False)

        p = self.build_src('Tests: pass\nDepends:\nRestrictions: allow-stderr',
                           {'pass': '#!/bin/sh\necho I am fine >&2\necho babble'})

        # drop "-i key" argument
        (code, out, err) = self.runtest(
            ['-d', '-B', p + '//'],
            self.virt_args[:-2] + ['-P', self.info['password']])
        self.assertEqual(code, 0, err)
        self.assertRegex(out, 'pass\s+PASS', out)

        # should show test stdout/err
        self.assertIn('babble\n', out)
        self.assertRegex(err, '-+\nI am fine\nadt-run \[[0-9: -]+\]: test pass: --')

        # no root
        self.assertIn('testbed capabilities: [', err)
        self.assertNotRegex(err, 'testbed capabilities:.*root-on-testbed')

    def test_background_process(self):
        '''leftover background process in test'''

        self.start_container(sudo_nopwd=True)

        # leak sleep processes during build and test
        p = self.build_src('Tests: bg\nDepends:\nRestrictions: build-needed',
                           {'bg': '#!/bin/bash -e\nexec -a bg_daemon sleep 30 &\n'
                                  'echo TEST DONE\n'})

        with open(os.path.join(p, 'debian', 'rules'), 'a') as f:
            f.write('\n\noverride_dh_auto_build:\n\tsleep 60 &\n'
                    '\tdh_auto_build\n')

        (code, out, err) = self.runtest(['-d', '-B', p + '//', '--timeout-build=10', '--timeout-test=5'])
        self.assertEqual(code, 0, err)
        self.assertRegex(out, 'bg\s+PASS', out)
        self.assertIn('TEST DONE\n', out)

    def test_reboot(self):
        '''test that reboots'''

        self.start_container(sudo_nopwd=True)
        p = self.build_src('Tests:r\nDepends:\nRestrictions: needs-root',
                           {'r': '''#!/bin/sh -e
        case "$AUTOPKGTEST_REBOOT_MARK" in
          mark1) echo "test in mark1"; echo one > $AUTOPKGTEST_ARTIFACTS/mark1.txt; /tmp/autopkgtest-reboot mark2 ;;
          mark2) echo "test in mark2"; echo two > $AUTOPKGTEST_ARTIFACTS/mark2.txt ;;
          "") echo "test beginning"; echo zero > $AUTOPKGTEST_ARTIFACTS/begin.txt; /tmp/autopkgtest-reboot mark1 ;;
          *) echo "unknown AUTOPKGTEST_REBOOT_MARK value $AUTOPKGTEST_REBOOT_MARK" >&2; exit 1 ;;
        esac
        echo "test end"'''})

        outdir = os.path.join(self.workdir, 'out')
        os.mkdir(outdir)

        (code, out, err) = self.runtest(['-d', '-o', outdir, '-B', p + '//'],
                                        self.virt_args + ['--reboot'])

        # test should succeed
        self.assertEqual(code, 0, err)
        self.assertRegex(out, 'r\s+PASS', out)

        # should have all three phases
        self.assertIn('test beginning\ntest in mark1\ntest in mark2\ntest end\n', out)
        self.assertIn('test process requested reboot with marker mark1', err)
        self.assertIn('test process requested reboot with marker mark2', err)

        # check artifacts
        with open(os.path.join(outdir, 'artifacts', 'begin.txt')) as f:
            self.assertEqual(f.read(), 'zero\n')
        with open(os.path.join(outdir, 'artifacts', 'mark1.txt')) as f:
            self.assertEqual(f.read(), 'one\n')
        with open(os.path.join(outdir, 'artifacts', 'mark2.txt')) as f:
            self.assertEqual(f.read(), 'two\n')

    def test_reboot_prepare(self):
        '''test that reboots by itself'''

        self.start_container(sudo_nopwd=True)
        p = self.build_src('Tests: s r\nDepends:\nRestrictions: needs-root',
                           {'s': '#!/bin/sh\necho simple_done',
                            'r': '''#!/bin/sh -e
        case "$AUTOPKGTEST_REBOOT_MARK" in
          "")
              echo "test beginning"
              echo zero > $AUTOPKGTEST_ARTIFACTS/begin.txt
              /tmp/autopkgtest-reboot-prepare mark1
              touch /var/post-prepare-1
              reboot ;;
          mark1)
              echo "test in mark1"
              echo one > $AUTOPKGTEST_ARTIFACTS/mark1.txt
              /tmp/autopkgtest-reboot-prepare mark2
              touch /var/post-prepare-2
              reboot ;;
          mark2)
              echo "test in mark2"
              echo two > $AUTOPKGTEST_ARTIFACTS/mark2.txt
              ls /var/post-prepare-1 /var/post-prepare-2 ;;
          *)
              echo "unknown AUTOPKGTEST_REBOOT_MARK value $AUTOPKGTEST_REBOOT_MARK" >&2
              exit 1 ;;
        esac'''})

        outdir = os.path.join(self.workdir, 'out')
        os.mkdir(outdir)

        (code, out, err) = self.runtest(['-d', '-o', outdir, '-B', p + '//'],
                                        self.virt_args + ['--reboot'])

        # test should succeed
        self.assertEqual(code, 0, err)
        self.assertRegex(out, 's\s+PASS', out)
        self.assertRegex(out, 'r\s+PASS', out)

        # should have all three phases
        self.assertIn('simple_done\n', out)
        self.assertIn('\ntest beginning\ntest in mark1\n'
                      'test in mark2\n/var/post-prepare-1\n/var/post-prepare-2\n', out)
        self.assertIn('test process requested preparation for reboot with marker mark1', err)
        self.assertIn('test process requested preparation for reboot with marker mark2', err)

        # should not have any test stderr
        self.assertNotIn(' stderr ', err)

        # check artifacts
        with open(os.path.join(outdir, 'artifacts', 'begin.txt')) as f:
            self.assertEqual(f.read(), 'zero\n')
        with open(os.path.join(outdir, 'artifacts', 'mark1.txt')) as f:
            self.assertEqual(f.read(), 'one\n')
        with open(os.path.join(outdir, 'artifacts', 'mark2.txt')) as f:
            self.assertEqual(f.read(), 'two\n')

    def test_exit_255(self):
        '''test exits with code 255

        This should not be considered a failure of the auxverb.
        '''
        self.start_container()

        p = self.build_src('Tests: t255\nDepends:\n', {'t255': '#!/bin/sh\nexit 255'})

        (code, out, err) = self.runtest(['-d', '-B', p + '//'])
        self.assertEqual(code, 4, err)
        self.assertRegex(out, 't255\s+FAIL non-zero exit status 253')
        self.assertNotIn('testbed auxverb failed with', err)


@unittest.skipUnless('AUTOPKGTEST_TEST_LXD' in os.environ,
                     'Set $AUTOPKGTEST_TEST_LXD to an existing container')
class SshRunnerWithScript(AdtTestCase, DebTestsAll):
    def __init__(self, *args, **kwargs):
        super(SshRunnerWithScript, self).__init__(
            ['ssh', '--debug', '--setup-script', os.path.join(test_dir, 'ssh-setup-lxd'),
             '--', os.environ.get('AUTOPKGTEST_TEST_LXD'), '-k', '-s'],
            *args, **kwargs)

    def test_no_root(self):
        '''no root'''

        p = self.build_src('Tests: pass\nDepends:\nRestrictions: allow-stderr\n\n'
                           'Tests: rootonly\nDepends:\nRestrictions: needs-root',
                           {'pass': '#!/bin/sh\necho I am fine >&2\necho babble\nwhoami',
                            'rootonly': '#!/bin/sh\necho NOTME>&2; exit 1'})

        # drop -s
        (code, out, err) = self.runtest(['-d', '-B', p + '//'],
                                        self.virt_args[:-1])
        self.assertEqual(code, 2, err)
        self.assertRegex(out, 'pass\s+PASS', out)
        self.assertRegex(out, 'rootonly\s+SKIP.*needs root', out)

        # should show test stdout/err
        self.assertIn('babble\nautopkgtest\n', out)
        self.assertIn('\nI am fine\n', err)
        self.assertRegex(err, 'adt-run \[[0-9: -]+\]: test pass: --')

        # no root
        self.assertIn('testbed capabilities: [', err)
        self.assertNotRegex(err, 'testbed capabilities:.*root-on-testbed')

    def test_with_root_and_revert(self):
        '''with root and revert'''

        p = self.build_src('Tests: p1\nDepends:\nRestrictions: needs-root, breaks-testbed\n\n'
                           'Tests: p2\nDepends:\n',
                           {'p1': '#!/bin/sh -e\necho hellop1\ntouch /stomp\nwhoami',
                            'p2': '#!/bin/sh -e\n[ ! -e /stomp ]\necho hellop2\nwhoami'})

        (code, out, err) = self.runtest(['-d', '-B', p + '//'])
        self.assertEqual(code, 0, out + err)
        self.assertRegex(out, 'p1\s+PASS', out)
        self.assertRegex(out, 'p2\s+PASS', out)

        self.assertIn('hellop1\nroot\n', out)
        self.assertIn('hellop2\nautopkgtest\n', out)

        # check capabilities
        self.assertRegex(err, 'testbed capabilities: \[.*root-on-testbed')
        self.assertRegex(err, 'testbed capabilities: \[.*revert-full-system')

    def test_passwordless_sudo(self):
        '''sudo without password'''

        p = self.build_src('Tests: pass\nDepends:\nRestrictions: needs-root',
                           {'pass': '#!/bin/sh\necho hello\nwhoami'})

        (code, out, err) = self.runtest(['-d', '-B', p + '//'],
                                        self.virt_args[:-1] + ['-S'])
        self.assertEqual(code, 0, out + err)
        self.assertRegex(out, 'pass\s+PASS', out)

        self.assertIn('hello\nroot\n', out)

        # has root
        self.assertRegex(err, 'testbed capabilities: \[.*root-on-testbed')

    @unittest.skip('ssh password auth is not implemented')
    def test_password(self):
        '''no root, password auth'''

        self.start_container(install_key=False)

        p = self.build_src('Tests: pass\nDepends:\nRestrictions: allow-stderr',
                           {'pass': '#!/bin/sh\necho I am fine >&2\necho babble'})

        # drop -k and -s options
        (code, out, err) = self.runtest(['-d', '-B', p + '//'], self.virt_args[:-2])
        self.assertEqual(code, 0, err)
        self.assertRegex(out, 'pass\s+PASS', out)

        # should show test stdout/err
        self.assertIn('babble\n', out)
        self.assertRegex(err, '-+\nI am fine\nadt-run \[[0-9: -]+\]: test pass: --')

        # no root
        self.assertIn('testbed capabilities: [', err)
        self.assertNotRegex(err, 'testbed capabilities:.*root-on-testbed')

    def test_apt_source(self):
        '''apt source'''

        (code, out, err) = self.runtest(['gdk-pixbuf'])
        self.assertEqual(code, 0, out + err)
        self.assertRegex(out, 'build\s+PASS', out)

    def test_click_root(self):
        '''click source, with root'''

        env = os.environ.copy()
        env['AUTOPKGTEST_CLICK_NO_FRAMEWORK_CHECK'] = '1'
        (code, out, err) = self.runtest(
            ['-d',
             '--setup-commands', 'apt-get install -y --no-install-recommends click',
             os.path.join(test_dir, 'testclick'),
             os.path.join(test_dir, 'testclick_0.1_all.click')],
            env=env)
        self.assertEqual(code, 4, err)

        # one test (broken) should fail, the others succeed
        self.assertRegex(out, 'inst\s+PASS')
        self.assertRegex(out, 'serr\s+PASS')
        self.assertRegex(out, 'shell\s+PASS')
        self.assertRegex(out, 'simple\s+PASS')
        self.assertRegex(out, 'broken\s+FAIL stderr: Bad Things!')

        self.assertIn('python3-evdev', err)

    def test_timeout(self):
        '''handling test that times out'''

        p = self.build_src('Tests: to\nDepends:\n',
                           {'to': '#!/bin/sh\necho StartTest; sleep 60; echo NotReached >&2'})

        time_start = time.time()
        (code, out, err) = self.runtest(['-d', '-B', p + '//', '--timeout-test=5'])
        duration = time.time() - time_start
        self.assertEqual(code, 4, err)
        self.assertIn("timed out on command", err)
        self.assertIn('StartTest\n', out)
        self.assertNotIn('PASS', out)
        self.assertRegex(out, 'to\s+FAIL timed out', out)
        self.assertNotIn('NotReached', err)
        self.assertLess(duration, 60, err)

    def test_ssh_failure(self):
        '''ssh failure is testbed failure'''

        p = self.build_src('Tests: f\nDepends:\n', {'f': '#!/bin/sh\npkill sshd'})

        (code, out, err) = self.runtest(['-d', '-B', p + '//'])
        self.assertEqual(code, 16, err)
        # this is a failure of the auxverb
        self.assertIn('testbed auxverb failed with exit code 255', err)
        self.assertEqual(out.strip(), '')

    def test_command_not_found(self):
        '''command-not-found failure is test failure'''

        p = self.build_src('Tests: f\nDepends:\n', {'f': '#!/bin/sh -e\nno_such_command'})

        (code, out, err) = self.runtest(['-d', '-B', p + '//'])
        self.assertEqual(code, 4, err)
        self.assertRegex(out, 'f\s+FAIL non-zero exit status 127', out)
        # this isn't a failure of the auxverb
        self.assertNotIn('testbed auxverb failed with', err)


if __name__ == '__main__':
    # Force encoding to UTF-8 even in non-UTF-8 locales.
    import io
    sys.stdout = io.TextIOWrapper(sys.stdout.detach(), encoding="UTF-8", line_buffering=True)
    unittest.main(testRunner=unittest.TextTestRunner(stream=sys.stdout, verbosity=2))
