#############################################################################
# Functions that implement steps.


#----------------------------------------------------------------------------
# This code comes from: vmdb2.py

def install_vmdb2(ctx):
    srcdir = globals()["srcdir"]
    runcmd_prepend_to_path = globals()["runcmd_prepend_to_path"]
    runcmd_prepend_to_path(ctx, srcdir)


#----------------------------------------------------------------------------
# This code comes from: lib/files.py

from datetime import datetime

import logging
import os
import re
import shutil
import time


def files_relative(filename):
    return "./" + filename


def files_create_from_embedded(ctx, embedded_file=None):
    files_make_directory(ctx, path=os.path.dirname(embedded_file) or ".")
    files_create_from_embedded_with_other_name(
        ctx, filename_on_disk=embedded_file, embedded_file=embedded_file
    )


def files_create_from_embedded_with_other_name(
    ctx, filename_on_disk=None, embedded_file=None
):
    get_file = globals()["get_file"]

    files_make_directory(ctx, path=os.path.dirname(filename_on_disk) or ".")
    filename_on_disk = files_relative(filename_on_disk)
    with open(filename_on_disk, "wb") as f:
        f.write(get_file(embedded_file))


def files_create_from_text(ctx, filename=None, text=None):
    files_make_directory(ctx, path=os.path.dirname(filename) or ".")
    filename = files_relative(filename)
    with open(filename, "w") as f:
        f.write(text)


def files_remove_file(ctx, filename=None):
    filename = files_relative(filename)
    os.remove(filename)


def files_make_directory(ctx, path=None):
    path = files_relative(path)
    if not os.path.exists(path):
        os.makedirs(path)


def files_remove_directory(ctx, path=None):
    path = files_relative(path)
    shutil.rmtree(path)


def files_remove_empty_directory(ctx, path=None):
    path = files_relative(path)
    os.rmdir(path)


def files_file_exists(ctx, filename=None):
    assert_eq = globals()["assert_eq"]
    filename = files_relative(filename)
    assert_eq(os.path.exists(filename), True)


def files_file_does_not_exist(ctx, filename=None):
    assert_eq = globals()["assert_eq"]
    filename = files_relative(filename)
    assert_eq(os.path.exists(filename), False)


def files_directory_exists(ctx, path=None):
    assert_eq = globals()["assert_eq"]
    path = files_relative(path)
    assert_eq(os.path.isdir(path), True)


def files_directory_does_not_exist(ctx, path=None):
    assert_eq = globals()["assert_eq"]
    path = files_relative(path)
    assert_eq(os.path.isdir(path), False)


def files_directory_is_empty(ctx, path=None):
    assert_eq = globals()["assert_eq"]
    path = files_relative(path)
    assert_eq(os.listdir(path), [])


def files_directory_is_not_empty(ctx, path=None):
    assert_ne = globals()["assert_ne"]
    path = files_relative(path)
    assert_ne(os.listdir(path), False)


def files_only_these_exist(ctx, filenames=None):
    assert_eq = globals()["assert_eq"]
    filenames = filenames.replace(",", "").split()
    assert_eq(set(os.listdir(".")), set(filenames))


def files_file_contains(ctx, filename=None, data=None):
    assert_eq = globals()["assert_eq"]
    filename = files_relative(filename)
    with open(filename, "rb") as f:
        actual = f.read()
        actual = actual.decode("UTF-8")
    assert_eq(data in actual, True)


def files_file_doesnt_contain(ctx, filename=None, data=None):
    assert_eq = globals()["assert_eq"]
    filename = files_relative(filename)
    with open(filename, "rb") as f:
        actual = f.read()
        actual = actual.decode("UTF-8")
    assert_eq(data in actual, False)


def files_file_matches_regex(ctx, filename=None, regex=None):
    assert_eq = globals()["assert_eq"]
    filename = files_relative(filename)
    with open(filename) as f:
        content = f.read()
    m = re.search(regex, content)
    if m is None:
        logging.debug("files_file_matches_regex: no match")
        logging.debug(f"  filenamed: {filename}")
        logging.debug(f"  regex: {regex}")
        logging.debug(f"  content: {content}")
        logging.debug(f"  match: {m}")
    assert_eq(bool(m), True)


def files_match(ctx, filename1=None, filename2=None):
    assert_eq = globals()["assert_eq"]
    filename1 = files_relative(filename1)
    filename2 = files_relative(filename2)
    with open(filename1, "rb") as f:
        data1 = f.read()
    with open(filename2, "rb") as f:
        data2 = f.read()
    assert_eq(data1, data2)


def files_do_not_match(ctx, filename1=None, filename2=None):
    assert_ne = globals()["assert_ne"]
    filename1 = files_relative(filename1)
    filename2 = files_relative(filename2)
    with open(filename1, "rb") as f:
        data1 = f.read()
    with open(filename2, "rb") as f:
        data2 = f.read()
    assert_ne(data1, data2)


def file_and_embedded_file_match(ctx, filename=None, embedded=None):
    assert_eq = globals()["assert_eq"]
    get_file = globals()["get_file"]
    filename = files_relative(filename)
    with open(filename, "rb") as f:
        data1 = f.read()
    data2 = get_file(embedded)
    assert_eq(data1, data2)


def file_and_embedded_file_do_not_match(ctx, filename=None, embedded=None):
    assert_ne = globals()["assert_ne"]
    get_file = globals()["get_file"]
    filename = files_relative(filename)
    with open(filename, "rb") as f:
        data1 = f.read()
    data2 = get_file(embedded)
    assert_ne(data1, data2)


def files_touch_with_timestamp(ctx, filename=None, mtime=None):
    t = datetime.strptime(mtime + " +0000", "%Y-%m-%d %H:%M:%S %z")
    ts = t.timestamp()
    _files_touch(filename, ts)


def files_touch(ctx, filename=None):
    _files_touch(filename, None)


def _files_touch(filename, ts):
    filename = files_relative(filename)
    if not os.path.exists(filename):
        open(filename, "w").close()
    times = None
    if ts is not None:
        times = (ts, ts)
    os.utime(filename, times=times)


def files_mtime_is_recent(ctx, filename=None):
    filename = files_relative(filename)
    st = os.stat(filename)
    age = abs(st.st_mtime - time.time())
    assert age < 1.0


def files_mtime_is_ancient(ctx, filename=None):
    filename = files_relative(filename)
    st = os.stat(filename)
    age = abs(st.st_mtime - time.time())
    year = 365 * 24 * 60 * 60
    required = 39 * year
    logging.debug(f"ancient? mtime={st.st_mtime} age={age} required={required}")
    assert age > required


def files_remember_metadata(ctx, filename=None):
    log_value = globals()["log_value"]

    meta = _files_remembered(ctx)
    meta[filename] = _files_get_metadata(filename)
    logging.debug("files_remember_metadata:")
    log_value("meta", 1, meta)
    log_value("ctx", 1, ctx.as_dict())


# Check that current metadata of a file is as stored in the context.
def files_has_remembered_metadata(ctx, filename=None):
    assert_dict_eq = globals()["assert_dict_eq"]
    log_value = globals()["log_value"]

    meta = _files_remembered(ctx)
    logging.debug("files_has_remembered_metadata:")
    log_value("meta", 1, meta)
    log_value("ctx", 1, ctx.as_dict())

    assert_dict_eq(meta[filename], _files_get_metadata(filename))


def files_has_different_metadata(ctx, filename=None):
    assert_ne = globals()["assert_ne"]
    meta = _files_remembered(ctx)
    assert_ne(meta[filename], _files_get_metadata(filename))


def _files_remembered(ctx):
    ns = ctx.declare("_files")
    return ns.get("remembered-metadata", {})


def _files_get_metadata(filename):
    filename = files_relative(filename)
    st = os.lstat(filename)
    keys = ["st_dev", "st_gid", "st_ino", "st_mode", "st_mtime", "st_size", "st_uid"]
    return {key: getattr(st, key) for key in keys}


#----------------------------------------------------------------------------
# This code comes from: lib/runcmd.py

import logging
import os
import re
import shlex
import subprocess


#
# Helper functions.
#

# Get exit code or other stored data about the latest command run by
# runcmd_run.


def _runcmd_get(ctx, name):
    ns = ctx.declare("_runcmd")
    return ns[name]


def runcmd_get_exit_code(ctx):
    return _runcmd_get(ctx, "exit")


def runcmd_get_stdout(ctx):
    return _runcmd_get(ctx, "stdout")


def runcmd_get_stdout_raw(ctx):
    return _runcmd_get(ctx, "stdout.raw")


def runcmd_get_stderr(ctx):
    return _runcmd_get(ctx, "stderr")


def runcmd_get_stderr_raw(ctx):
    return _runcmd_get(ctx, "stderr.raw")


def runcmd_get_argv(ctx):
    return _runcmd_get(ctx, "argv")


# Run a command, given an argv and other arguments for subprocess.Popen.
#
# This is meant to be a helper function, not bound directly to a step. The
# stdout, stderr, and exit code are stored in the "_runcmd" namespace in the
# ctx context.
def runcmd_run(ctx, argv, **kwargs):
    log_value = globals()["log_value"]

    ns = ctx.declare("_runcmd")

    # The Subplot Python template empties os.environ at startup, modulo a small
    # number of variables with carefully chosen values. Here, we don't need to
    # care about what those variables are, but we do need to not overwrite
    # them, so we just add anything in the env-update namespace value, and then the
    # env keyword argument, if any, to os.environ.
    env = dict(os.environ)

    for key, arg in ns.get("env-update", {}).items():
        if arg is None:
            if key in env:
                del env[key]
        else:
            env[key] = arg

    for key, arg in kwargs.pop("env", {}).items():
        env[key] = arg

    pp = ns.get("path-prefix")
    if pp:
        env["PATH"] = pp + ":" + env["PATH"]

    stdin_file = ns.get("stdin_file")
    feed = ns.get("stdin_feed", "")
    if stdin_file:
        kwargs["stdin"] = open(stdin_file)
    elif feed == "":
        kwargs["stdin"] = subprocess.DEVNULL
    else:
        kwargs["stdin"] = subprocess.PIPE

    kwargs["stdout"] = subprocess.PIPE
    kwargs["stderr"] = subprocess.PIPE

    logging.debug("runcmd_run: running command")
    log_value("argv", 1, dict(enumerate(argv)))
    log_value("kwargs:", 1, kwargs)

    p = subprocess.Popen(argv, env=env, **kwargs)
    stdout, stderr = p.communicate(input=feed.encode())

    ns["argv"] = argv
    ns["stdout.raw"] = stdout
    ns["stderr.raw"] = stderr
    ns["stdout"] = stdout.decode("utf-8")
    ns["stderr"] = stderr.decode("utf-8")
    ns["exit"] = p.returncode

    log_value("ns", 1, ns.as_dict())


# Step: prepend srcdir to PATH whenever runcmd runs a command.
def runcmd_helper_srcdir_path(ctx):
    srcdir = globals()["srcdir"]
    runcmd_prepend_to_path(ctx, srcdir)


# Step: This creates a helper script.
def runcmd_helper_script(ctx, script=None):
    get_file = globals()["get_file"]
    with open(script, "wb") as f:
        f.write(get_file(script))


#
# Step functions for stdio redirection.
#


def runcmd_stdin_from_file(ctx, file=None):
    ns = ctx.declare("_runcmd")
    ns["stdin_file"] = file


def runcmd_stdin_is(ctx, text=None):
    ns = ctx.declare("_runcmd")
    ns["stdin_feed"] = text


#
# Step functions for running commands.
#


def runcmd_prepend_to_path(ctx, dirname=None):
    ns = ctx.declare("_runcmd")
    pp = ns.get("path-prefix", "")
    if pp:
        pp = f"{pp}:{dirname}"
    else:
        pp = dirname
    ns["path-prefix"] = pp


def runcmd_step(ctx, argv0=None, args=None):
    runcmd_try_to_run(ctx, argv0=argv0, args=args)
    runcmd_exit_code_is_zero(ctx)


def runcmd_step_in(ctx, dirname=None, argv0=None, args=None):
    runcmd_try_to_run_in(ctx, dirname=dirname, argv0=argv0, args=args)
    runcmd_exit_code_is_zero(ctx)


def runcmd_try_to_run(ctx, argv0=None, args=None):
    runcmd_try_to_run_in(ctx, dirname=None, argv0=argv0, args=args)


def runcmd_try_to_run_in(ctx, dirname=None, argv0=None, args=None):
    argv = [shlex.quote(argv0)] + shlex.split(args)
    runcmd_run(ctx, argv, cwd=dirname)


#
# Step functions for examining exit codes.
#


def runcmd_exit_code_is_zero(ctx):
    runcmd_exit_code_is(ctx, exit=0)


def runcmd_exit_code_is(ctx, exit=None):
    assert_eq = globals()["assert_eq"]
    assert_eq(runcmd_get_exit_code(ctx), int(exit))


def runcmd_exit_code_is_nonzero(ctx):
    runcmd_exit_code_is_not(ctx, exit=0)


def runcmd_exit_code_is_not(ctx, exit=None):
    assert_ne = globals()["assert_ne"]
    assert_ne(runcmd_get_exit_code(ctx), int(exit))


#
# Step functions and helpers for examining output in various ways.
#


def runcmd_stdout_is(ctx, text=None):
    _runcmd_output_is(runcmd_get_stdout(ctx), text)


def runcmd_stdout_isnt(ctx, text=None):
    _runcmd_output_isnt(runcmd_get_stdout(ctx), text)


def runcmd_stderr_is(ctx, text=None):
    _runcmd_output_is(runcmd_get_stderr(ctx), text)


def runcmd_stderr_isnt(ctx, text=None):
    _runcmd_output_isnt(runcmd_get_stderr(ctx), text)


def _runcmd_output_is(actual, wanted):
    assert_eq = globals()["assert_eq"]
    log_lines = globals()["log_lines"]
    indent = " " * 4

    wanted = bytes(wanted, "utf8").decode("UTF-8")
    logging.debug("_runcmd_output_is:")

    logging.debug("  actual:")
    log_lines(indent, actual)

    logging.debug("  wanted:")
    log_lines(indent, wanted)

    assert_eq(actual, wanted)


def _runcmd_output_isnt(actual, wanted):
    assert_ne = globals()["assert_ne"]
    log_lines = globals()["log_lines"]
    indent = " " * 4

    wanted = bytes(wanted, "utf8").decode("UTF-8")
    logging.debug("_runcmd_output_isnt:")

    logging.debug("  actual:")
    log_lines(indent, actual)

    logging.debug("  wanted:")
    log_lines(indent, wanted)

    assert_ne(actual, wanted)


def runcmd_stdout_contains(ctx, text=None):
    _runcmd_output_contains(runcmd_get_stdout(ctx), text)


def runcmd_stdout_doesnt_contain(ctx, text=None):
    _runcmd_output_doesnt_contain(runcmd_get_stdout(ctx), text)


def runcmd_stderr_contains(ctx, text=None):
    _runcmd_output_contains(runcmd_get_stderr(ctx), text)


def runcmd_stderr_doesnt_contain(ctx, text=None):
    _runcmd_output_doesnt_contain(runcmd_get_stderr(ctx), text)


def _runcmd_output_contains(actual, wanted):
    assert_eq = globals()["assert_eq"]
    log_lines = globals()["log_lines"]
    indent = " " * 4

    wanted = bytes(wanted, "utf8").decode("UTF-8")
    logging.debug("_runcmd_output_contains:")

    logging.debug("  actual:")
    log_lines(indent, actual)

    logging.debug("  wanted:")
    log_lines(indent, wanted)

    assert_eq(wanted in actual, True)


def _runcmd_output_doesnt_contain(actual, wanted):
    assert_ne = globals()["assert_ne"]
    log_lines = globals()["log_lines"]
    indent = " " * 4

    wanted = bytes(wanted, "utf8").decode("UTF-8")
    logging.debug("_runcmd_output_doesnt_contain:")

    logging.debug("  actual:")
    log_lines(indent, actual)

    logging.debug("  wanted:")
    log_lines(indent, wanted)

    assert_ne(wanted in actual, True)


def runcmd_stdout_matches_regex(ctx, regex=None):
    _runcmd_output_matches_regex(runcmd_get_stdout(ctx), regex)


def runcmd_stdout_doesnt_match_regex(ctx, regex=None):
    _runcmd_output_doesnt_match_regex(runcmd_get_stdout(ctx), regex)


def runcmd_stderr_matches_regex(ctx, regex=None):
    _runcmd_output_matches_regex(runcmd_get_stderr(ctx), regex)


def runcmd_stderr_doesnt_match_regex(ctx, regex=None):
    _runcmd_output_doesnt_match_regex(runcmd_get_stderr(ctx), regex)


def _runcmd_output_matches_regex(actual, regex):
    assert_ne = globals()["assert_ne"]
    log_lines = globals()["log_lines"]
    indent = " " * 4

    r = re.compile(regex)
    m = r.search(actual)

    logging.debug("_runcmd_output_matches_regex:")
    logging.debug(f"  actual: {actual!r}")
    log_lines(indent, actual)

    logging.debug(f"  regex: {regex!r}")
    logging.debug(f"  match: {m}")

    assert_ne(m, None)


def _runcmd_output_doesnt_match_regex(actual, regex):
    assert_eq = globals()["assert_eq"]
    log_lines = globals()["log_lines"]
    indent = " " * 4

    r = re.compile(regex)
    m = r.search(actual)

    logging.debug("_runcmd_output_doesnt_match_regex:")
    logging.debug(f"  actual: {actual!r}")
    log_lines(indent, actual)

    logging.debug(f"  regex: {regex!r}")
    logging.debug(f"  match: {m}")

    assert_eq(m, None)


def runcmd_set_environment_variable(ctx, variable, value):
    ns = ctx.declare("_runcmd")
    env = ns.get("env-update", {})
    env[variable] = value


def runcmd_unset_environment_variable(ctx, variable):
    ns = ctx.declare("_runcmd")
    env = ns.get("env-update", {})
    env[variable] = None




#############################################################################
# Scaffolding for generated test program.

# import logging
import re


# Store context between steps.
class Context:
    def __init__(self):
        self._vars = {}
        self._ns = {}

    def as_dict(self):
        return dict(self._vars)

    def get(self, key, default=None):
        return self._vars.get(key, default)

    def __getitem__(self, key):
        return self._vars[key]

    def __setitem__(self, key, value):
        #        logging.debug("Context: key {!r} set to {!r}".format(key, value))
        self._vars[key] = value

    def keys(self):
        return self._vars.keys()

    def __contains__(self, key):
        return key in self._vars

    def __delitem__(self, key):
        del self._vars[key]

    def __repr__(self):
        return repr({"vars": self._vars, "namespaces": self._ns})

    def declare(self, name):
        if name not in self._ns:
            self._ns[name] = NameSpace(name)
        return self._ns[name]

    def remember_value(self, name, value):
        ns = self.declare("_values")
        if name in ns:
            raise KeyError(name)
        ns[name] = value

    def recall_value(self, name):
        ns = self.declare("_values")
        if name not in ns:
            raise KeyError(name)
        return ns[name]

    def expand_values(self, pattern):
        parts = []
        while pattern:
            m = re.search(r"(?<!\$)\$\{(?P<name>\S*)\}", pattern)
            if not m:
                parts.append(pattern)
                break
            name = m.group("name")
            if not name:
                raise KeyError("empty name in expansion")
            value = self.recall_value(name)
            parts.append(value)
            pattern = pattern[m.end() :]
        return "".join(parts)


class NameSpace:
    def __init__(self, name):
        self.name = name
        self._dict = {}

    def as_dict(self):
        return dict(self._dict)

    def get(self, key, default=None):
        if key not in self._dict:
            if default is None:
                return None
            self._dict[key] = default
        return self._dict[key]

    def __setitem__(self, key, value):
        self._dict[key] = value

    def __getitem__(self, key):
        return self._dict[key]

    def keys(self):
        return self._dict.keys()

    def __contains__(self, key):
        return key in self._dict

    def __delitem__(self, key):
        del self._dict[key]

    def __repr__(self):
        return repr(self._dict)

# Decode a base64 encoded string. Result is binary or unicode string.


import base64


def decode_bytes(s):
    return base64.b64decode(s)


def decode_str(s):
    return base64.b64decode(s).decode()

# Retrieve an embedded test data file using filename.


class Files:
    def __init__(self):
        self._files = {}

    def set(self, filename, content):
        self._files[filename] = content

    def get(self, filename):
        return self._files[filename]


_files = Files()


def store_file(filename, content):
    _files.set(filename, content)


def get_file(filename):
    return _files.get(filename)

# Check two values for equality and give an error if they are not equal
def assert_eq(a, b):
    assert a == b, "expected %r == %r" % (a, b)


# Check two values for inequality and give an error if they are equal
def assert_ne(a, b):
    assert a != b, "expected %r != %r" % (a, b)


# Check that two dict values are equal.
def _assert_dict_eq(a, b):
    for key in a:
        assert key in b, f"exected {key} in both dicts"
        av = a[key]
        bv = b[key]
        assert_eq(type(av), type(bv))
        if isinstance(av, list):
            _assert_list_eq(av, bv)
        elif isinstance(av, dict):
            _assert_dict_eq(av, bv)
        else:
            assert_eq(av, bv)
    for key in b:
        assert key in a, f"exected {key} in both dicts"


# Check that two list values are equal
def _assert_list_eq(a, b):
    assert_eq(len(a), len(b))
    for (av, bv) in zip(a, b):
        assert_eq(type(av), type(bv))
        if isinstance(av, list):
            _assert_list_eq(av, bv)
        elif isinstance(av, dict):
            _assert_dict_eq(av, bv)
        else:
            assert_eq(av, bv)


# Recursively check two dictionaries are equal
def assert_dict_eq(a, b):
    assert isinstance(a, dict)
    assert isinstance(b, dict)
    _assert_dict_eq(a, b)

import logging
import os
import tempfile


#############################################################################
# Code to implement the scenarios.


class Step:
    def __init__(self):
        self._kind = None
        self._text = None
        self._args = {}
        self._function = None
        self._cleanup = None

    def set_kind(self, kind):
        self._kind = kind

    def set_text(self, text):
        self._text = text

    def set_arg(self, name, value):
        self._args[name] = value

    def set_function(self, function):
        self._function = function

    def set_cleanup(self, cleanup):
        self._cleanup = cleanup

    def do(self, ctx):
        print("  step: {} {}".format(self._kind, self._text))
        logging.info("step: {} {}".format(self._kind, self._text))
        self._function(ctx, **self._args)

    def cleanup(self, ctx):
        if self._cleanup:
            print("  cleanup: {} {}".format(self._kind, self._text))
            logging.info("cleanup: {} {}".format(self._kind, self._text))
            self._cleanup(ctx, **self._args)


_logged_env = False


class Scenario:
    def __init__(self, ctx):
        self._title = None
        self._steps = []
        self._ctx = ctx

    def get_title(self):
        return self._title

    def set_title(self, title):
        self._title = title

    def append_step(self, step):
        self._steps.append(step)

    def run(self, datadir, extra_env):
        print("scenario: {}".format(self._title))
        logging.info("Scenario: {}".format(self._title))

        scendir = tempfile.mkdtemp(dir=datadir)
        os.chdir(scendir)
        self._set_environment_variables_to(scendir, extra_env)

        done = []
        ctx = self._ctx
        try:
            for step in self._steps:
                step.do(ctx)
                done.append(step)
        except Exception as e:
            logging.error(str(e), exc_info=True)
            for step in reversed(done):
                step.cleanup(ctx)
            raise
        for step in reversed(done):
            step.cleanup(ctx)

    def _set_environment_variables_to(self, scendir, extra_env):
        log_value = globals()["log_value"]

        overrides = {
            "SHELL": "/bin/sh",
            "HOME": scendir,
            "TMPDIR": scendir,
        }

        os.environ.update(overrides)
        os.environ.update(extra_env)
        global _logged_env
        if not _logged_env:
            _logged_env = True
            log_value("extra_env", 0, dict(extra_env))
            log_value("overrides", 0, dict(overrides))
            log_value("os.environ", 0, dict(os.environ))

import argparse
import logging
import os
import random
import shutil
import sys
import tarfile
import tempfile


class MultilineFormatter(logging.Formatter):
    def format(self, record):
        s = super().format(record)
        lines = list(s.splitlines())
        return lines.pop(0) + "\n".join("    %s" % line for line in lines)


def indent(n):
    return "  " * n


def log_value(msg, level, v):
    if is_multiline_string(v):
        logging.debug(f"{indent(level)}{msg}:")
        log_lines(indent(level + 1), v)
    elif isinstance(v, dict) and v:
        # Only non-empty dictionaries
        logging.debug(f"{indent(level)}{msg}:")
        for k in sorted(v.keys()):
            log_value(f"{k!r}", level + 1, v[k])
    elif isinstance(v, list) and v:
        # Only non-empty lists
        logging.debug(f"{indent(level)}{msg}:")
        for i, x in enumerate(v):
            log_value(f"{i}", level + 1, x)
    else:
        logging.debug(f"{indent(level)}{msg}: {v!r}")


def is_multiline_string(v):
    if isinstance(v, str) and "\n" in v:
        return True
    elif isinstance(v, bytes) and b"\n" in v:
        return True
    else:
        return False


def log_lines(prefix, v):
    if isinstance(v, str):
        nl = "\n"
    else:
        nl = b"\n"
    if nl in v:
        for line in v.splitlines(keepends=True):
            logging.debug(f"{prefix}{line!r}")
    else:
        logging.debug(f"{prefix}{v!r}")


# Remember where we started from. The step functions may need to refer
# to files there.
srcdir = os.getcwd()
print("srcdir", srcdir)

# Create a new temporary directory and chdir there. This allows step
# functions to create new files in the current working directory
# without having to be so careful.
_datadir = tempfile.mkdtemp()
print("datadir", _datadir)
os.chdir(_datadir)


def parse_command_line():
    p = argparse.ArgumentParser()
    p.add_argument("--log")
    p.add_argument("--env", action="append", default=[])
    p.add_argument("--run-all", "-k", action="store_true")
    p.add_argument("--save-on-failure")
    p.add_argument("patterns", nargs="*")
    return p.parse_args()


def setup_logging(args):
    if args.log:
        fmt = "%(asctime)s %(levelname)s %(message)s"
        datefmt = "%Y-%m-%d %H:%M:%S"
        formatter = MultilineFormatter(fmt, datefmt)

        filename = os.path.abspath(os.path.join(srcdir, args.log))
        handler = logging.FileHandler(filename)
        handler.setFormatter(formatter)
    else:
        handler = logging.NullHandler()

    logger = logging.getLogger()
    logger.addHandler(handler)
    logger.setLevel(logging.DEBUG)


def save_directory(dirname, tarname):
    print("tarname", tarname)
    logging.info("Saving {} to {}".format(dirname, tarname))
    tar = tarfile.open(tarname, "w")
    tar.add(dirname, arcname="datadir")
    tar.close()


def main(scenarios):
    args = parse_command_line()
    setup_logging(args)
    logging.info("Test program starts")

    logging.info("patterns: {}".format(args.patterns))
    if len(args.patterns) == 0:
        logging.info("Executing all scenarios")
        todo = list(scenarios)
        random.shuffle(todo)
    else:
        logging.info("Executing requested scenarios only: {}".format(args.patterns))
        patterns = [arg.lower() for arg in args.patterns]
        todo = [
            scen
            for scen in scenarios
            if any(pattern in scen.get_title().lower() for pattern in patterns)
        ]

    extra_env = {}
    for env in args.env:
        (name, value) = env.split("=", 1)
        extra_env[name] = value

    errors = []
    for scen in todo:
        try:
            scen.run(_datadir, extra_env)
        except Exception as e:
            logging.error(str(e), exc_info=True)
            errors.append((scen, e))
            if args.save_on_failure:
                print(args.save_on_failure)
                filename = os.path.abspath(os.path.join(srcdir, args.save_on_failure))
                print(filename)
                save_directory(_datadir, filename)
            if not args.run_all:
                raise

    shutil.rmtree(_datadir)

    if errors:
        sys.stderr.write(f"ERROR: {len(errors)} scenarios failed\n")
        for (scean, e) in errors:
            sys.stderr.write(f" - Scenario {scen.get_title()} failed:\n  {e}\n")
        if args.log:
            sys.stderr.write(f"Log file in {args.log}\n")
        sys.exit(1)

    print("OK, all scenarios finished successfully")
    logging.info("OK, all scenarios finished successfully")



#############################################################################
# Test data files that were embedded in the source document. Base64
# encoding is used to allow arbitrary data.


# happy.vmdb
filename = decode_str('aGFwcHkudm1kYg==')
contents = decode_bytes('c3RlcHM6Ci0gZWNobzogZm9vCiAgdGVhcmRvd246IGZvb190ZWFyZG93bgotIGVjaG86IGJhcgogIHRlYXJkb3duOiBiYXJfdGVhcmRvd24K')
store_file(filename, contents)

# j2.vmdb
filename = decode_str('ajIudm1kYg==')
contents = decode_bytes('c3RlcHM6Ci0gZWNobzogImltYWdlIGlzIHt7IG91dHB1dCB9fSIKLSBlY2hvOiBiYXIK')
store_file(filename, contents)

# unhappy.vmdb
filename = decode_str('dW5oYXBweS52bWRi')
contents = decode_bytes('c3RlcHM6Ci0gZWNobzogZm9vCiAgdGVhcmRvd246IGZvb190ZWFyZG93bgotIGVycm9yOiB5aWtlcwogIHRlYXJkb3duOiAiV0FUPyEiCi0gZWNobzogYmFyCiAgdGVhcmRvd246IGJhcl90ZWFyZG93bgo=')
store_file(filename, contents)




#############################################################################
# Classes for individual scenarios.


#----------------------------------------------------------------------------
# Scenario: A happy path
class Scenario_1():
    def __init__(self):
        ctx = Context()
        self._scenario = Scenario(ctx)
        self._scenario.set_title(decode_str('QSBoYXBweSBwYXRo'))
        
        # Step: an installed vmdb2
        step = Step()
        step.set_kind('given')
        step.set_text(decode_str('YW4gaW5zdGFsbGVkIHZtZGIy'))
        step.set_function(install_vmdb2)
        if '':
            step.set_cleanup()
        self._scenario.append_step(step)
        
        # Step: file happy.vmdb
        step = Step()
        step.set_kind('given')
        step.set_text(decode_str('ZmlsZSBoYXBweS52bWRi'))
        step.set_function(files_create_from_embedded)
        if '':
            step.set_cleanup()
        self._scenario.append_step(step)
        name = decode_str('ZW1iZWRkZWRfZmlsZQ==')
        text = decode_str('aGFwcHkudm1kYg==')
        step.set_arg(name, text)
        
        # Step: I run vmdb2 -v happy.vmdb --output=happy.img
        step = Step()
        step.set_kind('when')
        step.set_text(decode_str('SSBydW4gdm1kYjIgLXYgaGFwcHkudm1kYiAtLW91dHB1dD1oYXBweS5pbWc='))
        step.set_function(runcmd_step)
        if '':
            step.set_cleanup()
        self._scenario.append_step(step)
        name = decode_str('YXJndjA=')
        text = decode_str('dm1kYjI=')
        step.set_arg(name, text)
        name = decode_str('YXJncw==')
        text = decode_str('IC12IGhhcHB5LnZtZGIgLS1vdXRwdXQ9aGFwcHkuaW1n')
        step.set_arg(name, text)
        
        # Step: exit code is 0
        step = Step()
        step.set_kind('then')
        step.set_text(decode_str('ZXhpdCBjb2RlIGlzIDA='))
        step.set_function(runcmd_exit_code_is)
        if '':
            step.set_cleanup()
        self._scenario.append_step(step)
        name = decode_str('ZXhpdA==')
        text = decode_str('MA==')
        step.set_arg(name, text)
        
        # Step: stdout contains "foo\nbar\nbar_teardown\n"
        step = Step()
        step.set_kind('then')
        step.set_text(decode_str('c3Rkb3V0IGNvbnRhaW5zICJmb29cbmJhclxuYmFyX3RlYXJkb3duXG4i'))
        step.set_function(runcmd_stdout_contains)
        if '':
            step.set_cleanup()
        self._scenario.append_step(step)
        name = decode_str('dGV4dA==')
        text = decode_str('Zm9vCmJhcgpiYXJfdGVhcmRvd24K')
        step.set_arg(name, text)
        

    def get_title(self):
        return self._scenario.get_title()

    def run(self, datadir, extra_env):
        self._scenario.run(datadir, extra_env)

#----------------------------------------------------------------------------
# Scenario: Checking the version
class Scenario_2():
    def __init__(self):
        ctx = Context()
        self._scenario = Scenario(ctx)
        self._scenario.set_title(decode_str('Q2hlY2tpbmcgdGhlIHZlcnNpb24='))
        
        # Step: an installed vmdb2
        step = Step()
        step.set_kind('given')
        step.set_text(decode_str('YW4gaW5zdGFsbGVkIHZtZGIy'))
        step.set_function(install_vmdb2)
        if '':
            step.set_cleanup()
        self._scenario.append_step(step)
        
        # Step: I run vmdb2 --version
        step = Step()
        step.set_kind('when')
        step.set_text(decode_str('SSBydW4gdm1kYjIgLS12ZXJzaW9u'))
        step.set_function(runcmd_step)
        if '':
            step.set_cleanup()
        self._scenario.append_step(step)
        name = decode_str('YXJndjA=')
        text = decode_str('dm1kYjI=')
        step.set_arg(name, text)
        name = decode_str('YXJncw==')
        text = decode_str('IC0tdmVyc2lvbg==')
        step.set_arg(name, text)
        
        # Step: exit code is 0
        step = Step()
        step.set_kind('then')
        step.set_text(decode_str('ZXhpdCBjb2RlIGlzIDA='))
        step.set_function(runcmd_exit_code_is)
        if '':
            step.set_cleanup()
        self._scenario.append_step(step)
        name = decode_str('ZXhpdA==')
        text = decode_str('MA==')
        step.set_arg(name, text)
        
        # Step: stdout matches regex ^\\d+\\.\\d+$
        step = Step()
        step.set_kind('then')
        step.set_text(decode_str('c3Rkb3V0IG1hdGNoZXMgcmVnZXggXlxcZCtcXC5cXGQrJA=='))
        step.set_function(runcmd_stdout_matches_regex)
        if '':
            step.set_cleanup()
        self._scenario.append_step(step)
        name = decode_str('cmVnZXg=')
        text = decode_str('XlxkK1wuXGQrJA==')
        step.set_arg(name, text)
        

    def get_title(self):
        return self._scenario.get_title()

    def run(self, datadir, extra_env):
        self._scenario.run(datadir, extra_env)

#----------------------------------------------------------------------------
# Scenario: Jinja2 templating in specification file values
class Scenario_3():
    def __init__(self):
        ctx = Context()
        self._scenario = Scenario(ctx)
        self._scenario.set_title(decode_str('SmluamEyIHRlbXBsYXRpbmcgaW4gc3BlY2lmaWNhdGlvbiBmaWxlIHZhbHVlcw=='))
        
        # Step: an installed vmdb2
        step = Step()
        step.set_kind('given')
        step.set_text(decode_str('YW4gaW5zdGFsbGVkIHZtZGIy'))
        step.set_function(install_vmdb2)
        if '':
            step.set_cleanup()
        self._scenario.append_step(step)
        
        # Step: file j2.vmdb
        step = Step()
        step.set_kind('given')
        step.set_text(decode_str('ZmlsZSBqMi52bWRi'))
        step.set_function(files_create_from_embedded)
        if '':
            step.set_cleanup()
        self._scenario.append_step(step)
        name = decode_str('ZW1iZWRkZWRfZmlsZQ==')
        text = decode_str('ajIudm1kYg==')
        step.set_arg(name, text)
        
        # Step: I run vmdb2 -v j2.vmdb --output=foo.img
        step = Step()
        step.set_kind('when')
        step.set_text(decode_str('SSBydW4gdm1kYjIgLXYgajIudm1kYiAtLW91dHB1dD1mb28uaW1n'))
        step.set_function(runcmd_step)
        if '':
            step.set_cleanup()
        self._scenario.append_step(step)
        name = decode_str('YXJndjA=')
        text = decode_str('dm1kYjI=')
        step.set_arg(name, text)
        name = decode_str('YXJncw==')
        text = decode_str('IC12IGoyLnZtZGIgLS1vdXRwdXQ9Zm9vLmltZw==')
        step.set_arg(name, text)
        
        # Step: exit code is 0
        step = Step()
        step.set_kind('then')
        step.set_text(decode_str('ZXhpdCBjb2RlIGlzIDA='))
        step.set_function(runcmd_exit_code_is)
        if '':
            step.set_cleanup()
        self._scenario.append_step(step)
        name = decode_str('ZXhpdA==')
        text = decode_str('MA==')
        step.set_arg(name, text)
        
        # Step: stdout contains "image is foo.img\nbar"
        step = Step()
        step.set_kind('then')
        step.set_text(decode_str('c3Rkb3V0IGNvbnRhaW5zICJpbWFnZSBpcyBmb28uaW1nXG5iYXIi'))
        step.set_function(runcmd_stdout_contains)
        if '':
            step.set_cleanup()
        self._scenario.append_step(step)
        name = decode_str('dGV4dA==')
        text = decode_str('aW1hZ2UgaXMgZm9vLmltZwpiYXI=')
        step.set_arg(name, text)
        

    def get_title(self):
        return self._scenario.get_title()

    def run(self, datadir, extra_env):
        self._scenario.run(datadir, extra_env)

#----------------------------------------------------------------------------
# Scenario: Error handling
class Scenario_4():
    def __init__(self):
        ctx = Context()
        self._scenario = Scenario(ctx)
        self._scenario.set_title(decode_str('RXJyb3IgaGFuZGxpbmc='))
        
        # Step: an installed vmdb2
        step = Step()
        step.set_kind('given')
        step.set_text(decode_str('YW4gaW5zdGFsbGVkIHZtZGIy'))
        step.set_function(install_vmdb2)
        if '':
            step.set_cleanup()
        self._scenario.append_step(step)
        
        # Step: file unhappy.vmdb
        step = Step()
        step.set_kind('given')
        step.set_text(decode_str('ZmlsZSB1bmhhcHB5LnZtZGI='))
        step.set_function(files_create_from_embedded)
        if '':
            step.set_cleanup()
        self._scenario.append_step(step)
        name = decode_str('ZW1iZWRkZWRfZmlsZQ==')
        text = decode_str('dW5oYXBweS52bWRi')
        step.set_arg(name, text)
        
        # Step: I try to run vmdb2 -v unhappy.vmdb --output=unhappy.img
        step = Step()
        step.set_kind('when')
        step.set_text(decode_str('SSB0cnkgdG8gcnVuIHZtZGIyIC12IHVuaGFwcHkudm1kYiAtLW91dHB1dD11bmhhcHB5LmltZw=='))
        step.set_function(runcmd_try_to_run)
        if '':
            step.set_cleanup()
        self._scenario.append_step(step)
        name = decode_str('YXJndjA=')
        text = decode_str('dm1kYjI=')
        step.set_arg(name, text)
        name = decode_str('YXJncw==')
        text = decode_str('IC12IHVuaGFwcHkudm1kYiAtLW91dHB1dD11bmhhcHB5LmltZw==')
        step.set_arg(name, text)
        
        # Step: exit code is 1
        step = Step()
        step.set_kind('then')
        step.set_text(decode_str('ZXhpdCBjb2RlIGlzIDE='))
        step.set_function(runcmd_exit_code_is)
        if '':
            step.set_cleanup()
        self._scenario.append_step(step)
        name = decode_str('ZXhpdA==')
        text = decode_str('MQ==')
        step.set_arg(name, text)
        
        # Step: stdout contains "foo\nyikes\n"
        step = Step()
        step.set_kind('then')
        step.set_text(decode_str('c3Rkb3V0IGNvbnRhaW5zICJmb29cbnlpa2VzXG4i'))
        step.set_function(runcmd_stdout_contains)
        if '':
            step.set_cleanup()
        self._scenario.append_step(step)
        name = decode_str('dGV4dA==')
        text = decode_str('Zm9vCnlpa2VzCg==')
        step.set_arg(name, text)
        
        # Step: stdout contains "WAT?!\n"
        step = Step()
        step.set_kind('then')
        step.set_text(decode_str('c3Rkb3V0IGNvbnRhaW5zICJXQVQ/IVxuIg=='))
        step.set_function(runcmd_stdout_contains)
        if '':
            step.set_cleanup()
        self._scenario.append_step(step)
        name = decode_str('dGV4dA==')
        text = decode_str('V0FUPyEK')
        step.set_arg(name, text)
        
        # Step: stdout contains "foo_teardown\n"
        step = Step()
        step.set_kind('then')
        step.set_text(decode_str('c3Rkb3V0IGNvbnRhaW5zICJmb29fdGVhcmRvd25cbiI='))
        step.set_function(runcmd_stdout_contains)
        if '':
            step.set_cleanup()
        self._scenario.append_step(step)
        name = decode_str('dGV4dA==')
        text = decode_str('Zm9vX3RlYXJkb3duCg==')
        step.set_arg(name, text)
        
        # Step: stdout doesn't contain "bar_step"
        step = Step()
        step.set_kind('then')
        step.set_text(decode_str('c3Rkb3V0IGRvZXNuJ3QgY29udGFpbiAiYmFyX3N0ZXAi'))
        step.set_function(runcmd_stdout_doesnt_contain)
        if '':
            step.set_cleanup()
        self._scenario.append_step(step)
        name = decode_str('dGV4dA==')
        text = decode_str('YmFyX3N0ZXA=')
        step.set_arg(name, text)
        
        # Step: stdout contains "bar_teardown"
        step = Step()
        step.set_kind('then')
        step.set_text(decode_str('c3Rkb3V0IGNvbnRhaW5zICJiYXJfdGVhcmRvd24i'))
        step.set_function(runcmd_stdout_contains)
        if '':
            step.set_cleanup()
        self._scenario.append_step(step)
        name = decode_str('dGV4dA==')
        text = decode_str('YmFyX3RlYXJkb3du')
        step.set_arg(name, text)
        

    def get_title(self):
        return self._scenario.get_title()

    def run(self, datadir, extra_env):
        self._scenario.run(datadir, extra_env)


_scenarios = { 
    Scenario_1(),
    Scenario_2(),
    Scenario_3(),
    Scenario_4(),
}


#############################################################################
# Call main function and clean up.
main(_scenarios)
