#!/usr/bin/env python3
# -*- coding: utf-8 -*-

#***********************************************************************
# This file is part of OpenMolcas.                                     *
#                                                                      *
# OpenMolcas is free software; you can redistribute it and/or modify   *
# it under the terms of the GNU Lesser General Public License, v. 2.1. *
# OpenMolcas is distributed in the hope that it will be useful, but it *
# is provided "as is" and without any express or implied warranties.   *
# For more details see the full text of the license in the file        *
# LICENSE or in <http://www.gnu.org/licenses/>.                        *
#                                                                      *
# Copyright (C) 2000-2012, Valera Veryazov                             *
#               2015, Steven Vancoillie                                *
#               2017-2020, Ignacio Fdez. Galván                        *
#***********************************************************************
#
# verify
#
# Automated test verification for Molcas.
# For usage documentation read the help subroutines.
#
# Author:  Valera Veryazov                                                    
#          Dept. of Theoretical Chemistry                                     
#          Lund, Sweden                                                       
# Written: 2000-2012                                                          
#
# Re-written:
# Steven Vancoillie, summer 2015
#
# Support for two test directories
# Ignacio Fdez. Galván, February-March 2017
#
# Further updates for better interaction with dailymerge
# Ignacio Fdez. Galván, June 2017
#
# Add --fromfile option
# Ignacio Fdez. Galván, May 2018
#
# Add --timest and --validate options
# Ignacio Fdez. Galván, August-September 2019
#
# Add --grep option
# Ignacio Fdez. Galván, September 2020
#
# Translate from Perl to Python, include updatetest and chkunprint
# Ignacio Fdez. Galván, November 2020

import sys
import os
import subprocess as sp
import re

import locale
import signal
import datetime
import time
import argparse
import shutil
from collections import OrderedDict

opt = {'quiet': False}

#---------------
# output control
#---------------

def print_stdout(text):
  sys.stdout.write(text.encode())

def msg(text):
  if not opt['quiet']:
    print_stdout(text)

def msg_nl(lines):
  if not opt['quiet']:
    for line in lines:
      print_stdout(line)
      print_stdout('\n')

def msg_list(prefix, items):
  max_items = 10
  counter = 0
  padding = 4*' '
  msg_nl([prefix])
  for item in items:
    if counter == 0:
      msg(padding)
    msg(item)
    counter += 1
    if counter == max_items:
      msg('\n')
      counter = 0
  if counter:
    msg('\n')

def msg_overwrite(text, erase):
  l = len(erase)
  backup = l * '\b'
  spaces = l * ' '
  msg(backup + spaces + backup + text)

def warning(text):
  if not opt['quiet']:
    print(text, file=sys.stderr)

def error(text):
  warning(text)
  sys.exit(1)

#-------------------------------------------------------
# helper functions to deal with the group subdirectories
#-------------------------------------------------------

def list_groups():
  groups = {}
  vgroups = {}
  for TESTDIR in TESTDIRS:
    try:
      subdir_names = os.listdir(TESTDIR)
    except:
      pass
    else:
      for subdir_name in subdir_names:
        if subdir_name in ['log', 'failed', 'tmp']:
          continue
        subdir = os.path.join(TESTDIR, subdir_name)
        if not os.path.isdir(subdir):
          continue
        if subdir_name in groups:
          warning('duplicate directory "{}" at {} will be ignored'.format(subdir_name, TESTDIR))
          continue
        groups[subdir_name] = os.path.join(TESTDIR, subdir_name)
        try:
          dotfiles = os.listdir(subdir)
        except:
          pass
        else:
          for dotfile in dotfiles:
            path = os.path.join(subdir, dotfile)
            if not (os.path.isfile(path) and dotfile.startswith('.')):
              continue
            vgroups[dotfile] = True
  maxlength = 0
  # special groups
  vgroups['.none'] = 1
  vgroups['.everything'] = 1
  for key in groups:
    maxlength = max(maxlength, len(key))
  for key in vgroups:
    maxlength = max(maxlength, len(key))
  fmt = '  {{:{}}} = {{}}\n'.format(maxlength)
  msg('\nPhysical groups (directories)\n\n')
  for key in sorted(groups, key=by_type_first):
    group = vgroup_to_groups(key)
    msg(fmt.format(key, location[key]))
  msg('\nSpecial groups\n\n')
  for key in ['.none', '.everything']:
    group = vgroup_to_groups(key)
    group = [i for i in group if i in groups]
    msg(fmt.format(key, ' '.join(sorted(group, key=by_type_first))))
  msg('\nVirtual groups\n\n')
  for key in sorted(vgroups):
    if key in ['.none', '.everything']:
      continue
    group = vgroup_to_groups(key)
    group = [i for i in group if i in groups]
    msg(fmt.format(key, ' '.join(sorted(group, key=by_type_first))))

def vgroup_to_groups(vgroup):
  # convert virtual group name from task to array of real group names:
  #   standard  -> (standard)
  #   .critical -> (standard, additional)
  global location
  groups = []
  if vgroup.startswith('.'):
    for TESTDIR in TESTDIRS:
      try:
        subdir_names = os.listdir(TESTDIR)
      except:
        pass
      else:
        for subdir_name in subdir_names:
          subdir = os.path.join(TESTDIR, subdir_name)
          if not os.path.isdir(subdir):
            continue
          if location.get(subdir_name, subdir) != subdir:
            continue
          # special groups ".none" and ".everything" don't rely on dot files
          if vgroup == '.none':
            continue
          if vgroup == '.everything' or os.path.isfile(os.path.join(subdir, vgroup)):
            groups.append(subdir_name)
            location[subdir_name] = subdir
  else:
    for TESTDIR in TESTDIRS:
      if os.path.isdir(os.path.join(TESTDIR, vgroup)):
        groups.append(vgroup)
        location[vgroup] = os.path.join(TESTDIR, vgroup)
    if vgroup not in location:
      warning('non-existing group: {}'.format(vgroup))
  return groups

def add_names_from_group_to_dictionary(group, name_list, dictionary):
  for name in name_list:
    dictionary['{}:{}'.format(group, name)] = True

def flatten_numbers(numbers):
  # convert task list to flat list of numbers:
  #   002-004 -> (002, 003, 004)
  start, end = re.match(r'^(\d{3})(?:-(\d{3}))?$', numbers).groups()
  if end:
    flattened = ['{:03d}'.format(i) for i in range(int(start), int(end)+1)]
  else:
    flattened = [start]
  return flattened

def find_all_names_from_group(group):
  # convert a group name into a list of all test names in that group:
  #   standard -> (000, 001, ..., 099) + any input file basenames
  directory = location[group]
  try:
    inputs = os.listdir(directory)
  except:
    error('cannot open directory {} for group {}'.format(directory, group))
  names = []
  for inputfile in inputs:
    name, ext = os.path.splitext(inputfile)
    if ext == '.input':
      names.append(name)
  return names

def special_failure(filename):
  # function to check unprintable chars in a file (previously chkunprint)
  segfault = re.compile(b'Segmentation *Vio')
  controlchars = re.compile(b'[\x00-\x08\x0E-\x1F]')
  extendedchars = re.compile(b'[\xA0-\xFF].*[\xA0-\xFF].*[\xA0-\xFF]')
  failure = ''
  try:
    with open(filename, 'rb') as f:
      for line in f:
        if segfault.search(line):
          failure = 'segfault'
          break
        if controlchars.search(line) or extendedchars.search(line):
          failure = 'garbage'
          break
  except:
    error('failure checking for unprintable characters')
  return failure

def failed_module(filename):
  mod = ''
  with open(filename, 'rb') as f:
    for line in f:
      if not re.search(rb'(Start|Stop) Module', line):
        continue
      if re.search(rb'check|auto', line):
        continue
      mod = re.search(rb'(?:Start|Stop) Module: +([^ ]+)', line).group(1).decode()
  if not mod:
    mod = '[none]'
  return mod

def by_standard_first(item):
  # routine to sort keys starting with "standard"
  if item.startswith('standard'):
    return '0' + item
  else:
    return '1' + item

def by_type_first(item):
  # routine to sort keys according to the groups
  group = item.split(':')[0]
  if group in basic:
    return '1' + item
  elif group in critical:
    return '2' + item
  elif group in official:
    return '3' + item
  else:
    return '4' + item

#-----------------------------------------
# functions originally in separate modules
#-----------------------------------------

# Function that appends generated checkfile to a test input.
# First argument is the test input file and second is the
# checkfile
#
# A checkfile is imbedded inside a test file using the EMIL
# file command:
# >>FILE checkfile
# ... checkfile contents ...
# >>EOF
# We just replace the contents with the provided checkfile, or,
# if the ">>FILE checkfile" line is not found it will be added.
def updatetest(inputfile, checkfile):
  # read and store the first section of the input
  try:
    input_lines = []
    with open(inputfile, 'r', encoding='utf-8') as f:
      for line in f:
        if line.rstrip() == '>>FILE checkfile':
          break
        input_lines.append(line)
  except IOError as e:
    warning('cannot open test input: {}'.format(e))
    return 1
  # write the first section of the input and the contents of the
  # checkfile back to the input file.
  try:
    with open(inputfile, 'w+', encoding='utf-8') as fi, open(checkfile, 'r', encoding='utf-8') as fc:
      for line in input_lines:
        fi.write(line)
      fi.write('>>FILE checkfile\n')
      for line in fc:
        fi.write(line)
      fi.write('>>EOF\n')
  except IOError as e:
    warning('cannot open file: {}'.format(e))
    return 2
  return 0

# use proper locale
locale.setlocale(locale.LC_ALL, 'C')

# disable stdout buffering
# (note that we should use print_stdout instead of print)
sys.stdout = os.fdopen(sys.stdout.fileno(), 'wb', 0)

# my name
thisfile = os.path.basename(__file__)

# trap interrupt
active_pid = None
def killexit(*args):
  try:
    os.kill(-active_pid, signal.SIGKILL)
  except:
    pass
  error('\nSTOP: user has terminated {}'.format(thisfile))
signal.signal(signal.SIGINT, killexit)

starting_cwd = os.getcwd()

MOLCAS_DRIVER = os.environ.get('MOLCAS_DRIVER', 'pymolcas')
DRIVER_base = os.path.basename(MOLCAS_DRIVER)

def short_help():
  msg('''
{} {} [--keep|-k] [--debug|-d] [--list|-l] [--generate] [task]

where: task is a group name (standard, additional, benchmark,
grayzone, failed) followed by colon and then a comma-separated
list of numbers and/or ranges, e.g.: standard:000,005-121,-014

use the long option --help for a complete description!
'''.format(DRIVER_base, thisfile))

epilog='''
task:
  Tests are divided into different groups: e.g.
  standard, additional, benchmark, grayzone, ...
  These groups are subdirectories of the test/
  directory. If the group name begins with a dot,
  it is a virtual group which consists of all
  subdirectories which have a file of that name.

  To specify which tests to run, you need to specify
  a group, optionally followed by a colon with a list
  or range of test numbers. Numbers or ranges preceded
  by a '-' sign are excluded. When no numbers or ranges
  are given, all tests from the group are included. When
  only a '-' is give, all tests are excluded. When no
  group is specified, a default group is selected
  (i.e. the virtual '.default' group, which consists of
  'standard', 'additional', and 'grayzone').

  empty: run all tests from group '.default'
  a list of tests (group:nr1[,nr2,...]):
    000 (same as standard:000)
  a range of tests (two numbers separated by dash):
    005-134
  a combination of the above:
    003,005-009,054
  an additional exclude list:
    001-010,-004 (tests 1 to 10 except 4)
  a group (subdirectory), specified by name:
    standard
    additional
    performance
    benchmark
    grayzone
    ...
  a group (virtual), specified by name:
    .basic
    .default
    .critical
    .all
  a group followed by a colon and then any of the above
  lists of numbers/ranges:
    performance:000-100,-005-009,-043
  a file:
    path/to/file1.input

Individual tasks can be combined in any way as long as they
are separated by a space (you can mix e.g. number tasks and
files)

Examples:
  {driver} {script}                   - run default tests (standard, additional, grayzone)
  {driver} {script} .all              - run all tests (but not performance, benchmark)
  {driver} {script} .everything       - run _all_ tests (yes, ALL tests)
  {driver} {script} standard          - run all standard tests
  {driver} {script} performance       - run performance tests (ca. 30 min)
  {driver} {script} benchmark         - run benchmark tests (several hours)
  {driver} {script} -m caspt2         - run default tests that contain &caspt2 module
  {driver} {script} -w ksdft          - run default tests with 'ksdft' word
  {driver} {script} 001-005,-003      - run tests 001 to 005 but not 003
  {driver} {script} standard:050-070  - run standard tests from 050 to 070
  {driver} {script} .all standard:-   - run all tests except those from standard
  {driver} {script} --failed          - run (default) tests that failed the last time
  {driver} {script} -d 000            - run standard test 000 with output on the screen
'''.format(driver=DRIVER_base, script=thisfile)

# command-line options
try:
  parser = argparse.ArgumentParser(add_help=False, usage='{} {} [options] [task [task ...]]'.format(DRIVER_base, thisfile), epilog=epilog, formatter_class=argparse.RawDescriptionHelpFormatter)
  parser.add_argument('task', nargs='*', help=argparse.SUPPRESS)
  parser.add_argument('-h', action='store_true', help='print short help')
  parser.add_argument('--help', action='store_true', help='print long help (you\'re reading it)')
  parser.add_argument('--clean', action='store_true', help='clean up log directory before run')
  parser.add_argument('--cover', action='store_true', help='generate code coverage report (WARNING: can take a long time!')
  parser.add_argument('--cycles', type=int, help='cycle each test N times', metavar='N', default=1)
  parser.add_argument('--debug', '-d', action='store_true', help='print output to terminal')
  parser.add_argument('--existing', action='store_true', help='use existing scratch if available')
  parser.add_argument('--failed', action='store_true', help='rerun tests that failed the last time')
  parser.add_argument('--flatlist', action='store_true', help='only list matching tests (as a flat list of tasks)')
  parser.add_argument('--fromfile', help='read tasks from file FILE, in addition to command line', metavar='FILE')
  parser.add_argument('--fuzzy', action='store_true', help='do not fail if a label is not in the reference (old behavior)')
  parser.add_argument('--generate', action='store_true', help='generate checkfile and append to input')
  parser.add_argument('--grep', help='filter test files containing WORD (anywhere but comments)', metavar='WORD')
  parser.add_argument('--grouplist', action='store_true', help='list the groups (see below)')
  parser.add_argument('--keep', '-k', action='store_true', help='keep the work directory after running a test')
  parser.add_argument('--list', '-l', action='store_true', help='only list matching tests')
  parser.add_argument('--module', '-m', help='filter test files containing MODULE', metavar='MODULE')
  parser.add_argument('--parallel', type=int, help='double cycles: one with 1 process, and one with N processes', metavar='N')
  parser.add_argument('--pass', action='store_true', help='ignore checkfile failures')
  parser.add_argument('--path', help='run with PATH as temporary directory', metavar='PATH')
  parser.add_argument('--postproc', help='postprocessing command to run before cleaning up the work directory', metavar='CMD')
  parser.add_argument('--tmp', help='run with TMP as parent scratch directory', metavar='TMP')
  parser.add_argument('--quiet', '-q', action='store_true', help='do not print any messages to the screen')
  parser.add_argument('--rawlist', action='store_true', help='only list matching tests (as a list of input files)')
  parser.add_argument('--reset', action='store_true', help='clean up any results and tmp/log directories and exit')
  parser.add_argument('--status', action='store_true', help='print status of the verification (useful for redirecting the output)')
  parser.add_argument('--timest', action='store_true', help='write time estimates (for split_tests)')
  parser.add_argument('--trap', action='store_true', help='stop immediately after a failed test')
  parser.add_argument('--validate', action='store_true', help='only validate the input')
  parser.add_argument('--word', '-w', help='filter test files containing keyword WORD', metavar='WORD')
  opt = vars(parser.parse_args(sys.argv[1:]))
except:
  short_help()
  sys.exit(1)

# do we need help?
if opt['help']:
  print_stdout(parser.format_help())
  sys.exit(0)
if opt['h']:
  short_help()
  sys.exit(0)

tasklist = []
if opt['fromfile']:
  try:
    with open(opt['fromfile'], 'r', encoding='utf-8') as f:
      tasklist = f.read().split()
  except:
    error('cannot open file {}'.format(file_list))

tasklist.extend(opt['task'])

if not tasklist:
  tasklist = ['.default']

# early parsing of options
if opt['debug']:
  opt['keep'] = True

# are we running interactively?
interactive = False
if sys.stdout.isatty() and not opt['debug'] and not opt['status']:
  interactive = True

# store environment info
try:
  MOLCAS = os.environ['MOLCAS']
except KeyError:
  sys.exit('MOLCAS not set, use {} {}'.format(DRIVER_base, thisfile))
try:
  MOLCAS_ID = sp.check_output([MOLCAS_DRIVER, 'version', '-l'], cwd=MOLCAS).decode().strip()
except:
  MOLCAS_ID = None
MACHINE = sp.check_output(['uname', '-a']).decode().strip()
date = datetime.datetime.today()
DATE = date.strftime('%c')

header = '''{}
== verification run ==
machine: {}
date: {}
'''.format(MOLCAS_ID, MACHINE, DATE)

#-------------------------------------------------
# set up global Molcas settings used for each test
#-------------------------------------------------

# Get list of test directories, adding OPENMOLCAS at the beginning
# and MOLCAS at the end, and removing duplicates
TESTDIRS = []
if os.path.exists(os.path.join(MOLCAS, 'sbin', 'find_sources')):
  OPENMOLCAS = sp.check_output('. $MOLCAS/sbin/find_sources ; echo $OPENMOLCAS_SOURCE', shell=True).decode().strip()
  TESTDIRS.append(os.path.join(OPENMOLCAS, 'test'))
if os.path.exists(os.path.join(MOLCAS, 'test', 'testdirs')):
  with open(os.path.join(MOLCAS, 'test', 'testdirs'), 'r', encoding='utf-8') as f:
    TESTDIRS.extend(f.read().split())
TESTDIRS.append(os.path.join(MOLCAS, 'test'))
TESTDIRS = list(OrderedDict.fromkeys(TESTDIRS))

if opt['path']:
  testdir = os.path.abspath(opt['path'])
else:
  testdir = os.path.join(MOLCAS, 'test')

if opt['tmp']:
  tmpdir = os.path.abspath(opt['tmp'])
else:
  tmpdir = os.path.join(testdir, 'tmp')

result  = os.path.join(testdir, 'result')
timing  = os.path.join(testdir, 'timing.data')
logroot = os.path.join(testdir, 'log')
logdir  = os.path.join(logroot, date.strftime('%F_at_%H-%M-%S'))
failed  = os.path.join(testdir, 'failed')

# reset only cleans up and then quits
if opt['reset']:
  if os.path.islink(result):
    os.remove(result)
  if os.path.islink(timing):
    os.remove(timing)
  if os.path.isdir(tmpdir):
    shutil.rmtree(tmpdir)
  if os.path.isdir(logroot):
    shutil.rmtree(logroot)
  if os.path.islink(failed):
    os.remove(failed)
  sys.exit(0)

os.environ['MOLCAS_OUTPUT'] = 'WORKDIR'
os.environ['MOLCAS_TIME'] = 'YES'
# set test type: generating or checking?
if opt['generate']:
  os.environ['MOLCAS_TEST'] = 'GENE'
  os.environ['MOLCAS_NPROCS'] = '1'
else:
  os.environ['MOLCAS_TEST'] = 'CHECK'
# enable old behavior (do not fail with extra labels)
if opt['fuzzy']:
  os.environ['MOLCAS_CHECK_FUZZY'] = 'YES'
# to ignore failures, choose negative threshold
if opt['pass']:
  os.environ['MOLCAS_THR'] = '-1'
  os.environ['MOLCAS_PASSCHECK'] = '1'
os.environ['MOLCAS_VALIDATE'] = os.environ.get('MOLCAS_VALIDATE', 'YES')
# command-line options
cli_opts = '--ignore_environment'
if opt['validate']:
  cli_opts += ' --validate'
cli_opts = cli_opts.split()

#-------------------------------------------
# generate a list of files from the tasklist
#-------------------------------------------

location = {}
filelist = []
filegroup = {}
none = False

# (these must be done early in order to use it for sorting)
# groups belonging to basic
basic = {}
basic_list = vgroup_to_groups('.basic')
for group in basic_list:
  basic[group] = True
# groups belonging to critical
critical = {}
critical_list = vgroup_to_groups('.critical')
for group in critical_list:
  critical[group] = True
# groups belonging to official
official = {}
official_list = vgroup_to_groups('.official')
for group in official_list:
  official[group] = True
# groups with no time limit
nolimit = {'performance': True, 'benchmark': True}

# First convert the tasks to actual included/skipped tests.
# These are kept in two dictionaries to allow different tasks to
# influence each other, e.g.:
#   $ molcas verify .all standard:-
# would run all the tests from virtual group '.all', but
# at the same time exclude all tests from standard.
included = {} # keeps a dictionary of included test names (group:name)
excluded = {} # keeps a dictionary of excluded test names (group:name)
for task in tasklist:
  if task == '.none':
    # the special task .none will generate a result file
    # even if there are no tests to run
    none = True
  elif os.path.isfile(task):
    # the task is a file name, add immediately to the file list
    filename = os.path.abspath(task)
    filelist.append(filename)
    filegroup[filename] = 'external'
  else:
    # A single task is composed of:
    # - an optional group name (if omitted, defaults to '.default')
    # - an optional, comma-separated lists of the following items:
    #   * 3 digits: a test number to be included
    #   * a minus sign and 3 digits: a test number to be excluded
    #   * two groups of 3 digits separated by a dash:
    #     a range of test numbers to be included
    #   * a minus sign and two groups of 3 digits separated by a dash:
    #     a range of test numbers to be excluded
    #   * empty: includes all possible test numbers
    #   * a minus sign: excludes all possible test numbers
    # All of the above is condensed into a single regex:
    vgroup, task_string = re.match(r'^([^\d:][^:]*)?:?([^:]*)$', task).groups()
    if not vgroup:
      vgroup = '.default'
    # if no subtasks, set empty string as only element (this means all tests)
    subtasks = task_string.split(',')
    # translate virtual group to list of real group names
    for group in vgroup_to_groups(vgroup):
      for subtask in subtasks:
        # decide on exclusion
        exclude_flag = False
        if subtask.startswith('-'):
          subtask = subtask[1:]
          exclude_flag = True
        # create a list of file basenames
        basenames = []
        if not subtask:
          # empty task, match every possible input file
          basenames = find_all_names_from_group(group)
        else:
          number_string = re.match(r'\d{3}(?:-\d{3})?$', subtask)
          if number_string:
            # extract name from number(range)
            basenames = flatten_numbers(number_string.group(0))
          else:
            # no number, assume subtask is a basename
            basenames.append(subtask)
        # add basenames to proper dictionary
        if exclude_flag:
          add_names_from_group_to_dictionary(group, basenames, excluded)
        else:
          add_names_from_group_to_dictionary(group, basenames, included)

# add generated test names to file list
for key in sorted(included, key=by_type_first):
  if key in excluded:
    continue
  group, number = key.split(':')
  filename = os.path.join(location[group], number + '.input')
  if os.path.isfile(filename):
    if opt['generate']:
      shutil.copyfile(filename, filename + '.bak')
    filelist.append(filename)
  filegroup[filename] = group

#-------------------------------------------------------------------------
# now that we have the filelist, we can apply filters to the file contents
#-------------------------------------------------------------------------

# build up the regex pattern to use for filtering
# this is a very simple filter: match _any_ pattern
if opt['module'] or opt['word'] or opt['grep']:
  pattern_list = []
  if opt['module']:
    for mod in opt['module'].split(','):
      pattern_list.append('&' + mod + r'\b')
  if opt['word']:
    for key in opt['word'].split(','):
      pattern_list.append(key)
  if opt['grep']:
    for key in opt['grep'].split(','):
      pattern_list.append(r'[^*]*' + key)
  filter_pattern = r'^\s*(' + '|'.join(pattern_list) + ')'
  filtered_filelist = []
  for filename in filelist:
    with open(filename, 'rb') as f:
      for line in f:
        if re.match(filter_pattern.encode(), line, re.IGNORECASE):
          filtered_filelist.append(filename)
          break
  filelist = filtered_filelist

#--------------------------------------------------------------
# if we only rerun failed tests, then filter the filenames here
#--------------------------------------------------------------

if opt['failed']:
  failed_previously = {}
  failed_list = os.path.join(failed, 'list')
  if os.path.isfile(failed_list):
    try:
      with open(failed_list, 'r', encoding='utf-8') as f:
        for line in f:
          failed_previously[line.strip()] = True
    except IOError:
      sys.exit('cannot open file {}'.failed(failed_list))
  filtered_filelist = []
  for filename in filelist:
    if filename in failed_previously:
      filtered_filelist.append(filename)
  filelist = filtered_filelist

#------------------------------------------------------
# print the final, filtered list if requested, and exit
#------------------------------------------------------

if opt['rawlist']:
  msg_nl(filelist)
  sys.exit(0)
elif opt['flatlist']:
  test_list = []
  for filename in filelist:
    name = os.path.splitext(os.path.basename(filename))[0]
    test_list.append('{}:{}'.format(filegroup[filename], name))
  msg_nl(test_list)
  sys.exit(0)
elif opt['grouplist']:
  list_groups()
  sys.exit(0)
elif opt['list']:
  msg('matching tests:\n')
  test_list = {}
  for filename in filelist:
    name = os.path.splitext(os.path.basename(filename))[0]
    if filegroup[filename] not in test_list:
      test_list[filegroup[filename]] = []
    test_list[filegroup[filename]].append(' ' + name)
  for key in sorted(test_list, key=by_type_first):
    prefix = '  {}:'.format(key)
    msg_list(prefix, sorted(test_list[key]))
  sys.exit(0)

# final check: if empty filelist, just quit nicely
if not (filelist or none):
  msg('no tests requested, bye!\n')
  sys.exit(0)

#--------------------------
# set up the infrastructure
#--------------------------

if MOLCAS_ID is None:
  error('Could not find [Open]Molcas version')

# the base directory for running tests
if not os.path.isdir(testdir):
  try:
    os.mkdir(testdir)
  except:
    error('could not create {}'.format(testdir))
if not os.access(testdir, os.W_OK | os.X_OK):
  error('cannot write to {}'.format(testdir))
if os.path.isdir(tmpdir):
  shutil.rmtree(tmpdir)
if opt['clean'] and os.path.isdir(logroot):
  shutil.rmtree(logroot)
if not os.path.isdir(tmpdir):
  os.mkdir(tmpdir)
if not os.path.isdir(logroot):
  os.mkdir(logroot)
# remove log directories not ending in .bak and older than a day
try:
  subdir_names = os.listdir(logroot)
except:
  error('cannot open directory {}'.format(logroot))
for subdir_name in subdir_names:
  if subdir_name.endswith('.bak'):
    continue
  subdir = os.path.join(logroot, subdir_name)
  if not os.path.isdir(subdir):
    continue
  age = date - datetime.datetime.fromtimestamp(os.path.getmtime(subdir))
  if age.days > 1:
    shutil.rmtree(subdir)
# finally, create the new log directory we are about to use
if os.path.isdir(logdir):
  error('existing log: {}'.format(logdir))
else:
  os.mkdir(logdir)

try:
  os.chdir(tmpdir)
except:
  error('could not switch to {}'.format(tmpdir))

# Re-generate the version information from the build that will
# actually be used to run the tests
MOLCAS_ID = sp.check_output([MOLCAS_DRIVER, 'version', '-l'], cwd=MOLCAS).decode().strip()
if opt['generate']:
  if 'dirty' in MOLCAS_ID:
    error('''\
************************************************************
Dirty [Open]Molcas installation, cannot generate check files
************************************************************
  The compiled version of molcas does not correspond
to a clean source tree (does not match a git commit).
For reproducibility's sake generation of check files
is disabled.
  Please stash your changes and recompile.''')
  else:
    os.environ['MOLCAS_INFO'] = '{}!{}!{}!'.format(MOLCAS_ID, MACHINE, DATE)

log_result = os.path.join(logdir, 'result')
log_timing = os.path.join(logdir, 'result.timing')
log_failed = os.path.join(logdir, 'failed')
log_failed_list = os.path.join(log_failed, 'list')

try:
  os.mkdir(log_failed)
except:
  error('could not create {}'.format(log_failed))

# set up links to the actual result files
if os.path.islink(result):
  os.remove(result)
if os.path.islink(timing):
  os.remove(timing)
if os.path.islink(failed):
  os.remove(failed)
os.symlink(os.path.relpath(log_result, testdir), result)
os.symlink(os.path.relpath(log_timing, testdir), timing)
os.symlink(os.path.relpath(log_failed, testdir), failed)

# open the information files
try:
  RESULT = open(log_result, 'w+', encoding='utf-8', buffering=1)
except:
  error('cannot open file {}'.format(log_result))
try:
  TIMING = open(log_timing, 'w+', encoding='utf-8', buffering=1)
except:
  error('cannot open file {}'.format(log_timing))
try:
  FAILED_LIST = open(log_failed_list, 'w+', encoding='utf-8', buffering=1)
except:
  error('cannot open file {}'.format(log_failed_list))

# start by printing headers
print(header, file=RESULT)
if opt['timest']:
  print('''\
# Automatically-generated file. Do not modify!
#
# To generate this file run {} {} {}
#
# Note that what matters is relative timings, so do not
# mix runs with different settings or environments.
'''.format(DRIVER_base, thisfile, ' '.join(sys.argv[1:])), file=TIMING)
else:
  print(header, file=TIMING)

print('''\
1: Basic tests that must pass
2: Additional tests that must pass
3: Other tests that may fail, but should be fixed
4: Personal development tests that may fail or not
''', file=RESULT)

#-------------------------------------
# loop over tests and run verification
#-------------------------------------

# counters
failed_tests = 0
failed_critical_tests = 0
skipped_tests = 0

n_tests = len(filelist)
index = 0

if opt['cover']:
  msg('WARNING: you are running tests with code coverage,\n'
      '         this can add up to 1 min of time per test\n')
  msg('running code coverage startup... ')
  process = sp.Popen([MOLCAS_DRIVER, 'codecov', '-q', '--start'])
  active_pid = process.pid
  process.wait()
  msg('done\n')

prev_group = '.none'
endloop = False

for filename in filelist:
  group = filegroup[filename]
  name = os.path.splitext(os.path.basename(filename))[0]
  project = '{}__{}'.format(group, name)
  workdir = os.path.join(tmpdir, project)
  infile = os.path.join(workdir, project + '.input')
  outfile = os.path.join(tmpdir, project + '.out')
  errfile = os.path.join(tmpdir, project + '.err')
  status = os.path.join(tmpdir, project + '.status')
  index += 1
  pc = int(index / n_tests * 100)
  mark = '4'
  if group in official:
    mark = '3'
  if group in critical:
    mark = '2'
    if group in basic:
      mark = '1'
  #----------------------------
  # set up the work environment
  #----------------------------
  os.environ['Project'] = project
  os.environ['WorkDir'] = workdir
  if group in nolimit:
    os.environ.pop('MOLCAS_TIMELIM', None)
  else:
    os.environ['MOLCAS_TIMELIM'] = os.environ.get('MOLCAS_TIMELIM', '600')
  if opt['cover']:
    process = sp.Popen([MOLCAS_DRIVER, 'codecov', '-q', '--prep'])
    active_pid = process.pid
    process.wait()
  # start repeated cycles (e.g. when tests fail randomly)
  cycles = opt['cycles']
  if opt['parallel']:
    cycles *= 2
  if group != prev_group and group != 'external':
    print('----------\ngroup {} from: {}'.format(group, location[group]), file=RESULT)
  while cycles > 0:
    cycles -= 1
    sttime = time.time()
    if opt['parallel']:
      if cycles % 2 == 1:
        os.environ['MOLCAS_NPROCS'] = 1
      else:
        os.environ['MOLCAS_NPROCS'] = opt['parallel']
    # if the work directory exists, remove it unless existing mode is used
    if os.path.isdir(workdir):
      if opt['existing']:
        # we only need to delete the check counter, leave the rest
        check_counter = os.path.join(workdir, 'molcas_check_count')
        os.remove(check_counter)
      else:
        shutil.rmtree(workdir)
    if not os.path.isdir(workdir):
      os.mkdir(workdir)
    shutil.copyfile(filename, infile)
    if interactive or opt['status']:
      prompt = 'Running test {}: {}... ({}%)'.format(group, name, pc)
      msg(prompt)
    # run pymolcas and capture the return code.
    if opt['debug']:
      process = sp.Popen([MOLCAS_DRIVER] + cli_opts + [infile], stdout=sys.stdout, stderr=sys.stderr)
    else:
      with open(outfile, 'w+', encoding='utf-8') as fo, open(errfile, 'w+', encoding='utf-8') as fe:
        process = sp.Popen([MOLCAS_DRIVER] + cli_opts + [infile], stdout=fo, stderr=fe)
    active_pid = process.pid
    rc = process.wait()
    special = False
    if not opt['debug']:
      # check for special cases where the return code might be 0
      # but we should still cause a failure (segfault, garbage)
      special = special_failure(outfile)
      if special:
        rc = 30
    result = ''
    if rc:
      cycles = 0
      if not opt['debug']:
        shutil.copyfile(outfile, os.path.join(log_failed, project + '.out'))
        shutil.copyfile(errfile, os.path.join(log_failed, project + '.err'))
      # skip the test if a program is not available (RC_NOT_AVAILABLE)
      if rc == 36:
        result = ' S'
        # print results
        print('{}:{}:{} Skipped!'.format(mark, group, name), file=RESULT)
        if interactive:
          msg_overwrite('{}:{} Skipped!\n'.format(group, name), prompt)
        if opt['status']:
          msg(' Skipped!\n')
        # update counters
        skipped_tests += 1
        # pretend to be fine for the remainder of this test
        rc = 0
      else:
        result = ' F'
        # gather extra info
        mod = ''
        if not opt['debug']:
          mod = failed_module(outfile)
        if special:
          mod = ', '.join([mod, special])
        # print results
        print('{}:{}:{} Failed! ({})'.format(mark, group, name, mod), file=RESULT)
        if interactive:
          msg_overwrite('{}:{} Failed! ({})\n'.format(group, name, mod), prompt)
        if opt['status']:
          msg(' Failed! ({})\n'.format(mod))
        # update counters
        failed_tests += 1
        if group in critical:
          failed_critical_tests += 1
        # update list of failed input files
        print(filename, file=FAILED_LIST)
    else:
      print('{}:{}:{} OK'.format(mark, group, name), file=RESULT)
      if interactive:
        msg_overwrite('', prompt)
      if opt['status']:
        msg(' OK\n')
      if opt['generate']:
        rc = updatetest(filename, os.path.join(workdir, 'checkfile'))
        if rc:
          print_stdout('rc={}\n'.format(rc))
    # save timing info
    if opt['timest']:
      runtime = time.time() - sttime
      print('--- {}:{}{}'.format(group, name, result), file=TIMING)
      print(int(round(runtime)), file=TIMING)
    elif not opt['debug']:
      print('--- Run: {}'.format(project), file=TIMING)
      try:
        with open(errfile, 'r', encoding='utf-8') as f:
          for line in f:
            print(line, end='', file=TIMING)
      except:
        error('could not open {}'.format(errfile))
    if opt['postproc']:
      os.environ['project'] = project
      sp.call(opt['postproc'], shell=True)
    # clean up
    if not (rc or opt['keep']):
      if os.path.exists(infile):
        os.remove(infile)
      if os.path.exists(outfile):
        os.remove(outfile)
      if os.path.exists(errfile):
        os.remove(errfile)
      if os.path.exists(status):
        os.remove(status)
      shutil.rmtree(workdir)
    if rc and opt['trap']:
      endloop = True
      break
  if endloop:
    break
  if opt['cover']:
    if interactive:
      prompt = 'Capturing coverage data for test {}: {}... ({}%)'.format(group, name, pc)
      msg(prompt)
    process = sp.Popen([MOLCAS_DRIVER, 'codecov', '-q', '--measure', '--name', project])
    active_pid = process.pid
    process.wait()
    if interactive:
      msg_overwrite('', prompt)
  prev_group = group
if prev_group != '.none':
  print('----------', file=RESULT)
print('\n*Failed critical tests* {}'.format(failed_critical_tests), file=RESULT)

if opt['cover']:
  process = sp.Popen([MOLCAS_DRIVER, 'codecov', '--html'])
  active_pid = process.pid
  process.wait()

RESULT.close()
TIMING.close()
FAILED_LIST.close()

if failed_tests:
  # report directory with failed out/err relative to directory where verify was run
  log_rel = os.path.relpath(failed, starting_cwd)
  tmp_rel = os.path.relpath(tmpdir, starting_cwd)
  info = '''\
************************************************************************
A total of {} test(s) failed, with {} critical failure(s).
************************************************************************
Please check the directory:
  {}
for the .out/.err files of the failed tests,
and check the submit directory:
  {}
for the working directories of the last run.
'''.format(failed_tests, failed_critical_tests, log_rel, tmp_rel)
  if failed_critical_tests:
    error(info)
  else:
    msg(info)

if opt['generate']:
  msg('Generation of check files has been completed\n')
else:
  msg('Verification has been completed\n')
