#!/usr/bin/python3
# -*- coding: utf-8 -*-
#    TOMUSS: The Online Multi User Simple Spreadsheet
#    Copyright (C) 2008-2021 Thierry EXCOFFIER, Universite Claude Bernard
#
#    This program is free software; you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation; either version 2 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program; if not, write to the Free Software
#    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
#
#    Contact: Thierry.EXCOFFIER@univ-lyon1.fr

import time
import re
import os
import sys
import traceback
import gettext
import html
import cgi
import threading
import shutil
import importlib
import ast
import urllib
import collections
import weakref
import email.utils
import email.header
import tomuss_init
from . import configuration

def read_file(filename, encoding="utf-8"):
    if encoding == "bytes":
        f = open(filename, "rb")
    else:
        f = open(filename, "r", encoding=encoding)
    c = f.read()
    f.close()
    return c

def write_file(filename, content, encoding="utf-8"):
    warn('%s : %d' % (filename, len(content)), what='debug')
    opt = 'w'
    if encoding == "bytes":
        opt = opt + 'b'
        encoding = None
    f = open(filename + '~', opt, encoding = encoding)
    f.write(content)
    f.close()
    os.rename(filename + '~', filename)

def read_url(url, timeout=10):
    try:
        with urllib.request.urlopen(url, timeout=timeout) as f:
            if f.headers.get('Content-Encoding', '') == 'gzip':
                import gzip
                c = gzip.GzipFile(fileobj=f).read()
            else:
                c = f.read()
            encoding = f.headers.get_content_charset()
            if encoding:
                c = c.decode(encoding)
            else:
                try:
                    c = c.decode('utf-8')
                except UnicodeDecodeError: # pragma: no cover
                    c = c.decode('latin-1')
    except urllib.error.HTTPError as e: # pragma: no cover
        raise IOError(str(e))
    return c

write_file_safe = write_file # DEPRECATED

lock_list = []

def add_a_lock(fct):
    """Add a lock to a function to forbid simultaneous call"""
    def f(*arg, **keys):
        warn('[[[' + f.fct.__name__ + ']]]', what='debug')
        f.the_lock.acquire()
        try:
            r = f.fct(*arg, **keys)
        finally:
            f.the_lock.release()
        return r
    f.fct = fct
    f.the_lock = threading.Lock()
    f.__doc__ = fct.__doc__
    f.__name__ = fct.__name__
    f.__module__ = fct.__module__
    lock_list.append(f)
    return f

def append_file_unlocked(filename, content):
    """Paranoid : check file size before and after append"""
    try:
        before = os.path.getsize(filename)
    except OSError:
        before = 0
    f = open(filename, 'a', encoding = "utf-8")
    f.write(content)
    f.close()
    after = os.path.getsize(filename)
    if before + len(content.encode("utf-8")) != after:
        raise IOError("Append file failed %s before=%d + %d ==> %d" % (
            filename, before, len(content), after)) # pragma: no cover

filename_to_bufferize = None
filename_buffer = []

def bufferize_this_file(filename):
    """Should be called with None to flush the buffered content"""
    global filename_to_bufferize, filename_buffer
    if filename == filename_to_bufferize:
        return # pragma: no cover
    append_file.the_lock.acquire()
    try:
        if filename_to_bufferize:
            append_file_unlocked(filename_to_bufferize,
                                 ''.join(filename_buffer))
    finally:
        filename_to_bufferize = filename
        filename_buffer = []
        append_file.the_lock.release()
    
@add_a_lock
def append_file(filename, content):
    if filename == filename_to_bufferize:
        filename_buffer.append(content)
    else:
        append_file_unlocked(filename, content)

append_file_safe = append_file # DEPRECATED

def unlink_safe(filename, do_backup=True):
    if do_backup and os.path.exists(filename):
        dirname = os.path.join('Trash', time.strftime('%Y%m%d'))
        mkpath(dirname)

        shutil.move(filename,
                    os.path.join(dirname,
                                 filename.replace(os.path.sep, '___'))
                    )
    try:
        os.unlink(filename)
    except OSError:
        pass

def rename_safe(old_filename, new_filename): # pragma: no cover
    unlink_safe(new_filename)
    os.rename(old_filename, new_filename)

symlink_safe = os.symlink # DEPRECATED
    
def safe(txt):
    return re.sub('[^0-9a-zA-Z-.@]', '_', txt)

def safe_quote(txt): # pragma: no cover
    return re.sub('[^\'0-9a-zA-Z-.@]', '_', txt)

def safe_space(txt): # pragma: no cover
    return re.sub('[^0-9a-zA-Z-. @]', '_', txt)

def safe_space_quote(txt): # pragma: no cover
    return re.sub('[^\'0-9a-zA-Z-. @]', '_', txt)

def flat(txt):
    return txt.translate("\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f ! #$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~?\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f?????Y|?????????????'u?.????????AAAAAA?CEEEEIIIIDNOOOOOXOUUUUY?Baaaaaa?ceeeeiiiionooooo??uuuuy?y")

def same(a, b): # pragma: no cover
    return flat(a).lower() == flat(b).lower()

def stable_repr(dic):
    if isinstance(dic, dict):
        t = []
        for k in sorted(dic):
            t.append("{}:{},\n".format(repr(k), repr(dic[k])))
        return '{\n' + ''.join(t) + '}'
    else:
        return '[\n' + ''.join(repr(i) + ',\n' for i in sorted(dic)) + ']'

def is_an_int(txt):
    try:
        int(txt)
        return True
    except ValueError:
        return False

def remove_first_comment(txt):
    txt = txt.split('\n')
    while txt:
        if txt[0].lstrip().startswith('//'):
            txt.pop(0)
        elif txt[0].lstrip().startswith('/*'):
            while not txt[0].rstrip().endswith('*/'):
                txt.pop(0)
            txt.pop(0)
        else:
            # coverage bug ???
            break # pragma: no cover
    return '\n'.join(txt)

def university_year(year=None, semester=None):
    if semester is None:
        semester = configuration.year_semester[1]
    if year is None:
        year = configuration.year_semester[0]
    try:
        i = configuration.semesters.index(semester)
    except ValueError:
        return year
    return year + configuration.semesters_year[i]

def university_year_semester(year=None, semester=None):
    "Return the first year+semester of the university"
    if semester is None:
        semester = configuration.year_semester[1] # pragma: no cover
    if year is None:
        year = configuration.year_semester[0] # pragma: no cover
    try:
        i = configuration.semesters.index(semester)
    except ValueError:
        return year, semester
    
    return (year + configuration.semesters_year[i],
            configuration.university_semesters[0])


def next_year_semester(year, semester):
    try:
        i = (configuration.semesters.index(semester) + 1) % len(
            configuration.semesters)
    except ValueError: # pragma: no cover
        return year + 1, semester
    if i != 0:
        return year, configuration.semesters[i]
    else:
        return year + 1, configuration.semesters[i]

def previous_year_semester(year, semester): # pragma: no cover
    try:
        i = (configuration.semesters.index(semester)
             + len(configuration.semesters) - 1) % len(
                 configuration.semesters)
    except ValueError:
        return year - 1, semester
    if i != len(configuration.semesters) - 1:
        return year, configuration.semesters[i]
    else:
        return year - 1, configuration.semesters[i]

def semester_key(year, semester):
    try:
        return year, configuration.semesters.index(semester)
    except ValueError: # pragma: no cover
        return year, semester

def year_semester_from_date(yyyymm):
    """The time can be longer"""
    month = int(yyyymm[4:6])
    year = int(yyyymm[:4])

    for s, m in zip(configuration.semesters,configuration.semesters_months):
        if m[0] <= month <= m[1]: # pragma: no cover
            return year, s
        if m[0] <= 12+month <= m[1]: # pragma: no cover
            return year-1, s

def semester_span(year, semester):
    """Semester span in seconds"""
    sem_span = configuration.semester_span(year, semester)
    if sem_span:
        first_day, last_day = sem_span.split(' ')
        return (configuration.date_to_time(first_day),
                configuration.date_to_time(last_day))
    else:
        return (0, 8000000000)

def date_to_time(date, if_exception=None):
    try:
        return configuration.date_to_time(date)
    except:
        if if_exception is None:
            raise # pragma: no cover
        else:
            return if_exception

WARNS_TO_WRITE = []
def warn(text, what='info'):
    if what in configuration.do_not_display:
        return
    x = []
    try:
        for i in range(1, 4):
            x.append(sys._getframe(i).f_code.co_name)
    except ValueError: # pragma: no cover
        pass
    x.reverse()
    x = '/'.join(x).rjust(50)[-50:]
    x = '%c %13.2f %s %s\n' % (
        what[0].upper(),
        time.time(),
        x, text)
    if threading.current_thread() is main_thread:
        while WARNS_TO_WRITE:
            sys.stderr.write(WARNS_TO_WRITE.pop(0))
        sys.stderr.write(x)
    else:
        WARNS_TO_WRITE.append(x)

@add_a_lock
def send_mail_smtp(frome, recipients, body): # pragma: no cover
    import smtplib
    if configuration.real_regtest or configuration.regtest:
        with open('xxx.mail', 'a') as f:
            f.write("\n{}\n{}\n{}\n".format(
                frome, recipients, body if isinstance(body, str) else body.decode('utf-8')))
        return
    if not configuration.smtpserver:
        print("NO SMTP SERVER DEFINED\n{}\n{}\n{}\n".format(
              frome, recipients, body if isinstance(body, str) else body.decode('utf-8')))
        return

    smtpresult = 'NotSent'
    for server in re.split(' +', configuration.smtpserver) * 2:
        try:
            try:
                smtpresult = send_mail.session.sendmail(frome, recipients, body)
                # Mail sent
                break # pragma: no cover
            except (AttributeError, # because 'session' is None the first time
                    smtplib.SMTPServerDisconnected,
                    smtplib.SMTPSenderRefused): # pragma: no cover
                send_mail.session = smtplib.SMTP(server)
                smtpresult = send_mail.session.sendmail(frome, recipients, body)
                break # Mail sent
        except smtplib.SMTPRecipientsRefused:
            # Next server
            pass # pragma: no cover
        except:
            send_backtrace('BUG from=%s\nrecipients=%s\nbody=%s' % # pragma: no cover
                           (repr(frome), repr(recipients), repr(body)))
    if smtpresult == 'NotSent':
        if isinstance(recipients, str):
            smtpresult = {'': recipients}
        else:
            smtpresult = {k: '???' for k in recipients}
    try:
        if smtpresult:
            errstr = ""
            for recip, error in smtpresult.items():
                errstr += _("MSG_utilities_smtp_error") % repr(recip) \
                          + ' ' + repr(error) + '\n\n'
            return errstr
    except: # pragma: no cover
        print(to, subject, message, file=sys.stderr)
        send_backtrace(repr(smtpresult), 'BUG send_mail')
        return 'BUG in utilities.send_mail'


@add_a_lock
def send_mail(to, subject, message, frome=None, show_to=False, reply_to=None,
              error_to=None, cc=()):
    "Not safe with user given subject"
    def encode(x):
        return email.header.Header(x.strip()).encode()
    def decode(x): # pragma: no cover
        txt, enc = email.header.decode_header(x)[0]
        return txt.decode(enc)
    def cleanup(x):
        return [encode(addr)
                for addr in x
                if addr and '@' in addr and '.' in addr
                ]

    if configuration.regtest :
        # XXX
        to = "marianne.tery@univ-lyon1.fr"
        frome = "marianne.tery@univ-lyon1.fr"
    if isinstance(to, str):
        to = [to]
    if frome == None:
        frome = configuration.maintainer # pragma: no cover

    to = cleanup(to)
    cc = cleanup(cc)
    if len(to) == 0 and len(cc) == 0:
        return # pragma: no cover
    if len(to) == 1:
        show_to = True
    header = "From: {}\n".format(encode(reply_to or frome))

    s = subject.replace('\n',' ').replace('\r',' ')
    header += "Subject: " + encode(s) + '\n'
    if show_to:
        for tto in to:
            header += "To: {}\n".format(tto)
    for tto in cc:
        header += "CC: {}\n".format(tto)
    if reply_to:
        header += 'Return-Path: {}\n'.format(encode(frome))
    if error_to:
        header += 'Error-To: {}\n'.format(encode(error_to)) # pragma: no cover
        
    if message.startswith('<html>') or message.startswith('<!DOCTYPE html>') :
        header += 'Content-Type: text/html; charset="utf-8"\n'
    else:
        header += 'Content-Type: text/plain; charset="utf-8"\n'

    header += "Date: " + email.utils.formatdate(localtime=True) + '\n'
    header += "Content-Transfer-Encoding: 8bit\n"
    header += "MIME-Version: 1.0\n"
    header += '\n'
    header = header.replace("\n", "\r\n")
    recipients = tuple(to) + tuple(cc)
    body = (header + message).encode('utf-8')
    return send_mail_smtp(frome, recipients, body)

send_mail.session = None

thread_list = []

def start_new_thread_immortal(fct, args, send_mail=True):
    start_new_thread(fct, args, send_mail=send_mail, immortal=True)

class MyThread(threading.Thread):
    def __init__(self, send_mail, immortal, fct, args):
        self.send_mail = send_mail
        self.immortal = immortal
        self.fct = fct
        self.args = args
        threading.Thread.__init__(self)
    def run(self):
        thread_list.append(self)
        warn("Start %s" % self)
        while True:
            warn('Call ' + self.fct.__name__)
            try:
                self.fct(*self.args)
            except: # pragma: no cover
                try:
                    warn("Exception in %s" % self, what="error")
                    if self.send_mail:
                        send_backtrace("Exception in %s" % self)
                except: # pragma: no cover
                    pass
            if not self.immortal:
                break
        thread_list.remove(self)
    def backtrace_html(self):
        return str(self)
    def __str__(self):
        return 'Thread immortal=%-5s send_mail=%-5s %s' % (
            self.immortal, self.send_mail, self.fct.__name__)

    def stack(self):
        return (str(self) + '\n'
                + ''.join(traceback.format_stack(
                    sys._current_frames()[self.ident])[3:]))

def start_new_thread(fct, args, send_mail=True, immortal=False):
    t = MyThread(send_mail, immortal, fct, args)
    t.setDaemon(True)
    t.start()

def stop_threads():
    sys.exit(0)
    # for t in threading.enumerate():
    #     t.join()



send_mail_in_background_list = []
def sendmail_thread():
    """Send the mail in background with a minimal time between mails"""
    errors = collections.defaultdict(list)
    while send_mail_in_background_list:
        time.sleep(configuration.time_between_mails)
        args = send_mail_in_background_list.pop(0)
        error = send_mail(*args)
        if error:
            errors[args[3]].append(error) # pragma: no cover
        t = time.time()
    for frome, errors in errors.items():
        send_mail_in_background(frome, _("MSG_mail_unknow"), '\n'.join(errors)) # pragma: no cover
    return t

def send_mail_in_background(to, subject, message, frome=None, show_to=False,
                            reply_to=None, error_to=None, cc=()):
    send_mail_in_background_list.append((to, subject, message, frome,
                                         show_to, reply_to, error_to, cc))
    start_job(sendmail_thread, 1, important='send_mail_in_background')

def js(t):
    if isinstance(t, str):
        return '"' + t.replace('\\','\\\\').replace('"','\\"').replace('>','\\x3E').replace('<','\\x3C').replace('&', '\\x26').replace('\r','').replace('\n','\\n') + '"'
    elif isinstance(t, float):
        return '%s' % t # DO NOT USE %g: 4.9999998 => 5
    elif isinstance(t, dict):
        return '{' + ','.join("%s:%s" % (js(k), js(v))
                              for k, v in t.items()) + '}'
    elif isinstance(t, tuple):
        return str(list(t))
    elif isinstance(t, set):
        raise TypeError("not sets here") # pragma: no cover
    elif isinstance(t, bytes):
        raise TypeError("not bytes here") # pragma: no cover
    else:
        return str(t)

def mkpath(path, create_init=True, mode=configuration.umask):
    s = ''
    for i in path.split(os.path.sep):
        s += i + os.path.sep
        try:
            os.mkdir(s, mode)
            if create_init:
                write_file(os.path.join(s, '__init__.py'), '',"utf-8")
        except OSError:
            pass

mkpath_safe = mkpath # DEPRECATED

#REDEFINE
# If the student login in LDAP is not the same as the student ID.
# This function translate student ID to student login.
def the_login(student):
    return str(student).lower()

def frame_info(frame, displayed):
    s = '<tr><td class="name"><small><small>%s</small></small>/<b>%s</b><br><td class="line">%s<td>\n' % (
        frame.f_code.co_filename.replace(os.getcwd(), '').strip('/'),
        frame.f_code.co_name,
        frame.f_lineno)
    for k, v in frame.f_locals.items():
        if id(v) not in displayed:
            if hasattr(v, 'backtrace_html'):
                try:
                    s += ("<p><b>" + html.escape(k) + "</b>:<br>"
                          + v.backtrace_html() + "\n")
                except TypeError:
                    pass
            displayed[id(v)] = True
    s += '</tr>'
    return s

import socket

backtrace_times = []
backtrace_times_disable_until = 0

def send_backtrace(txt, subject='Backtrace', exception=True):
    global backtrace_times_disable_until
    now = time.time()
    backtrace_times.append(now)
    while backtrace_times[0] < now - configuration.backtrace_times_span:
        backtrace_times.pop(0)
    if (len(backtrace_times) > configuration.backtrace_times_number
        and now > backtrace_times_disable_until):
        if configuration.backtrace_times_url:
            for url in configuration.backtrace_times_url.split(' '):
                read_url(url)
        backtrace_times.clear()
        backtrace_times_disable_until = now+configuration.backtrace_times_stop

    s = configuration.version_real
    if exception and sys.exc_info()[0] != None \
       and sys.exc_info()[0] == BrokenPipeError:
        s += '*' # pragma: no cover
    else:
        s += ' '
    s += ' '.join(sys.argv) + ' ' + subject
    subject = s
    displayed = {}
    s = ''
    if txt:
        s += ('<h1>Information reported by the exception catcher</h1><pre>' +
               html.escape(txt) + '</pre>\n')
    if exception and sys.exc_info()[0] != None:
        s += '<h1>Exception stack</h1>\n'
        s += '<p>Exception class: ' + html.escape(str(sys.exc_info()[0])) + '\n'
        s += '<p>Exception value: ' + html.escape(str(sys.exc_info()[1])) + '\n'
        f = sys.exc_info()[2]
        s += '<p>Exception Stack:<table>\n'
        x = ''
        while f:
            ss = frame_info(f.tb_frame, displayed)
            x = ss + x
            f = f.tb_next
        s += x + '</table>'

    s += '<h1>Current Stack:</h1>\n<table>\n'
    try:
        for i in range(1, 20):
            ss = frame_info(sys._getframe(i), displayed)
            s += ss
    except ValueError:
        pass
    s += '</table>'
    filename = os.path.join("LOGS", "BACKTRACES",
                            time.strftime('%Y-%m-%d'
                                          + os.path.sep + "%H:%M:%S")
                            )
    mkpath(os.path.join(*filename.split(os.path.sep)[:-1]), create_init=False)

    s = '<html><style>TABLE TD { border: 1px solid black;} .name { text-align:right } PRE { background: white ; border: 2px solid red ;}</style><body>' + s + '</body></html>'
    
    f = open(filename, "a", encoding = "utf-8")
    f.write(subject + '\n' + s)
    f.close()
    warn(subject + '\n' + s, what="error")

    if (send_backtrace.last_subject != subject  # Not twice the same subject
        and '*./' not in subject                # Not about closed connection
        and now > backtrace_times_disable_until # Backtrace temporarely disabled
       ):
        send_mail_in_background(configuration.maintainer, subject, s)
        send_backtrace.last_subject = subject

send_backtrace.last_subject = ''

def compressBuf(buf):
    import gzip
    import io
    zbuf = io.BytesIO()
    zfile = gzip.GzipFile(None, 'wb', 9, zbuf)
    if isinstance(buf, str):
        buf = buf.encode("utf-8")
    zfile.write(buf)
    zfile.close()
    return zbuf.getvalue()

class StaticFile(object):
    """Emulate a string, but it is a file content"""
    mimetypes = {'html': 'text/html;charset=utf-8',
                 'css': 'text/css;charset=utf-8',
                 'png': 'image/png',
                 'ico': 'image/png',
                 'jpg': 'image/jpeg',
                 'gif': 'image/gif',
                 'xls': 'application/vnd.ms-excel',
                 'js': 'application/x-javascript;charset=utf-8',
                 'txt': 'text/plain;charset=utf-8',
                 'csv': 'text/csv;charset=utf-8',
                 'xml': 'application/rss+xml;charset=utf-8',
                 'json': 'application/json; charset=UTF-8',
                }
    _url_ = 'http://???/' # The current server (TOMUSS or 'suivi')
    gzipped = None

    def __init__(self, name, mimetype=None, content=None, cached=True):
        self.name = name
        self.cached = cached
        if mimetype == None:
            if '.' in name:
                n = name.split('.')[-1]
                mimetype = self.mimetypes.get(n, 'text/plain;charset=utf-8')
        self.mimetype = mimetype
        self.content = content
        if 'image' in self.mimetype or 'ms-excel' in self.mimetype or 'pdf' in self.mimetype:
            self.encoding = "bytes"
        else:
            self.encoding = "utf-8"
        self.append_text = {}
        self.replace_text = {}
        if self.content:
            # Not a file, so NEVER reload it
            self.time = -1
            self.copy_on_disc()
        else:
            self.time = 0

    def need_update(self):
        if self.time != -1:
            if (self.content == None and self.cached
                or self.time != os.path.getmtime(self.name)):
                return True
        else:
            if self.gzipped is None:
                return True

        for i in failsafe_tuple(self.append_text.values):
            if isinstance(i, StaticFile):
                if i.need_update() or i.time > self.time:
                    return True

    def copy_on_disc(self):
        if configuration.read_only is not None:
            return # Only TOMUSS server write on disc
        dirname = os.path.join("TMP", configuration.version_real)
        mkpath(dirname, mode=0o755) # The static web server needs access
        filename = os.path.join(dirname, self.name.split(os.path.sep)[-1])
        write_file(filename, self.get_content(), self.encoding)

    def get_content(self):
        if self.cached:
            return self.content or self.update_content()
        return read_file(self.name, self.encoding)

    def update_content(self):
        # print("{} cached={} need_update={} encoding={} content={} gzipped={}"
        #     .format(self.name, self.cached, self.need_update(), self.encoding,
        #             type(self.content), type(self.gzipped)))
        if not self.need_update():
            return self.content
        if self.time != -1:
            self.time = os.path.getmtime(self.name)
        if not self.cached:
            self.copy_on_disc()
            return
        content = read_file(self.name, self.encoding)
        if self.encoding != 'bytes':
            for old, new in failsafe_tuple(self.replace_text.values):
                content = content.replace(old, new)
            content += ''.join(i if isinstance(i, str) else i.update_content()
                               for i in failsafe_tuple(self.append_text.values))
            if self.name.endswith('.js') or self.name.endswith('.html'):
                content = content.replace('_FILES_', configuration.url_files)
            self.gzipped = compressBuf(content)
        self.content = content
        self.copy_on_disc()
        return content

    def bytes(self):
        assert self.encoding == 'bytes'
        return self.content or self.update_content()

    def __str__(self):
        """Should no more be used: use get_content()"""
        assert self.encoding != 'bytes'
        return self.get_content()

    def get_zipped(self):
        if self.gzipped is None:
            self.update_content() # XXX Possible critical section
        return self.gzipped

    def clear_cache(self):
        if self.time != -1:
            self.time = 0

    def replace(self, key, old, new):
        """The replacement is done each time the file is reloaded"""
        self.replace_text[key] = (old, new)
        self.clear_cache()

    def append(self, key, content):
        """The append is done each time the file is reloaded"""
        self.append_text[key] = content
        self.clear_cache()

caches = []

def register_cache(f, fct, timeout, the_type):
    f.__doc__ = fct.__doc__
    f.fct = fct
    f.timeout = timeout
    f.the_type = the_type
    caches.append(f)

def clean_cache0(f):
    if f.cache[1] and time.time() - f.cache[1] > f.timeout:
        f.cache = ('', 0) # pragma: no cover


def add_a_cache0(fct, timeout=None):
    """Add a cache to a function without parameters"""
    if timeout is None:
        timeout = 3600
    def f():
        cache = f.cache
        if time.time() - cache[1] > f.timeout:
            cache = (f.fct(), time.time())
            f.cache = cache
        return cache[0]
    f.cache = ('', 0)
    register_cache(f, fct, timeout, 'add_a_cache0')
    f.clean = clean_cache0
    return f


def failsafe_tuple(x):
    """x is a generator function as .__iter__ .items .values"""
    while True:
        try:
            return tuple(x())
        except RuntimeError: # pragma: no cover
            pass

def clean_cache(f):
    if getattr(f, 'last_value_on_exception', 0):
        # Do not erase in order to reuse if there is an exception
        return # pragma: no cover
    for key, value in failsafe_tuple(f.cache.items):
        if time.time() - value[1] > f.timeout:
            del f.cache[key] # pragma: no cover

def add_a_cache(fct, timeout=3600, not_cached='neverreturnedvalue',
                last_value_on_exception=False):
    """Add a cache to a function with one parameter.

    If the returned value is 'not_cached' then it is not cached.

    If the cached function may raise an exception,
    it may be interesting to set 'last_value_on_exception=True' in order
    to return the previously cached value and hide the exception.

    If 'last_value_on_exception="disk"' then the cache is stored on disk.
    So, if there is an exception on the first call,
    the last value is restored from disk.
    """
    def f(x):
        cache = f.cache.get(x, ('',0))
        # XXX The lock protect only the cache update, not the cache test.
        # So the cache may be updated twice in a row.
        if (time.time() - cache[1] > f.timeout
            and f.lock.acquire(blocking = cache[1]==0 )):
            on_disk = f.last_value_on_exception == 'disk'
            try:
                cache = (f.fct(x), time.time())
            except: # pragma: no cover
                if f.last_value_on_exception and cache[1] != 0:
                    # Do not retry immediatly
                    cache = (cache[0], time.time())
                    send_backtrace(str(f.fct), "Cache update failed")
                else:
                    if on_disk: # pragma: no cover
                        c = read_file(os.path.join(f.dirname, safe(repr(x))))
                        cache = (literal_eval(c), time.time())
                        send_backtrace(str(f.fct), "Restore cache from disk")
                    else:
                        raise
                on_disk = False
            finally:
                try:
                    if cache[0] != f.not_cached:
                        f.cache[x] = cache
                    if on_disk: # pragma: no cover
                        write_file(os.path.join(f.dirname, safe(repr(x))),
                                   repr(cache[0]))
                finally:
                    f.lock.release()
        return cache[0]
    f.cache = {}
    register_cache(f, fct, timeout, 'add_a_cache')
    f.clean = clean_cache
    f.not_cached = not_cached
    f.last_value_on_exception = last_value_on_exception
    if last_value_on_exception == 'disk':
        f.dirname = os.path.join('TMP', 'CACHE', f.fct.__name__)
        mkpath(f.dirname)
    f.lock = threading.Lock()
    return f

def add_a_method_cache(fct, timeout=None, not_cached='neverreturnedvalue'):
    """Add a cache to a method with one parameter.
    If the returned value is 'not_cached' then it is not cached.
    The CACHE IS COMMON TO EVERY INSTANCE of the class"""
    if timeout == None:
        timeout = 3600
    def f(self, x): # pragma: no cover
        cache = f.cache.get(x, ('',0))
        if time.time() - cache[1] > f.timeout:
            cache = (f.fct(self, x), time.time())
            
        if cache[0] == f.not_cached:
            return f.not_cached
        else:
            f.cache[x] = cache
            return cache[0]
    f.cache = {}
    register_cache(f, fct, timeout, 'add_a_method_cache')
    f.clean = clean_cache
    f.not_cached = not_cached
    return f

def import_reload(filename):
    now = time.time()
    mtime = os.path.getmtime(filename)
    name = filename.split(os.path.sep)
    name[-1] = name[-1].replace('.py','')
    name.insert(0, 'TOMUSS')
    module_name = '.'.join(name)
    old_module = sys.modules.get(module_name, None)
    if old_module:
        to_reload = getattr(old_module, 'tomuss_load_time', import_reload.start_time) < mtime
        if to_reload:
            importlib.reload(old_module)
    else:
        to_reload = True
        importlib.import_module(module_name)
        old_module = sys.modules.get(module_name, None)
    old_module.tomuss_load_time = now
    return old_module, to_reload

import_reload.start_time = time.time()

def nice_date(x):
    year = x[0:4]
    month = x[4:6]
    day = x[6:8]
    hours = x[8:10]
    minutes = x[10:12]
    seconds = x[12:14]
    return hours + 'h' + minutes + '.' + seconds + ' le ' + \
           day + '/' + month + '/' + year 

def wait_scripts():
    # Returns 'true' if the script are loaded and so processing must continue.
    # If it returns 'false' then the calling function must stop processing.
    # It will be recalled with a 'setTimeOut'
    
    # The parameter is a string evaluated if the loading is not fine,
    # It must be a function recalling 'wait_scripts'
    # By the way :
    #    * this function can not be stored in a script.
    #    * It must not be in a loop
    t = list(time.localtime()[:6])
    t[1] -= 1 # Month starts et 0 in JavaScript
    return """
    function wait_scripts(recall)
    {
    if ( navigator.userAgent.indexOf('Konqueror') == -1 )
        {
            var d = document.getElementsByTagName('SCRIPT'), e ;            
            for(var i=0; i<d.length; i++)
               {
               e = d[i] ;
               if ( e.src === undefined )
                   continue ;
               if ( e.src.substr(0, 4) != 'http' )
                   continue ;
               if ( e.onloadDone )
                   continue ;
               if ( e.readyState === "complete" )
                   continue ;
               setTimeout(recall, 1000) ;
               return ;
               }
         }
    var d = new Date%s ;
    millisec.delta = d.getTime() - millisec() + 1000 ; // 1s to load page
    return true ;
    }
         """ % str(tuple(t))



#REDEFINE
# This function returns True if the user uses a stupid password.
# Potential stupid passwords are in the 'passwords' list.
# Each of the passwords should be tried to login,
# if the login is a success, the password is bad.
def stupid_password(login, passwords):
    return False # pragma: no cover


def module_to_login(module):
    return module.replace('__','.').replace('_','-')

def login_to_module(login):
    return login.replace('.','__').replace('-','_')

class AtomicWrite(object):
    """Act as 'open' function but rename file once it is closed."""
    def __init__(self, filename, reduce_ok=True, display_diff=False):
        self.real_filename = filename
        self.filename = filename + '.new'
        self.file = open(self.filename, 'w', encoding = "utf-8")
        self.reduce_ok = reduce_ok
        self.display_diff = display_diff
    def write(self, v):
        self.file.write(v)
    def close(self):
        self.file.close()
        if not self.reduce_ok \
           and os.path.exists(self.real_filename) \
           and os.path.getsize(self.filename) \
                   < 0.5 * os.path.getsize(self.real_filename): # pragma: no cover
            send_mail(configuration.maintainer,
                      'BUG TOMUSS : AtomicWrite Reduce' +
                      self.real_filename, self.real_filename)   
            return
        if self.display_diff: # pragma: no cover
            os.system("diff -u '%s' '%s'" % (
                self.real_filename.replace("'","'\"'\"'"),
                self.filename.replace("'","'\"'\"'")))
        os.rename(self.filename, self.real_filename)


def python_files(dirname):
    a = os.listdir(dirname)
    for ue in a:
        if not ue.endswith('.py'):
            continue
        if ue.startswith('__'):
            continue
        yield ue

@add_a_lock
def manage_key(dirname, key, separation=3, content=None, reduce_ok=True,
                    append=False, delete=False, touch=False):
    """
    Store the content in the key and return the old content or False

    The write is not *process* safe.
    """
    key = key.replace('/.', '/_')
    if key == '':
        return False # pragma: no cover
    dirname = os.path.join(configuration.db, dirname)

    if content is None and not os.path.isdir(dirname):
        return False
    try:
        mkpath(dirname)
    except OSError: # pragma: no cover
        pass
    
    key_dir = key.split(os.path.sep)[0]
    f1 = os.path.join(dirname, key_dir[:separation])
    if content is None and not os.path.isdir(f1):
        return False
    try:
        os.mkdir(f1, configuration.umask)
    except OSError:
        pass

    if os.path.sep in key:
        if content is None and not os.path.isdir(os.path.join(f1, key_dir)):
            return False
        try:
            os.mkdir(os.path.join(f1, key_dir), configuration.umask)
        except OSError:
            pass

    f1 = os.path.join(f1, key)

    if append:
        f = open(f1, 'a', encoding = "utf-8")
        f.write(content)
        f.close()
        # Do not return content because it may be large
        return

    if os.path.exists(f1):
        if delete:
            os.unlink(f1)
            return
        f = open(f1, 'r', encoding = "utf-8")
        c = f.read()
        f.close()
    else:
        c = False

    if content is not None:
        if configuration.read_only: # pragma: no cover
            send_backtrace("Manage key with content in 'suivi' server",
                           exception=False)
            return

        if c is False:
            c = ''
        if not reduce_ok and len(content) < len(c)*0.5: # pragma: no cover
            warn("Size not reduced for " + f1)
            return c
        if content != c: # Write if modified (non-existent files are empty)
            f = open(f1, 'w', encoding = "utf-8")
            f.write(content)
            f.close()
        elif touch:
            os.utime(f1) # Force time change on identical content
    return c

def key_mtime(dirname, key, separation=3):
    """Return the modification time of the key"""
    try:
        return os.path.getmtime(os.path.join(configuration.db, dirname,
                                             key[:separation], key))
    except OSError: # pragma: no cover
        return 0

        
def charte(login, year=None, semester=None):
    if year == None:
        year, semester = configuration.year_semester # pragma: no cover
    return os.path.join(login, 'charte_%s_%s' % (str(year), semester))

def charte_signed(login, server=None, year=None, semester=None):
    from . import signature
    if server:
        year = server.year
        semester = server.semester
    year = int(year)
    qs = signature.get_state(configuration.User(login))
    for q in qs.get_by_content('suivi_student_charte'): # pragma: no cover
        if year_semester_from_date(q.date) == (year, semester):
            return q.answer
    # Not found : add the question only for the current semester
    if year_semester_from_date(time.strftime("%Y%m")) == (year, semester): # pragma: no cover
        server.the_file.write('<img src="%s/=%s/signature/-1/x">'
                              % (configuration.server_url,
                                 server.ticket.ticket) )
        time.sleep(1)

# DEPRECATED
def display_preferences_get(login):
    return configuration.User(login).preferences

def display_preferences_set(user, d):
    user.manage_key('preferences', content = stable_repr(d))
    user.pop('preferences')

def lock_state():
    s = '' # Global Python import locked: %s\n' % imp.lock_held()
    for f in lock_list:
        if f.the_lock.locked():
            s += 'Locked   '
        else:
            s += 'Unlocked '
        s += '%s [%s]\n' % (f.fct.__name__, f.fct.__module__)
    return s

main_thread =  threading.current_thread()

def all_the_stacks():
    me = threading.current_thread()
    frames = sys._current_frames()
    s = []
    for t in threading.enumerate():
        if t is not me:
            try:
                s.append(''.join(traceback.format_stack(frames[t.ident])))
            except KeyError: # pragma: no cover
                pass
    # Needed in order to close files quickly
    del me
    del t
    del frames
    return '\n'.join(s)

# On first kill : do not stop answering requests, wait the good time to stop
# On second kill: stop answering new requests   , wait the good time to stop
# On third kill : stop immediatly
def on_kill(dummy_x, dummy_y): # pragma: no cover
    sys.stderr.write('=' * 79 + '\n' +
                     'KILLED\n' +
                     '=' * 79 + '\n' +
                     'LOCKS\n' +
                     '-' * 79 + '\n' +
                     lock_state() +
                     '=' * 79 + '\n'
                     'THREADS\n' +
                     '-' * 79 + '\n' +
                     all_the_stacks() +
                     '=' * 79 + '\n'
                     )
    traceback.print_stack()
    if hasattr(configuration, 'restart_tomuss'):
        on_kill.received += 1
        if on_kill.received == 1:
            start_new_thread(configuration.restart_tomuss, [None, False])
            return
        if on_kill.received == 2:
            configuration.restart_tomuss(server=None, start=False)
            # Never here
    sys.exit(0)
on_kill.received = 0

def clear_caches():
    for cache in failsafe_tuple(caches.__iter__):
        yield cache.clean(cache)

def background_clean():
    """Cleaning and updating by small steps.
    This generator must NEVER call a function that may take some time,
    as requesting a remote database.
    So it does not call Users.update()
    """
    from . import document
    from . import files
    from . import ticket
    while True:
        try:
            yield from clear_caches()
            yield from configuration.Users.cleanup(unused=1 if configuration.regtest else 500)
            yield from clear_caches()
            yield update_log_file()
            yield from document.remove_unused_tables()
            yield from clear_caches()
            if not configuration.read_only:
                yield from document.check_students_in_tables()
                yield from files.update_one_static_file()
                yield from clear_caches()
            yield from ticket.remove_old_tickets()
        except: # pragma: no cover
            # The generator must NEVER stop
            send_backtrace('', "background_clean failure")

class Variables(object):
    """Map variables to a TOMUSS configuration table stored in 0/Variables/_group

    The default group is the name of the module using Variables.

    Usage Example :

    V = Variables({'foo': ('foo comment', 'default_value'),
                   'bar': ('bar comment', 5),
                   })
    print(V.foo)

    Beware :
       * There is no overhead on V.foo access (except on value change)
       * The V.foo value will change if the user modify
         the table 0/Variables/_group
       * The user may only enter values of the same type.
         With the example, only integer values are allowed for 'bar'
       * The table is filled only when it is used (V.foo will do it)
    """
    def __init__(self, variables, group=None):
        self.__dict__['_variables'] = variables
        if group is None:
            group = sys._getframe(1).f_code.co_filename
            group = group.split(os.path.sep)[-1].replace('.py', '')
        
        self.__dict__['_group'] = group
        # Can't create the table here: catch 22

    def __iter__(self):
        return (key
                for key in self.__dict__
                if not key.startswith('_')
                )

    def _clean_(self):
        for k in tuple(iter(self)):
            self.__dict__.pop(k)

    def __getattr__(self, name):
        from . import document
        # '_' to remove ambiguity between 'Variables' template
        # and the table template.
        t = document.table(0, "Variables", '_' + self._group)
        if t and t.modifiable and not configuration.read_only:
            rw = t.pages[1]
            t.lock()
            try:
                for k, v in self._variables.items():
                    new_line = k not in t.lines
                    t.cell_change(rw, '0', k, v[0], force_update=True)
                    t.cell_change(rw, '1', k, v[1].__class__.__name__,
                                    force_update=True)
                    if new_line:
                        t.cell_change(rw, '2', k, repr(v[1]))
            finally:
                t.unlock()
        t.variables.add(self)
        for k, v in t.lines.items():
            try:
                self.__dict__[k] = literal_eval(v[2].value)
            except SyntaxError: # pragma: no cover
                self.__dict__[k] = None
        return self.__dict__[name]

    def __setattr__(self, name, value): # pragma: no cover
        raise AttributeError("Edit the Variable table to change parameters")

@add_a_lock
def _(msgid, language=None):
    "Translate the message (local then global dictionary)"
    if language is None:
        language = (configuration.language, 'en', 'fr')
    else:
        language = tuple(language) + (configuration.language, 'en', 'fr')
    if _.language != language:
        _.language = language
        try:
            _.loc_tr = gettext.translation('tomuss',
                                           os.path.join('LOCAL',
                                                        'LOCAL_TRANSLATIONS'),
                                           language)
        except IOError: # pragma: no cover
            _.loc_tr = None
        _.glo_tr = gettext.translation('tomuss', 'TRANSLATIONS', language)

    if _.loc_tr:
        tr = _.loc_tr.gettext(msgid)
        if tr != msgid:
            return tr
    return _.glo_tr.gettext(msgid)

_.language = None

import http.server

class HTTPServer(http.server.HTTPServer):
    old_shutdown_request = http.server.HTTPServer.shutdown_request

    def shutdown_request(self, request):
        return

class FakeRequestHandler(http.server.BaseHTTPRequestHandler):
    """
    """
    please_do_not_close = False
    # 0.3 is too short for tablets
    timeout = 0.5 # For Opera that does not send GET on HTTP request
    it_is_a_post = False
    do_profile = False

    def do_HEAD(self): # pragma: no cover
        self.send_response(200)
        self.send_header('Content-Type', 'text/html')
        self.end_headers()

    def do_POST(self):
        self.wfile.write = self.wfile._sock.sendall
        self.it_is_a_post = True
        if self.path == '/POST' or self.path.startswith('/pageaction'): # pragma: no cover
            fs = cgi.FieldStorage(fp=self.rfile, headers=self.headers,
                                  environ={'REQUEST_METHOD' : 'POST'})
            if self.path == '/POST':
                self.path = fs.getfirst('content')
            else:
                self.content = fs.getfirst('content')
        self.do_GET()

    def get_field_storage(self, size=50000000):
        if not self.it_is_a_post:
            return None # pragma: no cover
        return cgi.FieldStorage(fp=self.rfile, headers=self.headers,
                                environ={'REQUEST_METHOD' : 'POST'})

    def get_posted_data(self, size=50000000):
        """Provide compatibility for the old usage.
        Do not use: it takes a lot of memory.
        """
        fs = self.get_field_storage(size)
        if fs is None:
            return None # pragma: no cover
        d = {}
        for k in fs.keys():
            d[k] = [fs.getfirst(k, '')]
        return d

    #def send_response(self, i, comment=None):
        #if comment:
            ## To answer HEAD request no handled
            #http.server.BaseHTTPRequestHandler.send_response(self,i,comment)
            #return
        #http.server.BaseHTTPRequestHandler.send_response(self, i)
        ## Needed for HTTP/1.1 requests
        #self.send_header('Connection', 'close')
        ## self.wfile.flush()

    def backtrace_html(self):
        s = repr(self) + '\nRequest started %f seconds before\n' % (
            time.time() - self.start_time, )
        if hasattr(self, 'start_time_old'):
            s+= 'Authentication started %f seconds before\n' % (
            time.time() - self.start_time, ) # pragma: no cover
        s += '<h2>SERVER HEADERS</h2>\n'
        for k,v in self.headers.items():
            if k != 'authorization':
                s += '<b>' + k + '</b>:' + html.escape(str(v)) + '<br>\n'
        s += '<h2>SERVER DICT</h2>\n'
        for k,v in self.__dict__.items():
            if k != 'headers' and k != 'uploaded':
                s += '<b>' + k + '</b>:' + html.escape(str(v)) + '<br>\n'
        return s

    def address_string(self):
        """Override to avoid DNS lookups"""
        return "%s:%d" % self.client_address

    def log_time(self, action, **keys): # pragma: no cover
        try:
            self.__class__.log_time.__func__(self, action, **keys)
        except TypeError:
            self.__class__.log_time.__func__(self, action, **keys)

    def do_not_close_connection(self):
        self.please_do_not_close = True
        def close(file=weakref.ref(self.wfile), old_close=weakref.ref(self.wfile.close)):
            # If the thread run the job before the request is handled
            # The flush is called by the Python library on a closed file.
            f = file()
            if f:
                setattr(f, 'flush', lambda: True)
                if f._sock:
                    f._sock.close()
                old = old_close()
                if old:
                    old() # pragma: no cover
                f.close = lambda: False
        self.wfile.close = close

    def close_connection_now(self):
        sock = self.wfile._sock
        if sock:
            self.server.old_shutdown_request(sock)
        http.server.BaseHTTPRequestHandler.finish(self)
        self.please_do_not_close = True

    old_finish = http.server.BaseHTTPRequestHandler.finish
    def finish(self):
       if not self.please_do_not_close:
            self.old_finish()

    def unsafe(self):
        if 'unsafe=1' in self.path:
            return True
        else:
            return False

    def allow_cross_origin(self):
        origin = self.headers.get('origin')
        if origin and origin in configuration.domains: # pragma: no cover
            self.send_header('Access-Control-Allow-Origin', origin)
            self.send_header('Vary', 'Origin')

    def _(self, msgid):
        return _(msgid, self.ticket.language.split(','))

    def table_access_allowed(self, path=None):
        """Return True if the table editor is allowed for this login.
        This is called inside the plugin one the user is authenticated"""
        if self.ticket.is_a_teacher:
            return True
        tables_allowed = configuration.exceptions.get(self.ticket.user_name, None)
        if tables_allowed:
            if re.match(tables_allowed, path or '{}/{}/{}'.format(
                    self.the_year, self.the_semester, self.the_ue)):
                return True
        return False

#    def __(self, msgid):
#        return str(self._(msgid), "utf-8")

def start_threads():
    pass

@add_a_lock
def start_job(fct, seconds, important=None):
    """
    If needed: run 'fct' in 'seconds' in a new thread.

    'fct' returns its completion time (the time just before the last work checking)
    or None if it is assumed that completion time is: now - 0.01 second

    A new thread is started only if the current job is completed.
    So the number of 'fct' call can be less than the number of 'start_job' call.

    'fct' may take more than 'seconds' to execute.

    The minimum time between 'fct' end and next start is 'seconds'
    """
    fct.last_request = time.time()
    if getattr(fct, 'processing', False):
        return
    fct.processing = True

    def wait():
        nr_errors = 0
        while fct.processing:
            time.sleep(seconds)
            t = None
            try:
                t = fct()
            except: # pylint: disable=bare-except # pragma: no cover
                nr_errors += 1
                if nr_errors < 10:
                    send_backtrace('', "start_job %s" % fct)
                time.sleep(nr_errors)
            finally:
                if t is None:
                    # -0.01 to be sure the function was not on its way out
                    t = time.time() - 0.01
                start_job.the_lock.acquire()
                if fct.last_request < t:
                    fct.processing = False
                    if important:
                        important_job_remove(important)
                start_job.the_lock.release()
    if fct.__doc__:
        wait.__doc__ = ('Wait %d before running:\n\n' % seconds) + fct.__doc__
    if important:
        important_job_add(important)
    start_new_thread(wait, ())


current_jobs = set()
no_more_important_job = False

def important_job_add(job_name):
    current_jobs.add(job_name)
    while no_more_important_job:
        time.sleep(0.1) # pragma: no cover

def important_job_remove(job_name):
    current_jobs.remove(job_name)

def important_job_running():
    """If it returns None, no more important job are allowed and
    it is safe to stop TOMUSS"""
    global no_more_important_job
    no_more_important_job = True
    if current_jobs:
        no_more_important_job = False
        return True



def display_stack_on_kill():
    import signal
    signal.signal(signal.SIGTERM, on_kill)

def init(launch_threads=True):
    if launch_threads:
        start_threads()
    display_stack_on_kill()
    configuration.ampms_full = [
        ampm for ampm in eval(_("MSG_ampms_full"))]
    s = ""
    for k in ("yes", "no", "abi", "abj", "pre", "tnr", "ppn"):
        configuration.__dict__[k] = _(k)
        s += "var %s = %s, " % (k, js(_(k)))
        k_short = k + '_short'
        if _(k_short) != k_short:
            s += "%s = %s, " % (k_short, js(_(k_short)))
            configuration.__dict__[k_short] = _(k_short)
        k += "_char"
        configuration.__dict__[k] = _(k)
        s += "%s = %s;\n" % (k, js(_(k)))
    configuration.or_keyword = _('or')
    s += "function or_keyword() { return %s; }" % js(_('or'))
    s += "var COL_TITLE_0_2 = %s;\n" % js(_("COL_TITLE_0_2"))
    s += "var COL_TITLE_0_4 = %s;\n" % js(_("COL_TITLE_0_4"))
    s += "var server_language = %s ;\n" % js(configuration.language)
    s += "var special_days = %s ;\n" % js(configuration.special_days)
    s += "var allowed_grades = %s ;\n" % js(configuration.allowed_grades)
    from . import files # Here to avoid circular import
    files.files['types.js'].append("utilities.py", s)
    files.files['allow_error.html'] = StaticFile(
        'allow_error.html',
        content=_("TIP_violet_square"))
    files.files['ip_error.html'] = StaticFile(
        'ip_error.html',
        content=_("ip_error.html"))
    files.add('PLUGINS', 'suivi_student_charte.html')
        # Update possible grade values

    def with_allowed_grades(full, short):
        """Compute the values list with synonyms"""
        d = {}
        d[short] = full # abi_short → abi
        d[full] = full # abi → abi
        for new, old in configuration.allowed_grades.items():
            if old[0] in d:
                d[new] = d[old[0]] # abs → abi_short → abi
        return d
    configuration.all_abi = with_allowed_grades(configuration.abi, configuration.abi_short)
    configuration.all_abj = with_allowed_grades(configuration.abj, configuration.abj_short)
    configuration.all_tnr = with_allowed_grades(configuration.tnr, configuration.tnr_short)
    configuration.all_pre = with_allowed_grades(configuration.pre, configuration.pre_short)
    configuration.all_ppn = with_allowed_grades(configuration.ppn, configuration.ppn_short)
    configuration.all_abitnr = {**configuration.all_abi, **configuration.all_tnr}
    configuration.all_abjppn = {**configuration.all_abj, **configuration.all_ppn}
    configuration.all_all = {
        **configuration.all_abitnr, **configuration.all_abjppn, **configuration.all_pre}

def update_log_file(): # pragma: no cover
    if not start_as_daemon.logdir:
        return # Interactive or regtest
    logname = time.strftime('%Y-%m-%d')
    if logname == start_as_daemon.logname:
        return
    start_as_daemon.logname = logname
    mkpath(start_as_daemon.logdir)
    log = open(os.path.join(start_as_daemon.logdir,start_as_daemon.logname),"a")
    os.dup2(log.fileno() ,1)
    if start_as_daemon.log:
        start_as_daemon.log.close()
    start_as_daemon.log = log
    # Update symlink to last log
    loglink = os.path.join(start_as_daemon.logdir, 'log')
    if os.path.islink(loglink):
        os.unlink(loglink)
    os.symlink(logname, loglink)

def start_as_daemon(logdir): # pragma: no cover
    start_as_daemon.log = None
    start_as_daemon.logdir = logdir
    start_as_daemon.logname = ''
    update_log_file()
    sys.stderr = sys.stdout
    pid = os.path.join(logdir, 'pid')
    write_file(pid, str(os.getpid()))

    # atexit can be called and the process can fail to exit.
    # In this case, the PID must remain in order to allow crontab_run.py
    # to kill the process.
    # So the next lines are commented.
    # import atexit
    # atexit.register(os.unlink, pid)
start_as_daemon.logdir = None

class ProgressBar:
    """Insert a progress bar in the generated HTML.

    pb = utilities.ProgressBar(server, message="<h1>Title</h1>")
    pb.update(n, n_max)
    pb.hide()
    """
    nr = 0
    last_update = 0

    def __init__(self, server, message = "", auto_hide = False,
                 show_numbers = True, nb_max = None):
        self.server = server
        self.html_id = "progressbar{}".format(self.nr)
        self.auto_hide = auto_hide
        self.show_numbers = show_numbers
        self.nb_max = nb_max
        self.current = 0
        ProgressBar.nr += 1
        server.the_file.write('''
<div><div>{}</div>
<div style="border:2px solid black;">
<div id="{}" style="background:#8F8; border-right: 2px solid #080">&nbsp;</div>
</div></div>'''.format(message, self.html_id))

    def update(self, nb = None, nb_max = None):
        if nb_max is None:
            nb_max = self.nb_max
        if nb is None:
            nb = self.current
            self.current += 1
        now = time.time()
        if now - self.last_update > 1 or nb == nb_max:
            if self.show_numbers:
                more = "x.innerHTML = '{}/{}' ;".format(nb, nb_max)
            else:
                more = ""
            self.server.the_file.write("""<script>
            var x = document.getElementById('{}') ;
            x.style.width = '{}%' ;
            {}
            </script>""".format(self.html_id, 100 * nb / nb_max, more))
            self.server.the_file.flush()
            self.last_update = now
        if self.auto_hide and nb >= nb_max:
            self.hide() # pragma: no cover

    def append_to_message(self, text):
        self.server.the_file.write("""<script>
        var x = document.getElementById('{}').parentNode.parentNode.firstChild;
        x.innerHTML += {} ;
        </script>""".format(self.html_id, js(text)))
        self.server.the_file.flush()

    def hide(self):
        self.server.the_file.write("""<script>
        var x = document.getElementById('{}').parentNode.parentNode ;
        x.parentNode.removeChild(x) ;
        </script>""".format(self.html_id))
        self.server.the_file.flush()

    def wait_mail_sent(self):
        try:
            last = send_mail_in_background_list[-1]
        except IndexError: # pragma: no cover
            last = None
        nb_mails = len(send_mail_in_background_list)

        while True:
            try:
                pos = send_mail_in_background_list.index(last)
            except ValueError:
                pos = 0
            try:
                self.update(nb_mails - pos, nb_mails)
            except: # pragma: no cover
                break
            if pos == 0:
                break
            time.sleep(1)

# Remove files and directories in background

_cleanup_list = []
def _cleanup():
    while _cleanup_list:
        try:
            name = _cleanup_list.pop()
            if name.startswith('/') or '..' in name: # pragma: no cover
                send_backtrace("Bad filename cleanup: " + name)
                continue
            if os.path.isdir(name):
                shutil.rmtree(name)
            else:
                os.unlink(name)
        except IOError:
            pass
        except OSError: # pragma: no cover
            pass

def remove_this(filename):
    _cleanup_list.append(filename)
    start_job(_cleanup, 0.1 if configuration.real_regtest else 10)


# A literal_eval without memory leak

from ast import Constant, Tuple, List, Set, Dict, USub, UnaryOp, Str, Bytes, Num, NameConstant

def literal_eval_(node):
    """Create the object"""
    if isinstance(node, Constant):
        return node.value
    if isinstance(node, (Str, Bytes)):
        return node.s # pragma: no cover
    if isinstance(node, Num):
        return node.n # pragma: no cover
    if isinstance(node, Tuple):
        return tuple(map(literal_eval_, node.elts))
    if isinstance(node, List):
        return list(map(literal_eval_, node.elts))
    if isinstance(node, Set):
        return set(map(literal_eval_, node.elts)) # pragma: no cover
    if isinstance(node, Dict):
        return dict(zip(map(literal_eval_, node.keys),
                        map(literal_eval_, node.values)))
    if isinstance(node, NameConstant):
        return node.value # pragma: no cover
    if (isinstance(node, UnaryOp)
            and isinstance(node.op, USub)
            and isinstance(node.operand, (Constant, Num))
       ):
        return - literal_eval_(node.operand)
    raise ValueError(f'malformed node or string: {node}')

def literal_eval(string):
    """Safely evaluate a string containing a Python expression"""
    try:
        return literal_eval_(ast.parse(string, mode='eval').body)
    except ValueError:
        raise ValueError('malformed node or string: {}'.format(repr(string)))

for val in ('1', '1.5', '-1', '-1.5', '" "', 'True', 'None',
            '[1, []]', '(1, 2)', '{1:1}'):
    assert literal_eval(val) == ast.literal_eval(val)

# literal_eval = ast.literal_eval