# python standard library
import asyncio
from collections import defaultdict
from contextlib import contextmanager  # `with` statement in db sessions
from datetime import datetime
import logging
from random import random
from os import path
from typing import Any, Dict, Iterable, List, Optional, Tuple, Set, DefaultDict

# third party libraries
import bcrypt
import networkx as nx
import sqlalchemy as sa

# this project
from .models import Student, Answer, Topic, StudentTopic
from .questions import Question, QFactory, QDict, QuestionException
from .student import StudentState
from .tools import load_yaml


# setup logger for this module
logger = logging.getLogger(__name__)


# ============================================================================
class LearnException(Exception):
    pass


class DatabaseUnusableError(LearnException):
    pass


# ============================================================================
# LearnApp - application logic
#
#   self.deps - networkx topic dependencies
#   self.courses - dict {course_id: {'title': ...,
#                                    'description': ...,
#                                    'goals': ...,}, ...}
#   self.factory = dict {qref: QFactory()}
#   self.online - dict {student_id: {'number': ...,
#                                    'name': ...,
#                                    'state': StudentState(),
#                                    'counter': ...}, ...}
# ============================================================================
class LearnApp(object):
    # ------------------------------------------------------------------------
    # helper to manage db sessions using the `with` statement, for example
    #   with self.db_session() as s:  s.query(...)
    # ------------------------------------------------------------------------
    @contextmanager
    def db_session(self, **kw):
        session = self.Session(**kw)
        try:
            yield session
            session.commit()
        except Exception:
            logger.error('!!! Database rollback !!!')
            session.rollback()
            raise
        finally:
            session.close()

    # ------------------------------------------------------------------------
    # init
    # ------------------------------------------------------------------------
    def __init__(self,
                 courses: str,  # filename with course configurations
                 prefix: str,   # path to topics
                 db: str,       # database filename
                 check: bool = False) -> None:

        self.db_setup(db)       # setup database and check students
        self.online: Dict[str, Dict] = dict()    # online students

        try:
            config: Dict[str, Any] = load_yaml(courses)
        except Exception:
            msg = f'Failed to load yaml file "{courses}"'
            logger.error(msg)
            raise LearnException(msg)

        # --- topic dependencies are shared between all courses
        self.deps = nx.DiGraph(prefix=prefix)
        logger.info('Populating topic graph:')

        t = config.get('topics', {})  # topics defined directly in courses file
        self.populate_graph(t)
        logger.info(f'{len(t):>6} topics in {courses}')
        for f in config.get('topics_from', []):
            c = load_yaml(f)    # course configuration
            # FIXME set defaults??
            logger.info(f'{len(c["topics"]):>6} topics imported from {f}')
            self.populate_graph(c)
        logger.info(f'Graph has {len(self.deps)} topics')

        # --- courses dict
        self.courses = config['courses']
        logger.info(f'Courses:  {", ".join(self.courses.keys())}')
        for c, d in self.courses.items():
            d.setdefault('title', '')  # course title undefined
            for goal in d['goals']:
                if goal not in self.deps.nodes():
                    msg = f'Goal "{goal}" from course "{c}" does not exist'
                    logger.error(msg)
                    raise LearnException(msg)

        # --- factory is a dict with question generators for all topics
        self.factory: Dict[str, QFactory] = self.make_factory()

        # if graph has topics that are not in the database, add them
        self.add_missing_topics(self.deps.nodes())

        if check:
            self.sanity_check_questions()

    # ------------------------------------------------------------------------
    def sanity_check_questions(self) -> None:
        logger.info('Starting sanity checks (may take a while...)')

        errors: int = 0
        for qref in self.factory:
            logger.debug(f'checking {qref}...')
            try:
                q = self.factory[qref].generate()
            except QuestionException as e:
                logger.error(e)
                errors += 1
                continue  # to next question

            if 'tests_right' in q:
                for t in q['tests_right']:
                    q['answer'] = t
                    q.correct()
                    if q['grade'] < 1.0:
                        logger.error(f'Failed right answer in "{qref}".')
                        errors += 1
                        continue  # to next test

            if 'tests_wrong' in q:
                for t in q['tests_wrong']:
                    q['answer'] = t
                    q.correct()
                    if q['grade'] >= 1.0:
                        logger.error(f'Failed wrong answer in "{qref}".')
                        errors += 1
                        continue  # to next test

        if errors > 0:
            logger.error(f'{errors:>6} error(s) found.')
            raise LearnException('Sanity checks')
        else:
            logger.info('     0 errors found.')

    # ------------------------------------------------------------------------
    # login
    # ------------------------------------------------------------------------
    async def login(self, uid: str, pw: str) -> bool:

        with self.db_session() as s:
            found = s.query(Student.name, Student.password) \
                     .filter_by(id=uid) \
                     .one_or_none()

        # wait random time to minimize timing attacks
        await asyncio.sleep(random())

        loop = asyncio.get_running_loop()
        if found is None:
            logger.info(f'User "{uid}" does not exist')
            await loop.run_in_executor(None, bcrypt.hashpw, b'',
                                       bcrypt.gensalt())  # just spend time
            return False

        else:
            name, hashed_pw = found
            pw_ok: bool = await loop.run_in_executor(None,
                                                     bcrypt.checkpw,
                                                     pw.encode('utf-8'),
                                                     hashed_pw)

        if pw_ok:
            if uid in self.online:
                logger.warning(f'User "{uid}" already logged in')
                counter = self.online[uid]['counter']
            else:
                logger.info(f'User "{uid}" logged in')
                counter = 0

            # get topics of this student and set its current state
            with self.db_session() as s:
                tt = s.query(StudentTopic).filter_by(student_id=uid)

            state = {t.topic_id: {
                'level': t.level,
                'date': datetime.strptime(t.date, "%Y-%m-%d %H:%M:%S.%f")
                } for t in tt}

            self.online[uid] = {
                'number': uid,
                'name': name,
                'state': StudentState(uid=uid, state=state,
                                      courses=self.courses, deps=self.deps,
                                      factory=self.factory),
                'counter': counter + 1,  # counts simultaneous logins
                }

        else:
            logger.info(f'User "{uid}" wrong password')

        return pw_ok

    # ------------------------------------------------------------------------
    # logout
    # ------------------------------------------------------------------------
    def logout(self, uid: str) -> None:
        del self.online[uid]
        logger.info(f'User "{uid}" logged out')

    # ------------------------------------------------------------------------
    # change_password. returns True if password is successfully changed.
    # ------------------------------------------------------------------------
    async def change_password(self, uid: str, pw: str) -> bool:
        if not pw:
            return False

        loop = asyncio.get_running_loop()
        pw = await loop.run_in_executor(None, bcrypt.hashpw,
                                        pw.encode('utf-8'), bcrypt.gensalt())

        with self.db_session() as s:
            u = s.query(Student).get(uid)
            u.password = pw

        logger.info(f'User "{uid}" changed password')
        return True

    # ------------------------------------------------------------------------
    # Checks answer and update database. Returns corrected question.
    # ------------------------------------------------------------------------
    async def check_answer(self, uid: str, answer) -> Question:
        student = self.online[uid]['state']
        await student.check_answer(answer)
        q: Question = student.get_current_question()

        logger.info(f'User "{uid}" got {q["grade"]:.2} in "{q["ref"]}"')

        # always save grade of answered question
        with self.db_session() as s:
            s.add(Answer(
                ref=q['ref'],
                grade=q['grade'],
                starttime=str(q['start_time']),
                finishtime=str(q['finish_time']),
                student_id=uid,
                topic_id=student.get_current_topic()))

        return q

    # ------------------------------------------------------------------------
    # get the question to show (current or new one)
    # if no more questions, save/update level in database
    # ------------------------------------------------------------------------
    async def get_question(self, uid: str) -> Optional[Question]:
        student = self.online[uid]['state']
        q: Optional[Question] = await student.get_question()

        # save topic to database if finished
        if student.topic_has_finished():
            topic: str = student.get_previous_topic()
            level: float = student.get_topic_level(topic)
            date: str = str(student.get_topic_date(topic))
            logger.info(f'User "{uid}" finished "{topic}" (level={level:.2})')

            with self.db_session() as s:
                a = s.query(StudentTopic) \
                    .filter_by(student_id=uid, topic_id=topic) \
                    .one_or_none()
                if a is None:
                    # insert new studenttopic into database
                    logger.debug('db insert studenttopic')
                    t = s.query(Topic).get(topic)
                    u = s.query(Student).get(uid)
                    # association object
                    a = StudentTopic(level=level, date=date, topic=t,
                                     student=u)
                    u.topics.append(a)
                else:
                    # update studenttopic in database
                    logger.debug(f'db update studenttopic to level {level}')
                    a.level = level
                    a.date = date

                s.add(a)

        return q

    # ------------------------------------------------------------------------
    # Start course
    # ------------------------------------------------------------------------
    def start_course(self, uid: str, course_id: str) -> None:
        student = self.online[uid]['state']
        try:
            student.start_course(course_id)
        except Exception:
            logger.warning(f'"{uid}" could not start course "{course_id}"')
            raise
        else:
            logger.info(f'User "{uid}" started course "{course_id}"')

    # ------------------------------------------------------------------------
    # Start new topic
    # ------------------------------------------------------------------------
    async def start_topic(self, uid: str, topic: str) -> None:
        student = self.online[uid]['state']
        if uid == '0':
            logger.warning(f'Reloading "{topic}"')
            self.factory.update(self.factory_for(topic))

        try:
            await student.start_topic(topic)
        except Exception as e:
            logger.warning(f'User "{uid}" could not start "{topic}": {e}')
        else:
            logger.info(f'User "{uid}" started topic "{topic}"')

    # ------------------------------------------------------------------------
    # Fill db table 'Topic' with topics from the graph if not already there.
    # ------------------------------------------------------------------------
    def add_missing_topics(self, topics: List[str]) -> None:
        with self.db_session() as s:
            new_topics = [Topic(id=t) for t in topics
                          if (t,) not in s.query(Topic.id)]

            if new_topics:
                s.add_all(new_topics)
                logger.info(f'Added {len(new_topics)} new topic(s) to the '
                            f'database')

    # ------------------------------------------------------------------------
    # setup and check database contents
    # ------------------------------------------------------------------------
    def db_setup(self, db: str) -> None:

        logger.info(f'Checking database "{db}":')
        if not path.exists(db):
            raise LearnException('Database does not exist. '
                                 'Use "initdb-aprendizations" to create')

        engine = sa.create_engine(f'sqlite:///{db}', echo=False)
        self.Session = sa.orm.sessionmaker(bind=engine)
        try:
            with self.db_session() as s:
                n: int = s.query(Student).count()
                m: int = s.query(Topic).count()
                q: int = s.query(Answer).count()
        except Exception:
            logger.error(f'Database "{db}" not usable!')
            raise DatabaseUnusableError()
        else:
            logger.info(f'{n:6} students')
            logger.info(f'{m:6} topics')
            logger.info(f'{q:6} answers')

    # ========================================================================
    # Populates a digraph.
    #
    # Nodes are the topic references e.g. 'my/topic'
    #   g.nodes['my/topic']['name']      name of the topic
    #   g.nodes['my/topic']['questions'] list of question refs
    #
    # Edges are obtained from the deps defined in the YAML file for each topic.
    # ------------------------------------------------------------------------
    def populate_graph(self, config: Dict[str, Any]) -> None:
        g = self.deps       # dependency graph
        defaults = {
            'type': 'topic',
            'file': 'questions.yaml',
            'shuffle_questions': True,
            'choose': 9999,
            'forgetting_factor': 1.0,  # no forgetting
            'max_tries': 1,            # in every question
            'append_wrong': True,
            'min_level': 0.01,         # to unlock topic
        }
        defaults.update(config.get('defaults', {}))

        # iterate over topics and populate graph
        topics: Dict[str, Dict] = config.get('topics', {})
        g.add_nodes_from(topics.keys())
        for tref, attr in topics.items():
            logger.debug(f'       + {tref}')
            for d in attr.get('deps', []):
                g.add_edge(d, tref)

            t = g.nodes[tref]  # get current topic node
            t['name'] = attr.get('name', tref)
            t['questions'] = attr.get('questions', [])

            for k, default in defaults.items():
                t[k] = attr.get(k, default)

            t['path'] = path.join(g.graph['prefix'], tref)    # prefix/topic

    # ========================================================================
    # methods that do not change state (pure functions)
    # ========================================================================

    # ------------------------------------------------------------------------
    # Buils dictionary of question factories
    #   - visits each topic in the graph,
    #   - adds factory for each topic.
    # ------------------------------------------------------------------------
    def make_factory(self) -> Dict[str, QFactory]:

        logger.info('Building questions factory:')
        factory = dict()
        g = self.deps
        for tref in g.nodes():
            factory.update(self.factory_for(tref))

        logger.info(f'Factory has {len(factory)} questions')
        return factory

    # ------------------------------------------------------------------------
    # makes factory for a single topic
    # ------------------------------------------------------------------------
    def factory_for(self, tref: str) -> Dict[str, QFactory]:
        factory: Dict[str, QFactory] = {}
        g = self.deps
        t = g.nodes[tref]  # get node

        # load questions as list of dicts
        topicpath: str = path.join(g.graph['prefix'], tref)
        try:
            fullpath: str = path.join(topicpath, t['file'])
        except Exception:
            msg1 = f'Invalid topic "{tref}"'
            msg2 = f'Check dependencies of {", ".join(g.successors(tref))}'
            raise LearnException(f'{msg1}. {msg2}')

        logger.debug(f'  Loading {fullpath}')
        try:
            questions: List[QDict] = load_yaml(fullpath)
        except Exception:
            if t['type'] == 'chapter':
                return factory  # chapters may have no "questions"
            else:
                msg = f'Failed to load "{fullpath}"'
                logger.error(msg)
                raise LearnException(msg)

        if not isinstance(questions, list):
            msg = f'File "{fullpath}" must be a list of questions'
            raise LearnException(msg)

        # update refs to include topic as prefix.
        # refs are required to be unique only within the file.
        # undefined are set to topic:n, where n is the question number
        # within the file
        localrefs: Set[str] = set()  # refs in current file
        for i, q in enumerate(questions):
            qref = q.get('ref', str(i))  # ref or number
            if qref in localrefs:
                msg = f'Duplicate ref "{qref}" in "{topicpath}"'
                raise LearnException(msg)
            localrefs.add(qref)

            q['ref'] = f'{tref}:{qref}'
            q['path'] = topicpath
            q.setdefault('append_wrong', t['append_wrong'])

        # if questions are left undefined, include all.
        if not t['questions']:
            t['questions'] = [q['ref'] for q in questions]

        t['choose'] = min(t['choose'], len(t['questions']))

        for q in questions:
            if q['ref'] in t['questions']:
                factory[q['ref']] = QFactory(q)
                logger.debug(f'       + {q["ref"]}')

        logger.info(f'{len(t["questions"]):6} questions in {tref}')

        return factory

    # ------------------------------------------------------------------------
    def get_login_counter(self, uid: str) -> int:
        return int(self.online[uid]['counter'])

    # ------------------------------------------------------------------------
    def get_student_name(self, uid: str) -> str:
        return self.online[uid].get('name', '')

    # ------------------------------------------------------------------------
    def get_student_state(self, uid: str) -> List[Dict[str, Any]]:
        return self.online[uid]['state'].get_knowledge_state()

    # ------------------------------------------------------------------------
    def get_student_progress(self, uid: str) -> float:
        return float(self.online[uid]['state'].get_topic_progress())

    # ------------------------------------------------------------------------
    def get_current_question(self, uid: str) -> Optional[Question]:
        q: Optional[Question] = self.online[uid]['state'].get_current_question()
        return q

    # ------------------------------------------------------------------------
    def get_current_question_id(self, uid: str) -> str:
        return str(self.online[uid]['state'].get_current_question()['qid'])

    # ------------------------------------------------------------------------
    def get_student_question_type(self, uid: str) -> str:
        return str(self.online[uid]['state'].get_current_question()['type'])

    # ------------------------------------------------------------------------
    def get_student_topic(self, uid: str) -> str:
        return str(self.online[uid]['state'].get_current_topic())

    # ------------------------------------------------------------------------
    def get_student_course_title(self, uid: str) -> str:
        return str(self.online[uid]['state'].get_current_course_title())

    # ------------------------------------------------------------------------
    def get_current_course_id(self, uid: str) -> Optional[str]:
        cid: Optional[str] = self.online[uid]['state'].get_current_course_id()
        return cid

    # ------------------------------------------------------------------------
    def get_topic_name(self, ref: str) -> str:
        return str(self.deps.nodes[ref]['name'])

    # ------------------------------------------------------------------------
    def get_current_public_dir(self, uid: str) -> str:
        topic: str = self.online[uid]['state'].get_current_topic()
        prefix: str = self.deps.graph['prefix']
        return path.join(prefix, topic, 'public')

    # ------------------------------------------------------------------------
    def get_courses(self) -> Dict[str, Dict[str, Any]]:
        return self.courses

    # ------------------------------------------------------------------------
    def get_course(self, course_id: str) -> Dict[str, Any]:
        return self.courses[course_id]

    # ------------------------------------------------------------------------
    def get_rankings(self, uid: str, course_id: str) -> Iterable[Tuple[str, str, float, float]]:

        logger.info(f'User "{uid}" get rankings for {course_id}')
        with self.db_session() as s:
            students = s.query(Student.id, Student.name).all()

            # topic progress
            student_topics = s.query(StudentTopic.student_id,
                                     StudentTopic.topic_id,
                                     StudentTopic.level,
                                     StudentTopic.date).all()

            # answer performance
            total = dict(s.query(Answer.student_id, sa.func.count(Answer.ref)).
                         group_by(Answer.student_id).
                         all())
            right = dict(s.query(Answer.student_id, sa.func.count(Answer.ref)).
                         filter(Answer.grade == 1.0).
                         group_by(Answer.student_id).
                         all())

        # compute percentage of right answers
        perf: Dict[str, float] = {uid: right.get(uid, 0.0)/total[uid]
                                  for uid in total}

        # compute topic progress
        now = datetime.now()
        goals = self.courses[course_id]['goals']
        prog: DefaultDict[str, float] = defaultdict(int)

        for uid, topic, level, date in student_topics:
            if topic in goals:
                date = datetime.strptime(date, "%Y-%m-%d %H:%M:%S.%f")
                prog[uid] += level**(now - date).days / len(goals)

        rankings = [(uid, name, prog[uid], perf.get(uid, 0.0))
                    for uid, name in students if uid != '0' and uid in prog]
        rankings.sort(key=lambda x: x[2], reverse=True)
        return rankings

    # ------------------------------------------------------------------------