diff --git a/mypy.ini b/mypy.ini index db7de3a..0db7c85 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,15 +1,15 @@ [mypy] -python_version = 3.7 +python_version = 3.8 # warn_return_any = True # warn_unused_configs = True -[mypy-sqlalchemy.*] +[mypy-setuptools.*] ignore_missing_imports = True -[mypy-pygments.*] +[mypy-sqlalchemy.*] ignore_missing_imports = True -[mypy-bcrypt.*] +[mypy-pygments.*] ignore_missing_imports = True [mypy-mistune.*] diff --git a/perguntations/app.py b/perguntations/app.py index 7d45d6d..8671926 100644 --- a/perguntations/app.py +++ b/perguntations/app.py @@ -5,7 +5,7 @@ Main application module # python standard libraries import asyncio -from contextlib import contextmanager # `with` statement in db sessions +# from contextlib import contextmanager # `with` statement in db sessions import csv import io import json @@ -14,8 +14,10 @@ from os import path # installed packages import bcrypt -from sqlalchemy import create_engine, exc -from sqlalchemy.orm import sessionmaker +from sqlalchemy import create_engine, select, func +from sqlalchemy.orm import Session +from sqlalchemy.exc import NoResultFound +# from sqlalchemy.orm import sessionmaker # this project from perguntations.models import Student, Test, Question @@ -24,6 +26,7 @@ from perguntations.testfactory import TestFactory, TestFactoryException import perguntations.test from perguntations.questions import question_from +# setup logger for this module logger = logging.getLogger(__name__) @@ -35,20 +38,20 @@ class AppException(Exception): # ============================================================================ # helper functions # ============================================================================ -async def check_password(try_pw, hashed_pw): - '''check password in executor''' - try_pw = try_pw.encode('utf-8') - loop = asyncio.get_running_loop() - hashed = await loop.run_in_executor(None, bcrypt.hashpw, try_pw, hashed_pw) - return hashed_pw == hashed +# async def check_password(try_pw, hashed_pw): +# '''check password in executor''' +# try_pw = try_pw.encode('utf-8') +# loop = asyncio.get_running_loop() +# hashed = await loop.run_in_executor(None, bcrypt.hashpw, try_pw, hashed_pw) +# return hashed_pw == hashed -async def hash_password(password): - '''hash password in executor''' - loop = asyncio.get_running_loop() - return await loop.run_in_executor(None, bcrypt.hashpw, - password.encode('utf-8'), - bcrypt.gensalt()) +# async def hash_password(password): +# '''hash password in executor''' +# loop = asyncio.get_running_loop() +# return await loop.run_in_executor(None, bcrypt.hashpw, +# password.encode('utf-8'), +# bcrypt.gensalt()) # ============================================================================ @@ -67,23 +70,23 @@ class App(): self.testfactory - TestFactory ''' - # ------------------------------------------------------------------------ - @contextmanager - def _db_session(self): - ''' - helper to manage db sessions using the `with` statement, for example: - with self._db_session() as s: s.query(...) - ''' - session = self.Session() - try: - yield session - session.commit() - except exc.SQLAlchemyError: - logger.error('DB rollback!!!') - session.rollback() - raise - finally: - session.close() + # # ------------------------------------------------------------------------ + # @contextmanager + # def _db_session(self): + # ''' + # helper to manage db sessions using the `with` statement, for example: + # with self._db_session() as s: s.query(...) + # ''' + # session = self.Session() + # try: + # yield session + # session.commit() + # except exc.SQLAlchemyError: + # logger.error('DB rollback!!!') + # session.rollback() + # raise + # finally: + # session.close() # ------------------------------------------------------------------------ def __init__(self, conf): @@ -94,18 +97,7 @@ class App(): self.pregenerated_tests = [] # list of tests to give to students self._make_test_factory(conf) - - # connect to database and check registered students - dbfile = self.testfactory['database'] - database = f'sqlite:///{path.expanduser(dbfile)}' - engine = create_engine(database, echo=False) - self.Session = sessionmaker(bind=engine) - try: - with self._db_session() as sess: - num = sess.query(Student).filter(Student.id != '0').count() - except Exception as exc: - raise AppException(f'Database unusable {dbfile}.') from exc - logger.info('Database "%s" has %s students.', dbfile, num) + self._db_setup() # command line option --allow-all if conf['allow_all']: @@ -126,6 +118,31 @@ class App(): if conf['correct']: self._correct_tests() + def _db_setup(self) -> None: + logger.info('Setup database') + + # connect to database and check registered students + dbfile = path.expanduser(self.testfactory['database']) + if not path.exists(dbfile): + raise AppException('Database does not exist.') + self._engine = create_engine(f'sqlite:///{dbfile}', future=True) + + try: + query = select(func.count(Student.id)).where(Student.id != '0') + with Session(self._engine, future=True) as session: + num = session.execute(query).scalar() + except Exception as exc: + raise AppException(f'Database unusable {dbfile}.') from exc + + # try: + # with self._db_session() as sess: + # num = sess.query(Student).filter(Student.id != '0').count() + # except Exception as exc: + # raise AppException(f'Database unusable {dbfile}.') from exc + logger.info('Database "%s" has %s students.', dbfile, num) + + + # ------------------------------------------------------------------------ def _correct_tests(self): with self._db_session() as sess: diff --git a/perguntations/initdb.py b/perguntations/initdb.py index 5ff5c72..5b93708 100644 --- a/perguntations/initdb.py +++ b/perguntations/initdb.py @@ -13,7 +13,9 @@ from concurrent.futures import ThreadPoolExecutor # installed packages import bcrypt -import sqlalchemy as sa +from sqlalchemy import create_engine, select +from sqlalchemy.orm import Session +from sqlalchemy.exc import IntegrityError # this project from perguntations.models import Base, Student @@ -22,6 +24,7 @@ from perguntations.models import Base, Student # ============================================================================ def parse_commandline_arguments(): '''Parse command line options''' + parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter, description='Insert new users into a database. Users can be imported ' @@ -102,7 +105,7 @@ def get_students_from_csv(filename): # ============================================================================ -def hashpw(student, password=None): +def hashpw(student, password=None) -> None: '''replace password by hash for a single student''' print('.', end='', flush=True) if password is None: @@ -113,14 +116,14 @@ def hashpw(student, password=None): # ============================================================================ -def insert_students_into_db(session, students): +def insert_students_into_db(session, students) -> None: '''insert list of students into the database''' try: session.add_all([Student(id=s['uid'], name=s['name'], password=s['pw']) for s in students]) session.commit() - except sa.exc.IntegrityError: + except IntegrityError: print('!!! Integrity error. Users already in database. Aborted !!!\n') session.rollback() @@ -128,11 +131,12 @@ def insert_students_into_db(session, students): # ============================================================================ def show_students_in_database(session, verbose=False): '''get students from database''' - users = session.query(Student).all() + users = session.execute(select(Student)).scalars().all() + # users = session.query(Student).all() + total = len(users) - total_users = len(users) print('Registered users:') - if total_users == 0: + if total == 0: print(' -- none --') else: users.sort(key=lambda u: f'{u.id:>12}') # sort by number @@ -141,13 +145,13 @@ def show_students_in_database(session, verbose=False): print(f'{user.id:>12} {user.name}') else: print(f'{users[0].id:>12} {users[0].name}') - if total_users > 1: + if total > 1: print(f'{users[1].id:>12} {users[1].name}') - if total_users > 3: + if total > 3: print(' | |') - if total_users > 2: + if total > 2: print(f'{users[-1].id:>12} {users[-1].name}') - print(f'Total: {total_users}.') + print(f'Total: {total}.') # ============================================================================ @@ -157,39 +161,41 @@ def main(): args = parse_commandline_arguments() # --- database - print(f'Using database: {args.db}') - engine = sa.create_engine(f'sqlite:///{args.db}', echo=False) + print(f'Database: {args.db}') + engine = create_engine(f'sqlite:///{args.db}', echo=False, future=True) Base.metadata.create_all(engine) # Criates schema if needed - SessionMaker = sa.orm.sessionmaker(bind=engine) - session = SessionMaker() + session = Session(engine, future=True) # --- make list of students to insert - new_students = [] + students = [] if args.admin: print('Adding user: 0, Admin.') - new_students.append({'uid': '0', 'name': 'Admin'}) + students.append({'uid': '0', 'name': 'Admin'}) for csvfile in args.csvfile: print('Adding users from:', csvfile) - new_students.extend(get_students_from_csv(csvfile)) + students.extend(get_students_from_csv(csvfile)) if args.add: for uid, name in args.add: print(f'Adding user: {uid}, {name}.') - new_students.append({'uid': uid, 'name': name}) + students.append({'uid': uid, 'name': name}) # --- insert new students - if new_students: + if students: print('Generating password hashes', end='') - with ThreadPoolExecutor() as executor: # hashing in parallel - executor.map(lambda s: hashpw(s, args.pw), new_students) - print(f'\nInserting {len(new_students)}') - insert_students_into_db(session, new_students) + with ThreadPoolExecutor() as executor: # hashing + executor.map(lambda s: hashpw(s, args.pw), students) + print(f'\nInserting {len(students)}') + insert_students_into_db(session, students) # --- update all students if args.update_all: - all_students = session.query(Student).filter(Student.id != '0').all() + all_students = session.execute( + select(Student).where(Student.id != '0') + ).all() + print(f'Updating password of {len(all_students)} users', end='') for student in all_students: password = (args.pw or student.id).encode('utf-8') @@ -202,7 +208,7 @@ def main(): else: for student_id in args.update: print(f'Updating password of {student_id}') - student = session.query(Student).get(student_id) + student = session.execute(select(Student.id)) password = (args.pw or student_id).encode('utf-8') student.password = bcrypt.hashpw(password, bcrypt.gensalt()) session.commit() diff --git a/perguntations/models.py b/perguntations/models.py index fd2fbfc..c28609e 100644 --- a/perguntations/models.py +++ b/perguntations/models.py @@ -1,18 +1,19 @@ ''' +perguntations/models.py SQLAlchemy ORM - -The classes below correspond to database tables ''' from sqlalchemy import Column, ForeignKey, Integer, Float, String -from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy.orm import relationship +from sqlalchemy.orm import declarative_base, relationship # ============================================================================ # Declare ORM -Base = declarative_base() +# FIXME Any is a workaround for mypy static type checking (see https://github.com/python/mypy/issues/6372) +from typing import Any +Base: Any = declarative_base() +# Base = declarative_base() # ---------------------------------------------------------------------------- @@ -26,11 +27,11 @@ class Student(Base): # --- tests = relationship('Test', back_populates='student') - def __str__(self): - return (f'Student:\n' - f' id: "{self.id}"\n' - f' name: "{self.name}"\n' - f' password: "{self.password}"\n') + def __repr__(self): + return (f'Student(' + f'id={self.id!r}, ' + f'name={self.name!r}, ' + f'password={self.password!r})') # ---------------------------------------------------------------------------- @@ -52,18 +53,18 @@ class Test(Base): student = relationship('Student', back_populates='tests') questions = relationship('Question', back_populates='test') - def __str__(self): - return (f'Test:\n' - f' id: {self.id}\n' - f' ref: "{self.ref}"\n' - f' title: "{self.title}"\n' - f' grade: {self.grade}\n' - f' state: "{self.state}"\n' - f' comment: "{self.comment}"\n' - f' starttime: "{self.starttime}"\n' - f' finishtime: "{self.finishtime}"\n' - f' filename: "{self.filename}"\n' - f' student_id: "{self.student_id}"\n') + def __repr__(self): + return (f'Test(' + f'id={self.id!r}, ' + f'ref={self.ref!r}, ' + f'title={self.title!r}, ' + f'grade={self.grade!r}, ' + f'state={self.state!r}, ' + f'comment={self.comment!r}, ' + f'starttime={self.starttime!r}, ' + f'finishtime={self.finishtime!r}, ' + f'filename={self.filename!r}, ' + f'student_id={self.student_id!r})') # --------------------------------------------------------------------------- @@ -82,13 +83,13 @@ class Question(Base): # --- test = relationship('Test', back_populates='questions') - def __str__(self): - return (f'Question:\n' - f' id: {self.id}\n' - f' number: {self.number}\n' - f' ref: "{self.ref}"\n' - f' grade: {self.grade}\n' - f' comment: "{self.comment}"\n' - f' starttime: "{self.starttime}"\n' - f' finishtime: "{self.finishtime}"\n' - f' test_id: "{self.test_id}"\n') + def __repr__(self): + return (f'Question(' + f'id={self.id!r}, ' + f'number={self.number!r}, ' + f'ref={self.ref!r}, ' + f'grade={self.grade!r}, ' + f'comment={self.comment!r}, ' + f'starttime={self.starttime!r}, ' + f'finishtime={self.finishtime!r}, ' + f'test_id={self.test_id!r})') diff --git a/perguntations/questions.py b/perguntations/questions.py index abdb5fb..3678799 100644 --- a/perguntations/questions.py +++ b/perguntations/questions.py @@ -1,5 +1,6 @@ ''' -Classes the implement several types of questions. +File: perguntations/questions.py +Description: Classes the implement several types of questions. ''' @@ -13,24 +14,15 @@ import re from typing import Any, Dict, NewType import uuid - -# from urllib.error import HTTPError -# import json -# import http.client - - # this project from perguntations.tools import run_script, run_script_async # setup logger for this module logger = logging.getLogger(__name__) - QDict = NewType('QDict', Dict[str, Any]) - - class QuestionException(Exception): '''Exceptions raised in this module''' @@ -45,8 +37,6 @@ class Question(dict): for each student. Instances can shuffle options or automatically generate questions. ''' - # def __init__(self, q: QDict) -> None: - # super().__init__(q) def gen(self) -> None: ''' @@ -96,9 +86,6 @@ class QuestionRadio(Question): ''' # ------------------------------------------------------------------------ - # def __init__(self, q: QDict) -> None: - # super().__init__(q) - def gen(self) -> None: ''' Sets defaults, performs checks and generates the actual question @@ -231,9 +218,6 @@ class QuestionCheckbox(Question): ''' # ------------------------------------------------------------------------ - # def __init__(self, q: QDict) -> None: - # super().__init__(q) - def gen(self) -> None: super().gen() @@ -288,19 +272,6 @@ class QuestionCheckbox(Question): f'Please fix "{self["ref"]}" in "{self["path"]}"') logger.error(msg) raise QuestionException(msg) - # msg0 = ('+--------------- BEHAVIOR CHANGE NOTICE ---------------+') - # msg1 = ('| Correct values in checkbox questions must be in the |') - # msg2 = ('| interval [0.0, 1.0]. I will convert to the new |') - # msg3 = ('| behavior, for now, but you should fix it. |') - # msg4 = ('+------------------------------------------------------+') - # logger.warning(msg0) - # logger.warning(msg1) - # logger.warning(msg2) - # logger.warning(msg3) - # logger.warning(msg4) - # logger.warning('please fix "%s"', self["ref"]) - # # normalize to [0,1] - # self['correct'] = [(x+1)/2 for x in self['correct']] # if an option is a list of (right, wrong), pick one options = [] @@ -356,9 +327,6 @@ class QuestionText(Question): ''' # ------------------------------------------------------------------------ - # def __init__(self, q: QDict) -> None: - # super().__init__(q) - def gen(self) -> None: super().gen() self.set_defaults(QDict({ @@ -389,6 +357,7 @@ class QuestionText(Question): def transform(self, ans): '''apply optional filters to the answer''' + # apply transformations in sequence for transform in self['transform']: if transform == 'remove_space': # removes all spaces ans = ans.replace(' ', '') @@ -410,7 +379,7 @@ class QuestionText(Question): super().correct() if self['answer'] is not None: - answer = self.transform(self['answer']) # apply transformations + answer = self.transform(self['answer']) self['grade'] = 1.0 if answer in self['correct'] else 0.0 @@ -427,9 +396,6 @@ class QuestionTextRegex(Question): ''' # ------------------------------------------------------------------------ - # def __init__(self, q: QDict) -> None: - # super().__init__(q) - def gen(self) -> None: super().gen() @@ -442,14 +408,6 @@ class QuestionTextRegex(Question): if not isinstance(self['correct'], list): self['correct'] = [self['correct']] - # converts patterns to compiled versions - # try: - # self['correct'] = [re.compile(a) for a in self['correct']] - # except Exception as exc: - # msg = f'Failed to compile regex in "{self["ref"]}"' - # logger.error(msg) - # raise QuestionException(msg) from exc - # ------------------------------------------------------------------------ def correct(self) -> None: super().correct() @@ -464,15 +422,6 @@ class QuestionTextRegex(Question): regex, self['answer']) self['grade'] = 0.0 - # try: - # if regex.match(self['answer']): - # self['grade'] = 1.0 - # return - # except TypeError: - # logger.error('While matching regex %s with answer "%s".', - # regex.pattern, self["answer"]) - - # ============================================================================ class QuestionNumericInterval(Question): '''An instance of QuestionTextNumeric will always have the keys: @@ -484,9 +433,6 @@ class QuestionNumericInterval(Question): ''' # ------------------------------------------------------------------------ - # def __init__(self, q: QDict) -> None: - # super().__init__(q) - def gen(self) -> None: super().gen() @@ -548,9 +494,6 @@ class QuestionTextArea(Question): ''' # ------------------------------------------------------------------------ - # def __init__(self, q: QDict) -> None: - # super().__init__(q) - def gen(self) -> None: super().gen() @@ -625,129 +568,6 @@ class QuestionTextArea(Question): # ============================================================================ -# class QuestionCode(Question): -# ''' -# Submits answer to a JOBE server to compile and run against the test cases. -# ''' - -# _outcomes = { -# 0: 'JOBE outcome: Successful run', -# 11: 'JOBE outcome: Compile error', -# 12: 'JOBE outcome: Runtime error', -# 13: 'JOBE outcome: Time limit exceeded', -# 15: 'JOBE outcome: Successful run', -# 17: 'JOBE outcome: Memory limit exceeded', -# 19: 'JOBE outcome: Illegal system call', -# 20: 'JOBE outcome: Internal error, please report', -# 21: 'JOBE outcome: Server overload', -# } - -# # ------------------------------------------------------------------------ -# def __init__(self, q: QDict) -> None: -# super().__init__(q) - -# self.set_defaults(QDict({ -# 'text': '', -# 'timeout': 5, # seconds -# 'server': '127.0.0.1', # JOBE server -# 'language': 'c', -# 'correct': [{'stdin': '', 'stdout': '', 'stderr': '', 'args': ''}], -# })) - - # ------------------------------------------------------------------------ - # def correct(self) -> None: - # super().correct() - - # if self['answer'] is None: - # return - - # # submit answer to JOBE server - # resource = '/jobe/index.php/restapi/runs/' - # headers = {"Content-type": "application/json; charset=utf-8", - # "Accept": "application/json"} - - # for expected in self['correct']: - # data_json = json.dumps({ - # 'run_spec' : { - # 'language_id': self['language'], - # 'sourcecode': self['answer'], - # 'input': expected.get('stdin', ''), - # }, - # }) - - # try: - # connect = http.client.HTTPConnection(self['server']) - # connect.request( - # method='POST', - # url=resource, - # body=data_json, - # headers=headers - # ) - # response = connect.getresponse() - # logger.debug('JOBE response status %d', response.status) - # if response.status != 204: - # content = response.read().decode('utf8') - # if content: - # result = json.loads(content) - # connect.close() - - # except (HTTPError, ValueError): - # logger.error('HTTPError while connecting to JOBE server') - - # try: - # outcome = result['outcome'] - # except (NameError, TypeError, KeyError): - # logger.error('Bad result returned from JOBE server: %s', result) - # return - # logger.debug(self._outcomes[outcome]) - - - - # if result['cmpinfo']: # compiler errors and warnings - # self['comments'] = f'Erros de compilação:\n{result["cmpinfo"]}' - # self['grade'] = 0.0 - # return - - # if result['stdout'] != expected.get('stdout', ''): - # self['comments'] = 'O output gerado é diferente do esperado.' # FIXME mostrar porque? - # self['grade'] = 0.0 - # return - - # self['comments'] = 'Ok!' - # self['grade'] = 1.0 - - - # # ------------------------------------------------------------------------ - # async def correct_async(self) -> None: - # self.correct() # FIXME there is no async correction!!! - - - # out = run_script( - # script=self['correct'], - # args=self['args'], - # stdin=self['answer'], - # timeout=self['timeout'] - # ) - - # if out is None: - # logger.warning('No grade after running "%s".', self["correct"]) - # self['comments'] = 'O programa de correcção abortou...' - # self['grade'] = 0.0 - # elif isinstance(out, dict): - # self['comments'] = out.get('comments', '') - # try: - # self['grade'] = float(out['grade']) - # except ValueError: - # logger.error('Output error in "%s".', self["correct"]) - # except KeyError: - # logger.error('No grade in "%s".', self["correct"]) - # else: - # try: - # self['grade'] = float(out) - # except (TypeError, ValueError): - # logger.error('Invalid grade in "%s".', self["correct"]) - -# ============================================================================ class QuestionInformation(Question): ''' Not really a question, just an information panel. @@ -755,9 +575,6 @@ class QuestionInformation(Question): ''' # ------------------------------------------------------------------------ - # def __init__(self, q: QDict) -> None: - # super().__init__(q) - def gen(self) -> None: super().gen() self.set_defaults(QDict({ @@ -770,7 +587,6 @@ class QuestionInformation(Question): self['grade'] = 1.0 # always "correct" but points should be zero! - # ============================================================================ def question_from(qdict: QDict) -> Question: ''' @@ -783,7 +599,6 @@ def question_from(qdict: QDict) -> Question: 'text-regex': QuestionTextRegex, 'numeric-interval': QuestionNumericInterval, 'textarea': QuestionTextArea, - # 'code': QuestionCode, # -- informative panels -- 'information': QuestionInformation, 'success': QuestionInformation, @@ -856,7 +671,7 @@ class QFactory(): logger.debug('generating %s...', self.qdict["ref"]) # Shallow copy so that script generated questions will not replace # the original generators - qdict = self.qdict.copy() + qdict = QDict(self.qdict.copy()) qdict['qid'] = str(uuid.uuid4()) # unique for each question # If question is of generator type, an external program will be run diff --git a/perguntations/testfactory.py b/perguntations/testfactory.py index 79ef45b..a59d7f7 100644 --- a/perguntations/testfactory.py +++ b/perguntations/testfactory.py @@ -10,7 +10,7 @@ import re from typing import Any, Dict # this project -from perguntations.questions import QFactory, QuestionException +from perguntations.questions import QFactory, QuestionException, QDict from perguntations.test import Test from perguntations.tools import load_yaml @@ -99,14 +99,14 @@ class TestFactory(dict): if question['ref'] in qrefs: question.update(zip(('path', 'filename', 'index'), path.split(fullpath) + (i,))) - if question['type'] == 'code' and 'server' not in question: - try: - question['server'] = self['jobe_server'] - except KeyError as exc: - msg = f'Missing JOBE server in "{question["ref"]}"' - raise TestFactoryException(msg) from exc - - self['question_factory'][question['ref']] = QFactory(question) + # if question['type'] == 'code' and 'server' not in question: + # try: + # question['server'] = self['jobe_server'] + # except KeyError as exc: + # msg = f'Missing JOBE server in "{question["ref"]}"' + # raise TestFactoryException(msg) from exc + + self['question_factory'][question['ref']] = QFactory(QDict(question)) qmissing = qrefs.difference(set(self['question_factory'].keys())) if qmissing: diff --git a/perguntations/tools.py b/perguntations/tools.py index e75663c..163eab7 100644 --- a/perguntations/tools.py +++ b/perguntations/tools.py @@ -1,7 +1,6 @@ ''' -This module contains helper functions to: -- load yaml files and report errors -- run external programs (sync and async) +File: perguntations/tools.py +Description: Helper functions to load yaml files and run external programs. ''' -- libgit2 0.21.2