# python standard library import asyncio import random import re from os import path import logging from typing import Any, Dict, NewType import uuid # this project from .tools import run_script, run_script_async # setup logger for this module logger = logging.getLogger(__name__) QDict = NewType('QDict', Dict[str, Any]) class QuestionException(Exception): pass # ============================================================================ # Questions derived from Question are already instantiated and ready to be # presented to students. # ============================================================================ class Question(dict): ''' Classes derived from this base class are meant to instantiate questions for each student. Instances can shuffle options or automatically generate questions. ''' def __init__(self, q: QDict) -> None: super().__init__(q) # add required keys if missing self.set_defaults(QDict({ 'title': '', 'answer': None, 'comments': '', 'solution': '', 'files': {}, })) def correct(self) -> None: self['comments'] = '' self['grade'] = 0.0 async def correct_async(self) -> None: self.correct() def set_defaults(self, d: QDict) -> None: 'Add k:v pairs from default dict d for nonexistent keys' for k, v in d.items(): self.setdefault(k, v) # ============================================================================ class QuestionRadio(Question): '''An instance of QuestionRadio will always have the keys: type (str) text (str) options (list of strings) correct (list of floats) discount (bool, default=True) answer (None or an actual answer) shuffle (bool, default=True) choose (int) # only used if shuffle=True ''' # ------------------------------------------------------------------------ def __init__(self, q: QDict) -> None: super().__init__(q) n = len(self['options']) self.set_defaults(QDict({ 'text': '', 'correct': 0, 'shuffle': True, 'discount': True, 'max_tries': (n + 3) // 4 # 1 try for each 4 options })) # check correct bounds and convert int to list, # e.g. correct: 2 --> correct: [0,0,1,0,0] if isinstance(self['correct'], int): if not (0 <= self['correct'] < n): msg = (f'Correct option not in range 0..{n-1} in ' f'"{self["ref"]}"') raise QuestionException(msg) self['correct'] = [1.0 if x == self['correct'] else 0.0 for x in range(n)] elif isinstance(self['correct'], list): # must match number of options if len(self['correct']) != n: msg = (f'Incompatible sizes: {n} options vs ' f'{len(self["correct"])} correct in "{self["ref"]}"') raise QuestionException(msg) # make sure is a list of floats try: self['correct'] = [float(x) for x in self['correct']] except (ValueError, TypeError): msg = (f'Correct list must contain numbers [0.0, 1.0] or ' f'booleans in "{self["ref"]}"') raise QuestionException(msg) # check grade boundaries if self['discount'] and not all(0.0 <= x <= 1.0 for x in self['correct']): msg = (f'Correct values must be in the interval [0.0, 1.0] in ' f'"{self["ref"]}"') raise QuestionException(msg) # at least one correct option if all(x < 1.0 for x in self['correct']): msg = (f'At least one correct option is required in ' f'"{self["ref"]}"') raise QuestionException(msg) # If shuffle==false, all options are shown as defined # otherwise, select 1 correct and choose a few wrong ones if self['shuffle']: # lists with indices of right and wrong options right = [i for i in range(n) if self['correct'][i] >= 1] wrong = [i for i in range(n) if self['correct'][i] < 1] self.set_defaults(QDict({'choose': 1+len(wrong)})) # try to choose 1 correct option if right: r = random.choice(right) options = [self['options'][r]] correct = [self['correct'][r]] else: options = [] correct = [] # choose remaining wrong options nwrong = self['choose'] - len(correct) wrongsample = random.sample(wrong, k=nwrong) options += [self['options'][i] for i in wrongsample] correct += [self['correct'][i] for i in wrongsample] # final shuffle of the options perm = random.sample(range(self['choose']), k=self['choose']) self['options'] = [str(options[i]) for i in perm] self['correct'] = [correct[i] for i in perm] # ------------------------------------------------------------------------ # can assign negative grades for wrong answers def correct(self) -> None: super().correct() if self['answer'] is not None: x = self['correct'][int(self['answer'])] # get grade of the answer n = len(self['options']) x_aver = sum(self['correct']) / n # expected value of grade # note: there are no numerical errors when summing 1.0s so the # x_aver can be exactly 1.0 if all options are right if self['discount'] and x_aver != 1.0: x = (x - x_aver) / (1.0 - x_aver) self['grade'] = x # ============================================================================ class QuestionCheckbox(Question): '''An instance of QuestionCheckbox will always have the keys: type (str) text (str) options (list of strings) shuffle (bool, default True) correct (list of floats) discount (bool, default True) choose (int) answer (None or an actual answer) ''' # ------------------------------------------------------------------------ def __init__(self, q: QDict) -> None: super().__init__(q) n = len(self['options']) # set defaults if missing self.set_defaults(QDict({ 'text': '', 'correct': [1.0] * n, # Using 0.0 breaks (right, wrong) options 'shuffle': True, 'discount': True, 'choose': n, # number of options 'max_tries': max(1, min(n - 1, 3)) })) # must be a list of numbers if not isinstance(self['correct'], list): msg = 'Correct must be a list of numbers or booleans' raise QuestionException(msg) # must match number of options if len(self['correct']) != n: msg = (f'Incompatible sizes: {n} options vs ' f'{len(self["correct"])} correct in "{self["ref"]}"') raise QuestionException(msg) # make sure is a list of floats try: self['correct'] = [float(x) for x in self['correct']] except (ValueError, TypeError): msg = (f'Correct list must contain numbers or ' f'booleans in "{self["ref"]}"') raise QuestionException(msg) # check grade boundaries if self['discount'] and not all(0.0 <= x <= 1.0 for x in self['correct']): msg0 = ('+-------------- BEHAVIOR CHANGE NOTICE --------------+') msg1 = ('| Correct values must be in the interval [0.0, 1.0]. |') msg2 = ('| I will convert to the new behavior, but you should |') msg3 = ('| fix it in the question. |') msg4 = ('+----------------------------------------------------+') logger.warning(msg0) logger.warning(msg1) logger.warning(msg2) logger.warning(msg3) logger.warning(msg4) logger.warning(f'-> please fix "{self["ref"]}"') # normalize to [0,1] self['correct'] = [(x+1)/2 for x in self['correct']] # if an option is a list of (right, wrong), pick one options = [] correct = [] for o, c in zip(self['options'], self['correct']): if isinstance(o, list): r = random.randint(0, 1) o = o[r] if r == 1: # c = -c c = 1.0 - c options.append(str(o)) correct.append(c) # generate random permutation, e.g. [2,1,4,0,3] # and apply to `options` and `correct` if self['shuffle']: perm = random.sample(range(n), k=self['choose']) self['options'] = [options[i] for i in perm] self['correct'] = [correct[i] for i in perm] else: self['options'] = options[:self['choose']] self['correct'] = correct[:self['choose']] # ------------------------------------------------------------------------ # can return negative values for wrong answers def correct(self) -> None: super().correct() if self['answer'] is not None: x = 0.0 if self['discount']: sum_abs = sum(abs(2*p-1) for p in self['correct']) for i, p in enumerate(self['correct']): x += 2*p-1 if str(i) in self['answer'] else 1-2*p else: sum_abs = sum(abs(p) for p in self['correct']) for i, p in enumerate(self['correct']): x += p if str(i) in self['answer'] else 0.0 try: self['grade'] = x / sum_abs except ZeroDivisionError: self['grade'] = 1.0 # limit p->0 # ============================================================================ class QuestionText(Question): '''An instance of QuestionText will always have the keys: type (str) text (str) correct (list of str) answer (None or an actual answer) ''' # ------------------------------------------------------------------------ def __init__(self, q: QDict) -> None: super().__init__(q) self.set_defaults(QDict({ 'text': '', 'correct': [], # no correct answers, always wrong 'transform': [], # transformations applied to the answer, in order })) # make sure its always a list of possible correct answers if not isinstance(self['correct'], list): self['correct'] = [str(self['correct'])] else: # make sure all elements of the list are strings self['correct'] = [str(a) for a in self['correct']] for f in self['transform']: if f not in ('remove_space', 'trim', 'normalize_space', 'lower', 'upper'): msg = (f'Unknown transform "{f}" in "{self["ref"]}"') raise QuestionException(msg) # check if answers are invariant with respect to the transforms if any(c != self.transform(c) for c in self['correct']): logger.warning(f'in "{self["ref"]}", correct answers are not ' 'invariant wrt transformations => never correct') # ------------------------------------------------------------------------ # apply optional filters to the answer def transform(self, ans): for f in self['transform']: if f == 'remove_space': # removes all spaces ans = ans.replace(' ', '') elif f == 'trim': # removes spaces around ans = ans.strip() elif f == 'normalize_space': # replaces multiple spaces by one ans = re.sub(r'\s+', ' ', ans.strip()) elif f == 'lower': # convert to lowercase ans = ans.lower() elif f == 'upper': # convert to uppercase ans = ans.upper() else: logger.warning(f'in "{self["ref"]}", unknown transform "{f}"') return ans # ------------------------------------------------------------------------ def correct(self) -> None: super().correct() if self['answer'] is not None: answer = self.transform(self['answer']) # apply transformations self['grade'] = 1.0 if answer in self['correct'] else 0.0 # ============================================================================ class QuestionTextRegex(Question): '''An instance of QuestionTextRegex will always have the keys: type (str) text (str) correct (str or list[str]) answer (None or an actual answer) The correct strings are python standard regular expressions. Grade is 1.0 when the answer matches any of the regex in the list. ''' # ------------------------------------------------------------------------ def __init__(self, q: QDict) -> None: super().__init__(q) self.set_defaults(QDict({ 'text': '', 'correct': ['$.^'], # will always return false })) # make sure its always a list of regular expressions if not isinstance(self['correct'], list): self['correct'] = [self['correct']] # converts patterns to compiled versions try: self['correct'] = [re.compile(a) for a in self['correct']] except Exception: msg = f'Failed to compile regex in "{self["ref"]}"' raise QuestionException(msg) # ------------------------------------------------------------------------ def correct(self) -> None: super().correct() if self['answer'] is not None: self['grade'] = 0.0 for r in self['correct']: try: if r.match(self['answer']): self['grade'] = 1.0 return except TypeError: logger.error(f'While matching regex {r.pattern} with ' f'answer "{self["answer"]}".') # ============================================================================ class QuestionNumericInterval(Question): '''An instance of QuestionTextNumeric will always have the keys: type (str) text (str) correct (list [lower bound, upper bound]) answer (None or an actual answer) An answer is correct if it's in the closed interval. ''' # ------------------------------------------------------------------------ def __init__(self, q: QDict) -> None: super().__init__(q) self.set_defaults(QDict({ 'text': '', 'correct': [1.0, -1.0], # will always return false })) # if only one number n is given, make an interval [n,n] if isinstance(self['correct'], (int, float)): self['correct'] = [float(self['correct']), float(self['correct'])] # make sure its a list of two numbers elif isinstance(self['correct'], list): if len(self['correct']) != 2: msg = (f'Numeric interval must be a list with two numbers, in ' f'{self["ref"]}') raise QuestionException(msg) try: self['correct'] = [float(n) for n in self['correct']] except Exception: msg = (f'Numeric interval must be a list with two numbers, in ' f'{self["ref"]}') raise QuestionException(msg) # invalid else: msg = (f'Numeric interval must be a list with two numbers, in ' f'{self["ref"]}') raise QuestionException(msg) # ------------------------------------------------------------------------ def correct(self) -> None: super().correct() if self['answer'] is not None: lower, upper = self['correct'] try: # replace , by . and convert to float answer = float(self['answer'].replace(',', '.', 1)) except ValueError: self['comments'] = ('A resposta tem de ser numérica, ' 'por exemplo `12.345`.') self['grade'] = 0.0 else: self['grade'] = 1.0 if lower <= answer <= upper else 0.0 # ============================================================================ class QuestionTextArea(Question): '''An instance of QuestionTextArea will always have the keys: type (str) text (str) correct (str with script to run) answer (None or an actual answer) ''' # ------------------------------------------------------------------------ def __init__(self, q: QDict) -> None: super().__init__(q) self.set_defaults(QDict({ 'text': '', 'timeout': 5, # seconds 'correct': '', # trying to execute this will fail => grade 0.0 'args': [] })) self['correct'] = path.join(self['path'], self['correct']) # ------------------------------------------------------------------------ def correct(self) -> None: super().correct() if self['answer'] is not None: # correct answer and parse yaml ouput out = run_script( script=self['correct'], args=self['args'], stdin=self['answer'], timeout=self['timeout'] ) if out is None: logger.warning(f'No grade after running "{self["correct"]}".') self['grade'] = 0.0 elif isinstance(out, dict): self['comments'] = out.get('comments', '') try: self['grade'] = float(out['grade']) except ValueError: logger.error(f'Output error in "{self["correct"]}".') except KeyError: logger.error(f'No grade in "{self["correct"]}".') else: try: self['grade'] = float(out) except (TypeError, ValueError): logger.error(f'Invalid grade in "{self["correct"]}".') # ------------------------------------------------------------------------ async def correct_async(self) -> None: super().correct() if self['answer'] is not None: # correct answer and parse yaml ouput out = await run_script_async( script=self['correct'], args=self['args'], stdin=self['answer'], timeout=self['timeout'] ) if out is None: logger.warning(f'No grade after running "{self["correct"]}".') self['grade'] = 0.0 elif isinstance(out, dict): self['comments'] = out.get('comments', '') try: self['grade'] = float(out['grade']) except ValueError: logger.error(f'Output error in "{self["correct"]}".') except KeyError: logger.error(f'No grade in "{self["correct"]}".') else: try: self['grade'] = float(out) except (TypeError, ValueError): logger.error(f'Invalid grade in "{self["correct"]}".') # ============================================================================ class QuestionInformation(Question): # ------------------------------------------------------------------------ def __init__(self, q: QDict) -> None: super().__init__(q) self.set_defaults(QDict({ 'text': '', })) # ------------------------------------------------------------------------ def correct(self) -> None: super().correct() self['grade'] = 1.0 # always "correct" but points should be zero! # ============================================================================ # # QFactory is a class that can generate question instances, e.g. by shuffling # options, running a script to generate the question, etc. # # To generate an instance of a question we use the method generate(). # It returns a question instance of the correct class. # There is also an asynchronous version called gen_async(). This version is # synchronous for all question types (radio, checkbox, etc) except for # generator types which run asynchronously. # # Example: # # # make a factory for a question # qfactory = QFactory({ # 'type': 'radio', # 'text': 'Choose one', # 'options': ['a', 'b'] # }) # # # generate synchronously # question = qfactory.generate() # # # generate asynchronously # question = await qfactory.gen_async() # # # answer one question and correct it # question['answer'] = 42 # set answer # question.correct() # correct answer # grade = question['grade'] # get grade # # ============================================================================ class QFactory(object): # Depending on the type of question, a different question class will be # instantiated. All these classes derive from the base class `Question`. _types = { 'radio': QuestionRadio, 'checkbox': QuestionCheckbox, 'text': QuestionText, 'text-regex': QuestionTextRegex, 'numeric-interval': QuestionNumericInterval, 'textarea': QuestionTextArea, # -- informative panels -- 'information': QuestionInformation, 'success': QuestionInformation, 'warning': QuestionInformation, 'alert': QuestionInformation, } def __init__(self, qdict: QDict = QDict({})) -> None: self.question = qdict # ------------------------------------------------------------------------ # generates a question instance of QuestionRadio, QuestionCheckbox, ..., # which is a descendent of base class Question. # ------------------------------------------------------------------------ async def gen_async(self) -> Question: logger.debug(f'generating {self.question["ref"]}...') # Shallow copy so that script generated questions will not replace # the original generators q = self.question.copy() q['qid'] = str(uuid.uuid4()) # unique for each generated question # If question is of generator type, an external program will be run # which will print a valid question in yaml format to stdout. This # output is then yaml parsed into a dictionary `q`. if q['type'] == 'generator': logger.debug(f' \\_ Running "{q["script"]}".') q.setdefault('args', []) q.setdefault('stdin', '') script = path.join(q['path'], q['script']) out = await run_script_async(script=script, args=q['args'], stdin=q['stdin']) q.update(out) # Get class for this question type try: qclass = self._types[q['type']] except KeyError: logger.error(f'Invalid type "{q["type"]}" in "{q["ref"]}"') raise # Finally create an instance of Question() try: qinstance = qclass(QDict(q)) except QuestionException as e: # logger.error(e) raise e return qinstance # ------------------------------------------------------------------------ def generate(self) -> Question: return asyncio.get_event_loop().run_until_complete(self.gen_async())