refactor checking questions.

add check for correction of code and textarea.

refactor checking questions.
add check for correction of code and textarea.
Miguel Barão
1 parent bab5f360
Showing 3 changed files with 42 additions and 30 deletions Show diff stats
BUGS.md
perguntations/__init__.py
perguntations/testfactory.py
 # BUGS
+- JOBE correct async
 - em caso de timeout na submissão (e.g. JOBE ou script nao responde) a correcção não termina e o teste não é guardado.
 - QuestionCode falta reportar nos comments os vários erros que podem ocorrer (timeout, etc)
 - permitir remover alunos que estão online para poderem comecar de novo.
@@ -32,7 +32,7 @@ proof of submission and for review.
 '''
 APP_NAME = 'perguntations'
-APP_VERSION = '2020.11.dev3'
+APP_VERSION = '2020.11.dev4'
 APP_DESCRIPTION = __doc__
 __author__ = 'Miguel Barão'
@@ -43,7 +43,7 @@ class TestFactory(dict):
         super().__init__({  # defaults
             'title': '',
             'show_points': True,
-            'scale': None,  # or [0, 20]
+            'scale': None,
             'duration': 0,  # 0=infinite
             'autosubmit': False,
             'debug': False,
@@ -66,12 +66,10 @@ class TestFactory(dict):
                     len(qrefs), len(self["questions"]))
         # --- load and build question factories
-        self.question_factory = {}
+        self['question_factory'] = {}
-        counter = 1
         for file in self["files"]:
             fullpath = path.normpath(path.join(self["questions_dir"], file))
-            (dirname, filename) = path.split(fullpath)
             logger.info('Loading "%s"...', fullpath)
             questions = load_yaml(fullpath)   # , default=[])
@@ -88,8 +86,8 @@ class TestFactory(dict):
                     logger.warning('Missing ref set to "%s"', question["ref"])
                 # check for duplicate refs
-                if question['ref'] in self.question_factory:
-                    other = self.question_factory[question['ref']]
+                if question['ref'] in self['question_factory']:
+                    other = self['question_factory'][question['ref']]
                     otherfile = path.join(other.question['path'],
                                           other.question['filename'])
                     msg = (f'Duplicate reference "{question["ref"]}" in files '
@@ -98,36 +96,23 @@ class TestFactory(dict):
                 # make factory only for the questions used in the test
                 if question['ref'] in qrefs:
-                    # question.setdefault('type', 'information')
-                    question.update({
-                        'filename': filename,
-                        'path': dirname,
-                        'index': i            # position in the file, 0 based
-                        })
+                    question.update(zip(('path', 'filename', 'index'),
+                                        path.split(fullpath) + (i,)))
                     if question['type'] == 'code' and 'server' not in question:
                         try:
                             question['server'] = self['jobe_server']
                         except KeyError as exc:
                             msg = f'Missing JOBE server in "{question["ref"]}"'
-                            raise TestFactoryException(msg)
-
-                    self.question_factory[question['ref']] = QFactory(question)
-
-                    # check if all the questions can be correctly generated
-                    # TODO and corrected
-                    try:
-                        self.question_factory[question['ref']].generate()
-                    except Exception as exc:
-                        msg = f'Failed to generate "{question["ref"]}"'
-                        raise TestFactoryException(msg) from exc
-                    else:
-                        logger.info('%4d.  "%s" Ok.', counter, question["ref"])
-                    counter += 1
-
-        qmissing = qrefs.difference(set(self.question_factory.keys()))
+                            raise TestFactoryException(msg) from exc
+
+                    self['question_factory'][question['ref']] = QFactory(question)
+
+        qmissing = qrefs.difference(set(self['question_factory'].keys()))
         if qmissing:
             raise TestFactoryException(f'Could not find questions {qmissing}.')
+        self.check_questions()
+
         logger.info('Test factory ready. No errors found.')
@@ -231,6 +216,32 @@ class TestFactory(dict):
         self.check_grade_scaling()
     # ------------------------------------------------------------------------
+    def check_questions(self) -> None:
+        '''
+        checks if questions can be correctly generated and corrected
+        '''
+        logger.info('Checking if questions can be generated and corrected...')
+        for i, (qref, qfact) in enumerate(self['question_factory'].items()):
+            logger.info('%4d.  %s:', i, qref)
+            try:
+                question = qfact.generate()
+            except Exception as exc:
+                msg = f'Failed to generate "{qref}"'
+                raise TestFactoryException(msg) from exc
+            else:
+                logger.info('         generate Ok')
+
+            if question['type'] in ('code', 'textarea'):
+                try:
+                    question.set_answer('')
+                    question.correct()
+                except Exception as exc:
+                    msg = f'Failed to correct "{qref}"'
+                    raise TestFactoryException(msg) from exc
+                else:
+                    logger.info('         correct Ok')
+
+    # ------------------------------------------------------------------------
     async def generate(self):
         '''
         Given a dictionary with a student dict {'name':'john', 'number': 123}
@@ -250,7 +261,7 @@ class TestFactory(dict):
             for qref in qrefs:
                 # generate instance of question
                 try:
-                    question = await self.question_factory[qref].gen_async()
+                    question = await self['question_factory'][qref].gen_async()
                 except QuestionException:
                     logger.error('Can\'t generate question "%s". Skipping.', qref)
                     nerr += 1
1		1
2	# BUGS	2	# BUGS
3		3
		4	+- JOBE correct async
4	- em caso de timeout na submissão (e.g. JOBE ou script nao responde) a correcção não termina e o teste não é guardado.	5	- em caso de timeout na submissão (e.g. JOBE ou script nao responde) a correcção não termina e o teste não é guardado.
5	- QuestionCode falta reportar nos comments os vários erros que podem ocorrer (timeout, etc)	6	- QuestionCode falta reportar nos comments os vários erros que podem ocorrer (timeout, etc)
6	- permitir remover alunos que estão online para poderem comecar de novo.	7	- permitir remover alunos que estão online para poderem comecar de novo.
	@@ -32,7 +32,7 @@ proof of submission and for review.		@@ -32,7 +32,7 @@ proof of submission and for review.
32	'''	32	'''
33		33
34	APP_NAME = 'perguntations'	34	APP_NAME = 'perguntations'
35	-APP_VERSION = '2020.11.dev3'	35	+APP_VERSION = '2020.11.dev4'
36	APP_DESCRIPTION = __doc__	36	APP_DESCRIPTION = __doc__
37		37
38	__author__ = 'Miguel Barão'	38	__author__ = 'Miguel Barão'