Use asyncio subprocesses to correct textarea questions.

Miguel Barão
1 parent c2abd6bf
Showing 7 changed files with 102 additions and 28 deletions Show diff stats
BUGS.md
aprendizations/learnapp.py
aprendizations/questions.py
aprendizations/serve.py
aprendizations/tools.py
demo/math/questions.yaml
demo/solar_system/questions.yaml
 # BUGS
+- apos clicar no botao responder, inactivar o input (importante quando o tempo de correcção é grande)
+- devia mostrar timout para o aluno saber a razao.
 - permitir configuracao para escolher entre static files locais ou remotos
 - sqlalchemy.pool.impl.NullPool: Exception during reset or similar
 sqlite3.ProgrammingError: SQLite objects created in a thread can only be used in that same thread.
@@ -102,7 +102,7 @@ class LearnApp(object):
                         continue  # to next test
         if errors > 0:
-            logger.info(f'{errors:>6} errors found.')
+            logger.error(f'{errors:>6} errors found.')
             raise LearnException('Sanity checks')
         else:
             logger.info('No errors found.')
@@ -4,12 +4,11 @@ import random
 import re
 from os import path
 import logging
-import asyncio
 from typing import Any, Dict, NewType
 import uuid
 # this project
-from .tools import run_script
+from .tools import run_script, run_script_async
 # setup logger for this module
 logger = logging.getLogger(__name__)
@@ -50,8 +49,9 @@ class Question(dict):
         self['grade'] = 0.0
     async def correct_async(self) -> None:
-        loop = asyncio.get_running_loop()
-        await loop.run_in_executor(None, self.correct)
+        self.correct()
+        # loop = asyncio.get_running_loop()
+        # await loop.run_in_executor(None, self.correct)
     def set_defaults(self, d: QDict) -> None:
         'Add k:v pairs from default dict d for nonexistent keys'
@@ -305,7 +305,7 @@ class QuestionNumericInterval(Question):
                 answer = float(self['answer'].replace(',', '.', 1))
             except ValueError:
                 self['comments'] = ('A resposta tem de ser numérica, '
-                                    'por exemplo 12.345.')
+                                    'por exemplo `12.345`.')
                 self['grade'] = 0.0
             else:
                 self['grade'] = 1.0 if lower <= answer <= upper else 0.0
@@ -327,13 +327,13 @@ class QuestionTextArea(Question):
         self.set_defaults(QDict({
             'text': '',
-            'lines': 8,
+            # 'lines': 8,
             'timeout': 5,  # seconds
             'correct': '',  # trying to execute this will fail => grade 0.0
             'args': []
             }))
-        self['correct'] = path.join(self['path'], self['correct'])   # FIXME
+        self['correct'] = path.join(self['path'], self['correct'])
     # ------------------------------------------------------------------------
     def correct(self) -> None:
@@ -347,7 +347,39 @@ class QuestionTextArea(Question):
                 timeout=self['timeout']
                 )
-            if isinstance(out, dict):
+            if out is None:
+                logger.warning(f'No grade after running "{self["correct"]}".')
+                self['grade'] = 0.0
+            elif isinstance(out, dict):
+                self['comments'] = out.get('comments', '')
+                try:
+                    self['grade'] = float(out['grade'])
+                except ValueError:
+                    logger.error(f'Output error in "{self["correct"]}".')
+                except KeyError:
+                    logger.error(f'No grade in "{self["correct"]}".')
+            else:
+                try:
+                    self['grade'] = float(out)
+                except (TypeError, ValueError):
+                    logger.error(f'Invalid grade in "{self["correct"]}".')
+
+    # ------------------------------------------------------------------------
+    async def correct_async(self) -> None:
+        super().correct()
+
+        if self['answer'] is not None:  # correct answer and parse yaml ouput
+            out = await run_script_async(
+                script=self['correct'],
+                args=self['args'],
+                stdin=self['answer'],
+                timeout=self['timeout']
+                )
+
+            if out is None:
+                logger.warning(f'No grade after running "{self["correct"]}".')
+                self['grade'] = 0.0
+            elif isinstance(out, dict):
                 self['comments'] = out.get('comments', '')
                 try:
                     self['grade'] = float(out['grade'])
@@ -372,7 +404,6 @@ class QuestionInformation(Question):
             }))
     # ------------------------------------------------------------------------
-    # can return negative values for wrong answers
     def correct(self) -> None:
         super().correct()
         self['grade'] = 1.0  # always "correct" but points should be zero!
 #!/usr/bin/env python3
 # python standard library
-import sys
+import argparse
+import asyncio
 import base64
-import uuid
+import functools
 import logging.config
-import argparse
 import mimetypes
+from os import path, environ
 import signal
-import functools
 import ssl
-import asyncio
-from os import path, environ
+import sys
+import uuid
 # third party libraries
 import tornado.ioloop
@@ -20,9 +20,9 @@ import tornado.web
 from tornado.escape import to_unicode
 # this project
+from . import APP_NAME
 from .learnapp import LearnApp
 from .tools import load_yaml, md_to_html
-from . import APP_NAME
 # ----------------------------------------------------------------------------
 # python standard library
-from os import path
-import subprocess
+import asyncio
 import logging
+from os import path
 import re
+import subprocess
 from typing import Any, List
 # third party libraries
-import yaml
 import mistune
 from pygments import highlight
 from pygments.lexers import get_lexer_by_name
 from pygments.formatters import HtmlFormatter
+import yaml
 # setup logger for this module
 logger = logging.getLogger(__name__)
@@ -100,14 +101,14 @@ class HighlightRenderer(mistune.Renderer):
         return highlight(code, lexer, formatter)
     def table(self, header, body):
-        return '<table class="table table-sm"><thead class="thead-light">' \
-               + header + '</thead><tbody>' + body + "</tbody></table>"
+        return (f'<table class="table table-sm"><thead class="thead-light">'
+                f'{header}</thead><tbody>{body}</tbody></table>')
     def image(self, src, title, alt):
         alt = mistune.escape(alt, quote=True)
         title = mistune.escape(title or '', quote=True)
-        return f'<img src="/file/{src}" class="img-fluid mx-auto d-block" ' \
-               f'alt="{alt}" title="{title}">'
+        return (f'<img src="/file/{src}" class="img-fluid mx-auto d-block" '
+                f'alt="{alt}" title="{title}">')
     # Pass math through unaltered - mathjax does the rendering in the browser
     def block_math(self, text):
@@ -200,3 +201,41 @@ def run_script(script: str,
                 logger.error(f'Error parsing yaml output of "{script}"')
             else:
                 return output
+
+
+# ----------------------------------------------------------------------------
+# Same as above, but asynchronous
+# ----------------------------------------------------------------------------
+async def run_script_async(script: str,
+                           args: List[str] = [],
+                           stdin: str = '',
+                           timeout: int = 5) -> Any:
+
+    script = path.expanduser(script)
+    cmd = [script] + [str(a) for a in args]
+
+    p = await asyncio.create_subprocess_shell(
+        *cmd,
+        stdin=asyncio.subprocess.PIPE,
+        stdout=asyncio.subprocess.PIPE,
+        stderr=asyncio.subprocess.DEVNULL,
+        )
+
+    try:
+        stdout, stderr = await asyncio.wait_for(
+            p.communicate(input=stdin.encode('utf-8')),
+            timeout=timeout
+            )
+    except asyncio.TimeoutError:
+        logger.warning(f'Timeout {timeout}s running script "{script}".')
+        return
+
+    if p.returncode != 0:
+        logger.error(f'Return code {p.returncode} running "{script}".')
+    else:
+        try:
+            output = yaml.safe_load(stdout.decode('utf-8'))
+        except Exception:
+            logger.error(f'Error parsing yaml output of "{script}"')
+        else:
+            return output
@@ -16,6 +16,7 @@
     - $2\times(9-2\times 3)$
   correct: [1, 1, 0, 0, 0]
   choose: 3
+  max_tries: 1
   solution: |
     Colocando o 2 em evidência, obtém-se a resposta correcta $2\times(9 - 3)$
     ou $(9-3)\times 2$.
@@ -50,6 +51,7 @@
     - 12
     - 14
     - 1, a **unidade**
+  max_tries: 1
   solution: |
     O único número primo é o 13.
@@ -76,9 +78,10 @@
     números negativos são menores que os positivos.
 # ---------------------------------------------------------------------------
+# the program should print a question in yaml format. The args will be
+# sent as command line options when the program is run.
 - type: generator
   ref: overflow
   script: gen-multiples-of-3.py
   args: [11, 120]
-  # the program should print a question in yaml format. The args will be
-  # sent as command line options when the program is run.
+  choose: 3
@@ -64,7 +64,6 @@
   # correct: correct-timeout.py
   # opcional
   answer: Vulcano, Krypton, Plutão
-  lines: 3
-  timeout: 50
+  timeout: 3
   solution: |
     Os 3 planetas mais perto do Sol são: Mercúrio, Vénus e Terra.
1		1
2	# BUGS	2	# BUGS
3		3
		4	+- apos clicar no botao responder, inactivar o input (importante quando o tempo de correcção é grande)
		5	+- devia mostrar timout para o aluno saber a razao.
4	- permitir configuracao para escolher entre static files locais ou remotos	6	- permitir configuracao para escolher entre static files locais ou remotos
5	- sqlalchemy.pool.impl.NullPool: Exception during reset or similar	7	- sqlalchemy.pool.impl.NullPool: Exception during reset or similar
6	sqlite3.ProgrammingError: SQLite objects created in a thread can only be used in that same thread.	8	sqlite3.ProgrammingError: SQLite objects created in a thread can only be used in that same thread.
	@@ -102,7 +102,7 @@ class LearnApp(object):		@@ -102,7 +102,7 @@ class LearnApp(object):
102	continue # to next test	102	continue # to next test
103		103
104	if errors > 0:	104	if errors > 0:
105	- logger.info(f'{errors:>6} errors found.')	105	+ logger.error(f'{errors:>6} errors found.')
106	raise LearnException('Sanity checks')	106	raise LearnException('Sanity checks')
107	else:	107	else:
108	logger.info('No errors found.')	108	logger.info('No errors found.')
1		1
2	# python standard library	2	# python standard library
3	-from os import path
4	-import subprocess	3	+import asyncio
5	import logging	4	import logging
		5	+from os import path
6	import re	6	import re
		7	+import subprocess
7	from typing import Any, List	8	from typing import Any, List
8		9
9	# third party libraries	10	# third party libraries
10	-import yaml
11	import mistune	11	import mistune
12	from pygments import highlight	12	from pygments import highlight
13	from pygments.lexers import get_lexer_by_name	13	from pygments.lexers import get_lexer_by_name
14	from pygments.formatters import HtmlFormatter	14	from pygments.formatters import HtmlFormatter
		15	+import yaml
15		16
16	# setup logger for this module	17	# setup logger for this module
17	logger = logging.getLogger(__name__)	18	logger = logging.getLogger(__name__)
	@@ -100,14 +101,14 @@ class HighlightRenderer(mistune.Renderer):		@@ -100,14 +101,14 @@ class HighlightRenderer(mistune.Renderer):
100	return highlight(code, lexer, formatter)	101	return highlight(code, lexer, formatter)
101		102
102	def table(self, header, body):	103	def table(self, header, body):
103	- return '<table class="table table-sm"><thead class="thead-light">' \
104	- + header + '</thead><tbody>' + body + "</tbody></table>"	104	+ return (f'<table class="table table-sm"><thead class="thead-light">'
		105	+ f'{header}</thead><tbody>{body}</tbody></table>')
105		106
106	def image(self, src, title, alt):	107	def image(self, src, title, alt):
107	alt = mistune.escape(alt, quote=True)	108	alt = mistune.escape(alt, quote=True)
108	title = mistune.escape(title or '', quote=True)	109	title = mistune.escape(title or '', quote=True)
109	- return f'<img src="/file/{src}" class="img-fluid mx-auto d-block" ' \
110	- f'alt="{alt}" title="{title}">'	110	+ return (f'<img src="/file/{src}" class="img-fluid mx-auto d-block" '
		111	+ f'alt="{alt}" title="{title}">')
111		112
112	# Pass math through unaltered - mathjax does the rendering in the browser	113	# Pass math through unaltered - mathjax does the rendering in the browser
113	def block_math(self, text):	114	def block_math(self, text):
	@@ -200,3 +201,41 @@ def run_script(script: str,		@@ -200,3 +201,41 @@ def run_script(script: str,
200	logger.error(f'Error parsing yaml output of "{script}"')	201	logger.error(f'Error parsing yaml output of "{script}"')
201	else:	202	else:
202	return output	203	return output
		204	+
		205	+
		206	+# ----------------------------------------------------------------------------
		207	+# Same as above, but asynchronous
		208	+# ----------------------------------------------------------------------------
		209	+async def run_script_async(script: str,
		210	+ args: List[str] = [],
		211	+ stdin: str = '',
		212	+ timeout: int = 5) -> Any:
		213	+
		214	+ script = path.expanduser(script)
		215	+ cmd = [script] + [str(a) for a in args]
		216	+
		217	+ p = await asyncio.create_subprocess_shell(
		218	+ *cmd,
		219	+ stdin=asyncio.subprocess.PIPE,
		220	+ stdout=asyncio.subprocess.PIPE,
		221	+ stderr=asyncio.subprocess.DEVNULL,
		222	+ )
		223	+
		224	+ try:
		225	+ stdout, stderr = await asyncio.wait_for(
		226	+ p.communicate(input=stdin.encode('utf-8')),
		227	+ timeout=timeout
		228	+ )
		229	+ except asyncio.TimeoutError:
		230	+ logger.warning(f'Timeout {timeout}s running script "{script}".')
		231	+ return
		232	+
		233	+ if p.returncode != 0:
		234	+ logger.error(f'Return code {p.returncode} running "{script}".')
		235	+ else:
		236	+ try:
		237	+ output = yaml.safe_load(stdout.decode('utf-8'))
		238	+ except Exception:
		239	+ logger.error(f'Error parsing yaml output of "{script}"')
		240	+ else:
		241	+ return output