initial support for submitting code to a JOBE server

Miguel Barão
1 parent 4f960e53
Showing 7 changed files with 188 additions and 7 deletions Show diff stats
demo/demo.yaml
demo/questions/questions-tutorial.yaml
perguntations/__init__.py
perguntations/app.py
perguntations/questions.py
perguntations/serve.py
perguntations/templates/review-question.html
@@ -70,6 +70,8 @@ questions:
   - [tut-alert1, tut-alert2]
   - tut-generator
   - tut-yamllint
+  - tut-code
+
  
 # test:
 #   - ref1
@@ -609,3 +609,49 @@
         generate-question | yamllint -
         correct-answer | yamllint -
     ```
+
+# ----------------------------------------------------------------------------
+- type: code
+  ref: tut-code
+  title: Submissão de código (JOBE)
+  text: |
+    É possível enviar código para ser compilado e executado por um servidor
+    JOBE instalado separadamente, ver [JOBE](https://github.com/trampgeek/jobe).
+
+    ```yaml
+    - type: code
+      ref: tut-code
+      title: Submissão de código (JOBE)
+      text: |
+        Escreva um programa em C que recebe uma string no standard input e
+        mostra a mensagem `hello ` seguida da string.
+        Por exemplo, se o input for `Maria`, o output deverá ser `hello Maria`.
+      server: 127.0.0.1  # replace by appropriate address
+      language: c
+      correct:
+        - stdin: 'Maria'
+          stdout: 'hello Maria'
+        - stdin: 'xyz'
+          stdout: 'hello xyz'
+    ```
+
+    Existem várias linguagens suportadas pelo servidor JOBE (C, C++, Java,
+    Python2, Python3, Octave, Pascal, PHP).
+    O campo `correct` deverá ser uma lista de casos a testar.
+    Se um caso incluir `stdin`, este será enviado para o programa e o `stdout`
+    obtido será comparado com o declarado. A pergunta é considerada correcta se
+    todos os outputs coincidirem.
+  answer: |
+    #include <stdio.h>
+    int main() {
+      char name[20];
+      scanf("%s", name);
+      printf("hello %s", name);
+    }
+  server: 192.168.1.141
+  language: c
+  correct:
+    - stdin: 'Maria'
+      stdout: 'hello Maria'
+    - stdin: 'xyz'
+      stdout: 'hello xyz'
@@ -32,7 +32,7 @@ proof of submission and for review.
 '''
  
 APP_NAME = 'perguntations'
-APP_VERSION = '2020.11.dev2'
+APP_VERSION = '2020.11.dev3'
 APP_DESCRIPTION = __doc__
  
 __author__ = 'Miguel Barão'
@@ -190,7 +190,7 @@ class App():
             raise AppException('Failed to create test factory!') from exc
  
     # ------------------------------------------------------------------------
-    def _pregenerate_tests(self, num):
+    def _pregenerate_tests(self, num):  # TODO needs improvement
         event_loop = asyncio.get_event_loop()
         self.pregenerated_tests += [
                 event_loop.run_until_complete(self.testfactory.generate())
@@ -13,6 +13,12 @@ import re
 from typing import Any, Dict, NewType
 import uuid
  
+
+from urllib.error import HTTPError
+import json
+import http.client
+
+
 # this project
 from perguntations.tools import run_script, run_script_async
  
@@ -584,6 +590,130 @@ class QuestionTextArea(Question):
  
  
 # ============================================================================
+class QuestionCode(Question):
+    '''An instance of QuestionCode will always have the keys:
+        type (str)
+        text (str)
+        correct (str with script to run)
+        answer (None or an actual answer)
+    '''
+
+    _outcomes = {
+        0:  'JOBE outcome: Successful run',
+        11: 'JOBE outcome: Compile error',
+        12: 'JOBE outcome: Runtime error',
+        13: 'JOBE outcome: Time limit exceeded',
+        15: 'JOBE outcome: Successful run',
+        17: 'JOBE outcome: Memory limit exceeded',
+        19: 'JOBE outcome: Illegal system call',
+        20: 'JOBE outcome: Internal error, please report',
+        21: 'JOBE outcome: Server overload',
+        }
+
+    # ------------------------------------------------------------------------
+    def __init__(self, q: QDict) -> None:
+        super().__init__(q)
+
+        self.set_defaults(QDict({
+            'text': '',
+            'timeout': 5,   # seconds
+            'server': '127.0.0.1',  # JOBE server
+            'language': 'c',
+            'correct': [{'stdin': '', 'stdout': '', 'stderr': '', 'args': ''}],
+            }))
+
+    # ------------------------------------------------------------------------
+    def correct(self) -> None:
+        super().correct()
+
+        if self['answer'] is None:
+            return
+
+        # submit answer to JOBE server
+        resource = '/jobe/index.php/restapi/runs/'
+        headers = {"Content-type": "application/json; charset=utf-8",
+                   "Accept": "application/json"}
+
+        for expected in self['correct']:
+            data_json = json.dumps({
+                'run_spec' : {
+                    'language_id': self['language'],
+                    'sourcecode': self['answer'],
+                    'input': expected.get('stdin', ''),
+                     },
+                })
+
+            try:
+                connect = http.client.HTTPConnection(self['server'])
+                connect.request(
+                    method='POST',
+                    url=resource,
+                    body=data_json,
+                    headers=headers
+                    )
+                response = connect.getresponse()
+                logger.debug('JOBE response status %d', response.status)
+                if response.status != 204:
+                    content = response.read().decode('utf8')
+                    if content:
+                        result = json.loads(content)
+                connect.close()
+
+            except (HTTPError, ValueError):
+                logger.error('HTTPError while connecting to JOBE server')
+
+            try:
+                outcome = result['outcome']
+            except (NameError, TypeError, KeyError):
+                logger.error('Bad result returned from JOBE server: %s', result)
+                return
+            logger.debug(self._outcomes[outcome])
+
+            if result['cmpinfo']:  # compiler errors and warnings
+                self['comments'] = f'Erros de compilação:\n{result["cmpinfo"]}'
+                self['grade'] = 0.0
+                return
+
+            if result['stdout'] != expected['stdout']:
+                self['comments'] = 'O output gerado é diferente do esperado.'
+                self['grade'] = 0.0
+                return
+
+        self['comments'] = 'Ok!'
+        self['grade'] = 1.0
+
+
+    # ------------------------------------------------------------------------
+    async def correct_async(self) -> None:
+        self.correct()
+
+
+            # out = run_script(
+            #     script=self['correct'],
+            #     args=self['args'],
+            #     stdin=self['answer'],
+            #     timeout=self['timeout']
+            #     )
+
+            # if out is None:
+            #     logger.warning('No grade after running "%s".', self["correct"])
+            #     self['comments'] = 'O programa de correcção abortou...'
+            #     self['grade'] = 0.0
+            # elif isinstance(out, dict):
+            #     self['comments'] = out.get('comments', '')
+            #     try:
+            #         self['grade'] = float(out['grade'])
+            #     except ValueError:
+            #         logger.error('Output error in "%s".', self["correct"])
+            #     except KeyError:
+            #         logger.error('No grade in "%s".', self["correct"])
+            # else:
+            #     try:
+            #         self['grade'] = float(out)
+            #     except (TypeError, ValueError):
+            #         logger.error('Invalid grade in "%s".', self["correct"])
+
+# ============================================================================
 class QuestionInformation(Question):
     '''
     Not really a question, just an information panel.
@@ -645,6 +775,7 @@ class QFactory():
         'text-regex':       QuestionTextRegex,
         'numeric-interval': QuestionNumericInterval,
         'textarea':         QuestionTextArea,
+        'code':             QuestionCode,
         # -- informative panels --
         'information':      QuestionInformation,
         'success':          QuestionInformation,
@@ -227,6 +227,7 @@ class RootHandler(BaseHandler):
         'text-regex':   'question-text.html',
         'numeric-interval': 'question-text.html',
         'textarea':     'question-textarea.html',
+        'code':         'question-textarea.html',
         # -- information panels --
         'information':  'question-information.html',
         'success':      'question-information.html',
@@ -289,7 +290,7 @@ class RootHandler(BaseHandler):
                     else:
                         ans[i] = ans[i][0]
                 elif question['type'] in ('text', 'text-regex', 'textarea',
-                                          'numeric-interval'):
+                                          'numeric-interval', 'code'):
                     ans[i] = ans[i][0]
  
         # correct answered questions and logout
@@ -477,6 +478,7 @@ class ReviewHandler(BaseHandler):
         'text-regex':   'review-question-text.html',
         'numeric-interval': 'review-question-text.html',
         'textarea':     'review-question-text.html',
+        'code':         'review-question-text.html',
         # -- information panels --
         'information':  'review-question-information.html',
         'success':      'review-question-information.html',
@@ -47,7 +47,7 @@
             pontos
           </p>
           <p class="text-warning">{{ q['comments'] }}</p>
-          {% if q.get('solution', '') %}
+          {% if q['solution'] %}
             <hr>
             {{ md('**Solução:** \n\n' + q['solution']) }}
           {% end %}
@@ -57,8 +57,8 @@
             {{ round(q['grade'] * q['points'], 2) }}
             pontos
           </p>
-          <p class="text-danger">{{ q['comments'] }}</p>
-          {% if q.get('solution', '') %}
+          <p class="text-danger"><pre>{{ q['comments'] }}</pre></p>
+          {% if q['solution'] %}
             <hr>
             {{ md('**Solução:** \n\n' + q['solution']) }}
           {% end %}
@@ -102,7 +102,7 @@
           <i class="fas fa-ban fa-3x" aria-hidden="true"></i>
           {{ round(q['grade'] * q['points'], 2) }} pontos<br>
           {{ q['comments'] }}
-          {% if q.get('solution', '') %}
+          {% if q['solution'] %}
             <hr>
             {{ md('**Solução:** \n\n' + q['solution']) }}
           {% end %}
...	...	@@ -70,6 +70,8 @@ questions:
70	70	- [tut-alert1, tut-alert2]
71	71	- tut-generator
72	72	- tut-yamllint
	73	+ - tut-code
	74	+
73	75
74	76	# test:
75	77	# - ref1
...	...
...	...	@@ -609,3 +609,49 @@
609	609	generate-question \| yamllint -
610	610	correct-answer \| yamllint -
611	611	```
	612	+
	613	+# ----------------------------------------------------------------------------
	614	+- type: code
	615	+ ref: tut-code
	616	+ title: Submissão de código (JOBE)
	617	+ text: \|
	618	+ É possível enviar código para ser compilado e executado por um servidor
	619	+ JOBE instalado separadamente, ver [JOBE](https://github.com/trampgeek/jobe).
	620	+
	621	+ ```yaml
	622	+ - type: code
	623	+ ref: tut-code
	624	+ title: Submissão de código (JOBE)
	625	+ text: \|
	626	+ Escreva um programa em C que recebe uma string no standard input e
	627	+ mostra a mensagem `hello ` seguida da string.
	628	+ Por exemplo, se o input for `Maria`, o output deverá ser `hello Maria`.
	629	+ server: 127.0.0.1 # replace by appropriate address
	630	+ language: c
	631	+ correct:
	632	+ - stdin: 'Maria'
	633	+ stdout: 'hello Maria'
	634	+ - stdin: 'xyz'
	635	+ stdout: 'hello xyz'
	636	+ ```
	637	+
	638	+ Existem várias linguagens suportadas pelo servidor JOBE (C, C++, Java,
	639	+ Python2, Python3, Octave, Pascal, PHP).
	640	+ O campo `correct` deverá ser uma lista de casos a testar.
	641	+ Se um caso incluir `stdin`, este será enviado para o programa e o `stdout`
	642	+ obtido será comparado com o declarado. A pergunta é considerada correcta se
	643	+ todos os outputs coincidirem.
	644	+ answer: \|
	645	+ #include <stdio.h>
	646	+ int main() {
	647	+ char name[20];
	648	+ scanf("%s", name);
	649	+ printf("hello %s", name);
	650	+ }
	651	+ server: 192.168.1.141
	652	+ language: c
	653	+ correct:
	654	+ - stdin: 'Maria'
	655	+ stdout: 'hello Maria'
	656	+ - stdin: 'xyz'
	657	+ stdout: 'hello xyz'
...	...
...	...	@@ -32,7 +32,7 @@ proof of submission and for review.
32	32	'''
33	33
34	34	APP_NAME = 'perguntations'
35		-APP_VERSION = '2020.11.dev2'
	35	+APP_VERSION = '2020.11.dev3'
36	36	APP_DESCRIPTION = __doc__
37	37
38	38	__author__ = 'Miguel Barão'
...	...
...	...	@@ -190,7 +190,7 @@ class App():
190	190	raise AppException('Failed to create test factory!') from exc
191	191
192	192	# ------------------------------------------------------------------------
193		- def _pregenerate_tests(self, num):
	193	+ def _pregenerate_tests(self, num): # TODO needs improvement
194	194	event_loop = asyncio.get_event_loop()
195	195	self.pregenerated_tests += [
196	196	event_loop.run_until_complete(self.testfactory.generate())
...	...
...	...	@@ -13,6 +13,12 @@ import re
13	13	from typing import Any, Dict, NewType
14	14	import uuid
15	15
	16	+
	17	+from urllib.error import HTTPError
	18	+import json
	19	+import http.client
	20	+
	21	+
16	22	# this project
17	23	from perguntations.tools import run_script, run_script_async
18	24
...	...	@@ -584,6 +590,130 @@ class QuestionTextArea(Question):
584	590
585	591
586	592	# ============================================================================
	593	+class QuestionCode(Question):
	594	+ '''An instance of QuestionCode will always have the keys:
	595	+ type (str)
	596	+ text (str)
	597	+ correct (str with script to run)
	598	+ answer (None or an actual answer)
	599	+ '''
	600	+
	601	+ _outcomes = {
	602	+ 0: 'JOBE outcome: Successful run',
	603	+ 11: 'JOBE outcome: Compile error',
	604	+ 12: 'JOBE outcome: Runtime error',
	605	+ 13: 'JOBE outcome: Time limit exceeded',
	606	+ 15: 'JOBE outcome: Successful run',
	607	+ 17: 'JOBE outcome: Memory limit exceeded',
	608	+ 19: 'JOBE outcome: Illegal system call',
	609	+ 20: 'JOBE outcome: Internal error, please report',
	610	+ 21: 'JOBE outcome: Server overload',
	611	+ }
	612	+
	613	+ # ------------------------------------------------------------------------
	614	+ def __init__(self, q: QDict) -> None:
	615	+ super().__init__(q)
	616	+
	617	+ self.set_defaults(QDict({
	618	+ 'text': '',
	619	+ 'timeout': 5, # seconds
	620	+ 'server': '127.0.0.1', # JOBE server
	621	+ 'language': 'c',
	622	+ 'correct': [{'stdin': '', 'stdout': '', 'stderr': '', 'args': ''}],
	623	+ }))
	624	+
	625	+ # ------------------------------------------------------------------------
	626	+ def correct(self) -> None:
	627	+ super().correct()
	628	+
	629	+ if self['answer'] is None:
	630	+ return
	631	+
	632	+ # submit answer to JOBE server
	633	+ resource = '/jobe/index.php/restapi/runs/'
	634	+ headers = {"Content-type": "application/json; charset=utf-8",
	635	+ "Accept": "application/json"}
	636	+
	637	+ for expected in self['correct']:
	638	+ data_json = json.dumps({
	639	+ 'run_spec' : {
	640	+ 'language_id': self['language'],
	641	+ 'sourcecode': self['answer'],
	642	+ 'input': expected.get('stdin', ''),
	643	+ },
	644	+ })
	645	+
	646	+ try:
	647	+ connect = http.client.HTTPConnection(self['server'])
	648	+ connect.request(
	649	+ method='POST',
	650	+ url=resource,
	651	+ body=data_json,
	652	+ headers=headers
	653	+ )
	654	+ response = connect.getresponse()
	655	+ logger.debug('JOBE response status %d', response.status)
	656	+ if response.status != 204:
	657	+ content = response.read().decode('utf8')
	658	+ if content:
	659	+ result = json.loads(content)
	660	+ connect.close()
	661	+
	662	+ except (HTTPError, ValueError):
	663	+ logger.error('HTTPError while connecting to JOBE server')
	664	+
	665	+ try:
	666	+ outcome = result['outcome']
	667	+ except (NameError, TypeError, KeyError):
	668	+ logger.error('Bad result returned from JOBE server: %s', result)
	669	+ return
	670	+ logger.debug(self._outcomes[outcome])
	671	+
	672	+ if result['cmpinfo']: # compiler errors and warnings
	673	+ self['comments'] = f'Erros de compilação:\n{result["cmpinfo"]}'
	674	+ self['grade'] = 0.0
	675	+ return
	676	+
	677	+ if result['stdout'] != expected['stdout']:
	678	+ self['comments'] = 'O output gerado é diferente do esperado.'
	679	+ self['grade'] = 0.0
	680	+ return
	681	+
	682	+ self['comments'] = 'Ok!'
	683	+ self['grade'] = 1.0
	684	+
	685	+
	686	+ # ------------------------------------------------------------------------
	687	+ async def correct_async(self) -> None:
	688	+ self.correct()
	689	+
	690	+
	691	+ # out = run_script(
	692	+ # script=self['correct'],
	693	+ # args=self['args'],
	694	+ # stdin=self['answer'],
	695	+ # timeout=self['timeout']
	696	+ # )
	697	+
	698	+ # if out is None:
	699	+ # logger.warning('No grade after running "%s".', self["correct"])
	700	+ # self['comments'] = 'O programa de correcção abortou...'
	701	+ # self['grade'] = 0.0
	702	+ # elif isinstance(out, dict):
	703	+ # self['comments'] = out.get('comments', '')
	704	+ # try:
	705	+ # self['grade'] = float(out['grade'])
	706	+ # except ValueError:
	707	+ # logger.error('Output error in "%s".', self["correct"])
	708	+ # except KeyError:
	709	+ # logger.error('No grade in "%s".', self["correct"])
	710	+ # else:
	711	+ # try:
	712	+ # self['grade'] = float(out)
	713	+ # except (TypeError, ValueError):
	714	+ # logger.error('Invalid grade in "%s".', self["correct"])
	715	+
	716	+# ============================================================================
587	717	class QuestionInformation(Question):
588	718	'''
589	719	Not really a question, just an information panel.
...	...	@@ -645,6 +775,7 @@ class QFactory():
645	775	'text-regex': QuestionTextRegex,
646	776	'numeric-interval': QuestionNumericInterval,
647	777	'textarea': QuestionTextArea,
	778	+ 'code': QuestionCode,
648	779	# -- informative panels --
649	780	'information': QuestionInformation,
650	781	'success': QuestionInformation,
...	...
...	...	@@ -227,6 +227,7 @@ class RootHandler(BaseHandler):
227	227	'text-regex': 'question-text.html',
228	228	'numeric-interval': 'question-text.html',
229	229	'textarea': 'question-textarea.html',
	230	+ 'code': 'question-textarea.html',
230	231	# -- information panels --
231	232	'information': 'question-information.html',
232	233	'success': 'question-information.html',
...	...	@@ -289,7 +290,7 @@ class RootHandler(BaseHandler):
289	290	else:
290	291	ans[i] = ans[i][0]
291	292	elif question['type'] in ('text', 'text-regex', 'textarea',
292		- 'numeric-interval'):
	293	+ 'numeric-interval', 'code'):
293	294	ans[i] = ans[i][0]
294	295
295	296	# correct answered questions and logout
...	...	@@ -477,6 +478,7 @@ class ReviewHandler(BaseHandler):
477	478	'text-regex': 'review-question-text.html',
478	479	'numeric-interval': 'review-question-text.html',
479	480	'textarea': 'review-question-text.html',
	481	+ 'code': 'review-question-text.html',
480	482	# -- information panels --
481	483	'information': 'review-question-information.html',
482	484	'success': 'review-question-information.html',
...	...
...	...	@@ -47,7 +47,7 @@
47	47	pontos
48	48	</p>
49	49	<p class="text-warning">{{ q['comments'] }}</p>
50		- {% if q.get('solution', '') %}
	50	+ {% if q['solution'] %}
51	51	<hr>
52	52	{{ md('Solução: \n\n' + q['solution']) }}
53	53	{% end %}
...	...	@@ -57,8 +57,8 @@
57	57	{{ round(q['grade'] * q['points'], 2) }}
58	58	pontos
59	59	</p>
60		- <p class="text-danger">{{ q['comments'] }}</p>
61		- {% if q.get('solution', '') %}
	60	+ <p class="text-danger"><pre>{{ q['comments'] }}</pre></p>
	61	+ {% if q['solution'] %}
62	62	<hr>
63	63	{{ md('Solução: \n\n' + q['solution']) }}
64	64	{% end %}
...	...	@@ -102,7 +102,7 @@
102	102	<i class="fas fa-ban fa-3x" aria-hidden="true"></i>
103	103	{{ round(q['grade'] * q['points'], 2) }} pontos<br>
104	104	{{ q['comments'] }}
105		- {% if q.get('solution', '') %}
	105	+ {% if q['solution'] %}
106	106	<hr>
107	107	{{ md('Solução: \n\n' + q['solution']) }}
108	108	{% end %}
...	...