initial support for submitting code to a JOBE server

Miguel Barão
1 parent 4f960e53
Showing 7 changed files with 188 additions and 7 deletions Show diff stats
demo/demo.yaml
demo/questions/questions-tutorial.yaml
perguntations/__init__.py
perguntations/app.py
perguntations/questions.py
perguntations/serve.py
perguntations/templates/review-question.html
@@ -70,6 +70,8 @@ questions:
   - [tut-alert1, tut-alert2]
   - tut-generator
   - tut-yamllint
+  - tut-code
+
 # test:
 #   - ref1
@@ -609,3 +609,49 @@
         generate-question | yamllint -
         correct-answer | yamllint -
     ```
+
+# ----------------------------------------------------------------------------
+- type: code
+  ref: tut-code
+  title: Submissão de código (JOBE)
+  text: |
+    É possível enviar código para ser compilado e executado por um servidor
+    JOBE instalado separadamente, ver [JOBE](https://github.com/trampgeek/jobe).
+
+    ```yaml
+    - type: code
+      ref: tut-code
+      title: Submissão de código (JOBE)
+      text: |
+        Escreva um programa em C que recebe uma string no standard input e
+        mostra a mensagem `hello ` seguida da string.
+        Por exemplo, se o input for `Maria`, o output deverá ser `hello Maria`.
+      server: 127.0.0.1  # replace by appropriate address
+      language: c
+      correct:
+        - stdin: 'Maria'
+          stdout: 'hello Maria'
+        - stdin: 'xyz'
+          stdout: 'hello xyz'
+    ```
+
+    Existem várias linguagens suportadas pelo servidor JOBE (C, C++, Java,
+    Python2, Python3, Octave, Pascal, PHP).
+    O campo `correct` deverá ser uma lista de casos a testar.
+    Se um caso incluir `stdin`, este será enviado para o programa e o `stdout`
+    obtido será comparado com o declarado. A pergunta é considerada correcta se
+    todos os outputs coincidirem.
+  answer: |
+    #include <stdio.h>
+    int main() {
+      char name[20];
+      scanf("%s", name);
+      printf("hello %s", name);
+    }
+  server: 192.168.1.141
+  language: c
+  correct:
+    - stdin: 'Maria'
+      stdout: 'hello Maria'
+    - stdin: 'xyz'
+      stdout: 'hello xyz'
@@ -32,7 +32,7 @@ proof of submission and for review.
 '''
 APP_NAME = 'perguntations'
-APP_VERSION = '2020.11.dev2'
+APP_VERSION = '2020.11.dev3'
 APP_DESCRIPTION = __doc__
 __author__ = 'Miguel Barão'
@@ -190,7 +190,7 @@ class App():
             raise AppException('Failed to create test factory!') from exc
     # ------------------------------------------------------------------------
-    def _pregenerate_tests(self, num):
+    def _pregenerate_tests(self, num):  # TODO needs improvement
         event_loop = asyncio.get_event_loop()
         self.pregenerated_tests += [
                 event_loop.run_until_complete(self.testfactory.generate())
@@ -13,6 +13,12 @@ import re
 from typing import Any, Dict, NewType
 import uuid
+
+from urllib.error import HTTPError
+import json
+import http.client
+
+
 # this project
 from perguntations.tools import run_script, run_script_async
@@ -584,6 +590,130 @@ class QuestionTextArea(Question):
 # ============================================================================
+class QuestionCode(Question):
+    '''An instance of QuestionCode will always have the keys:
+        type (str)
+        text (str)
+        correct (str with script to run)
+        answer (None or an actual answer)
+    '''
+
+    _outcomes = {
+        0:  'JOBE outcome: Successful run',
+        11: 'JOBE outcome: Compile error',
+        12: 'JOBE outcome: Runtime error',
+        13: 'JOBE outcome: Time limit exceeded',
+        15: 'JOBE outcome: Successful run',
+        17: 'JOBE outcome: Memory limit exceeded',
+        19: 'JOBE outcome: Illegal system call',
+        20: 'JOBE outcome: Internal error, please report',
+        21: 'JOBE outcome: Server overload',
+        }
+
+    # ------------------------------------------------------------------------
+    def __init__(self, q: QDict) -> None:
+        super().__init__(q)
+
+        self.set_defaults(QDict({
+            'text': '',
+            'timeout': 5,   # seconds
+            'server': '127.0.0.1',  # JOBE server
+            'language': 'c',
+            'correct': [{'stdin': '', 'stdout': '', 'stderr': '', 'args': ''}],
+            }))
+
+    # ------------------------------------------------------------------------
+    def correct(self) -> None:
+        super().correct()
+
+        if self['answer'] is None:
+            return
+
+        # submit answer to JOBE server
+        resource = '/jobe/index.php/restapi/runs/'
+        headers = {"Content-type": "application/json; charset=utf-8",
+                   "Accept": "application/json"}
+
+        for expected in self['correct']:
+            data_json = json.dumps({
+                'run_spec' : {
+                    'language_id': self['language'],
+                    'sourcecode': self['answer'],
+                    'input': expected.get('stdin', ''),
+                     },
+                })
+
+            try:
+                connect = http.client.HTTPConnection(self['server'])
+                connect.request(
+                    method='POST',
+                    url=resource,
+                    body=data_json,
+                    headers=headers
+                    )
+                response = connect.getresponse()
+                logger.debug('JOBE response status %d', response.status)
+                if response.status != 204:
+                    content = response.read().decode('utf8')
+                    if content:
+                        result = json.loads(content)
+                connect.close()
+
+            except (HTTPError, ValueError):
+                logger.error('HTTPError while connecting to JOBE server')
+
+            try:
+                outcome = result['outcome']
+            except (NameError, TypeError, KeyError):
+                logger.error('Bad result returned from JOBE server: %s', result)
+                return
+            logger.debug(self._outcomes[outcome])
+
+            if result['cmpinfo']:  # compiler errors and warnings
+                self['comments'] = f'Erros de compilação:\n{result["cmpinfo"]}'
+                self['grade'] = 0.0
+                return
+
+            if result['stdout'] != expected['stdout']:
+                self['comments'] = 'O output gerado é diferente do esperado.'
+                self['grade'] = 0.0
+                return
+
+        self['comments'] = 'Ok!'
+        self['grade'] = 1.0
+
+
+    # ------------------------------------------------------------------------
+    async def correct_async(self) -> None:
+        self.correct()
+
+
+            # out = run_script(
+            #     script=self['correct'],
+            #     args=self['args'],
+            #     stdin=self['answer'],
+            #     timeout=self['timeout']
+            #     )
+
+            # if out is None:
+            #     logger.warning('No grade after running "%s".', self["correct"])
+            #     self['comments'] = 'O programa de correcção abortou...'
+            #     self['grade'] = 0.0
+            # elif isinstance(out, dict):
+            #     self['comments'] = out.get('comments', '')
+            #     try:
+            #         self['grade'] = float(out['grade'])
+            #     except ValueError:
+            #         logger.error('Output error in "%s".', self["correct"])
+            #     except KeyError:
+            #         logger.error('No grade in "%s".', self["correct"])
+            # else:
+            #     try:
+            #         self['grade'] = float(out)
+            #     except (TypeError, ValueError):
+            #         logger.error('Invalid grade in "%s".', self["correct"])
+
+# ============================================================================
 class QuestionInformation(Question):
     '''
     Not really a question, just an information panel.
@@ -645,6 +775,7 @@ class QFactory():
         'text-regex':       QuestionTextRegex,
         'numeric-interval': QuestionNumericInterval,
         'textarea':         QuestionTextArea,
+        'code':             QuestionCode,
         # -- informative panels --
         'information':      QuestionInformation,
         'success':          QuestionInformation,
@@ -227,6 +227,7 @@ class RootHandler(BaseHandler):
         'text-regex':   'question-text.html',
         'numeric-interval': 'question-text.html',
         'textarea':     'question-textarea.html',
+        'code':         'question-textarea.html',
         # -- information panels --
         'information':  'question-information.html',
         'success':      'question-information.html',
@@ -289,7 +290,7 @@ class RootHandler(BaseHandler):
                     else:
                         ans[i] = ans[i][0]
                 elif question['type'] in ('text', 'text-regex', 'textarea',
-                                          'numeric-interval'):
+                                          'numeric-interval', 'code'):
                     ans[i] = ans[i][0]
         # correct answered questions and logout
@@ -477,6 +478,7 @@ class ReviewHandler(BaseHandler):
         'text-regex':   'review-question-text.html',
         'numeric-interval': 'review-question-text.html',
         'textarea':     'review-question-text.html',
+        'code':         'review-question-text.html',
         # -- information panels --
         'information':  'review-question-information.html',
         'success':      'review-question-information.html',
@@ -47,7 +47,7 @@
             pontos
           </p>
           <p class="text-warning">{{ q['comments'] }}</p>
-          {% if q.get('solution', '') %}
+          {% if q['solution'] %}
             <hr>
             {{ md('**Solução:** \n\n' + q['solution']) }}
           {% end %}
@@ -57,8 +57,8 @@
             {{ round(q['grade'] * q['points'], 2) }}
             pontos
           </p>
-          <p class="text-danger">{{ q['comments'] }}</p>
-          {% if q.get('solution', '') %}
+          <p class="text-danger"><pre>{{ q['comments'] }}</pre></p>
+          {% if q['solution'] %}
             <hr>
             {{ md('**Solução:** \n\n' + q['solution']) }}
           {% end %}
@@ -102,7 +102,7 @@
           <i class="fas fa-ban fa-3x" aria-hidden="true"></i>
           {{ round(q['grade'] * q['points'], 2) }} pontos<br>
           {{ q['comments'] }}
-          {% if q.get('solution', '') %}
+          {% if q['solution'] %}
             <hr>
             {{ md('**Solução:** \n\n' + q['solution']) }}
           {% end %}
	@@ -609,3 +609,49 @@		@@ -609,3 +609,49 @@
609	generate-question \| yamllint -	609	generate-question \| yamllint -
610	correct-answer \| yamllint -	610	correct-answer \| yamllint -
611	```	611	```
		612	+
		613	+# ----------------------------------------------------------------------------
		614	+- type: code
		615	+ ref: tut-code
		616	+ title: Submissão de código (JOBE)
		617	+ text: \|
		618	+ É possível enviar código para ser compilado e executado por um servidor
		619	+ JOBE instalado separadamente, ver [JOBE](https://github.com/trampgeek/jobe).
		620	+
		621	+ ```yaml
		622	+ - type: code
		623	+ ref: tut-code
		624	+ title: Submissão de código (JOBE)
		625	+ text: \|
		626	+ Escreva um programa em C que recebe uma string no standard input e
		627	+ mostra a mensagem `hello ` seguida da string.
		628	+ Por exemplo, se o input for `Maria`, o output deverá ser `hello Maria`.
		629	+ server: 127.0.0.1 # replace by appropriate address
		630	+ language: c
		631	+ correct:
		632	+ - stdin: 'Maria'
		633	+ stdout: 'hello Maria'
		634	+ - stdin: 'xyz'
		635	+ stdout: 'hello xyz'
		636	+ ```
		637	+
		638	+ Existem várias linguagens suportadas pelo servidor JOBE (C, C++, Java,
		639	+ Python2, Python3, Octave, Pascal, PHP).
		640	+ O campo `correct` deverá ser uma lista de casos a testar.
		641	+ Se um caso incluir `stdin`, este será enviado para o programa e o `stdout`
		642	+ obtido será comparado com o declarado. A pergunta é considerada correcta se
		643	+ todos os outputs coincidirem.
		644	+ answer: \|
		645	+ #include <stdio.h>
		646	+ int main() {
		647	+ char name[20];
		648	+ scanf("%s", name);
		649	+ printf("hello %s", name);
		650	+ }
		651	+ server: 192.168.1.141
		652	+ language: c
		653	+ correct:
		654	+ - stdin: 'Maria'
		655	+ stdout: 'hello Maria'
		656	+ - stdin: 'xyz'
		657	+ stdout: 'hello xyz'
	@@ -32,7 +32,7 @@ proof of submission and for review.		@@ -32,7 +32,7 @@ proof of submission and for review.
32	'''	32	'''
33		33
34	APP_NAME = 'perguntations'	34	APP_NAME = 'perguntations'
35	-APP_VERSION = '2020.11.dev2'	35	+APP_VERSION = '2020.11.dev3'
36	APP_DESCRIPTION = __doc__	36	APP_DESCRIPTION = __doc__
37		37
38	__author__ = 'Miguel Barão'	38	__author__ = 'Miguel Barão'
	@@ -190,7 +190,7 @@ class App():		@@ -190,7 +190,7 @@ class App():
190	raise AppException('Failed to create test factory!') from exc	190	raise AppException('Failed to create test factory!') from exc
191		191
192	# ------------------------------------------------------------------------	192	# ------------------------------------------------------------------------
193	- def _pregenerate_tests(self, num):	193	+ def _pregenerate_tests(self, num): # TODO needs improvement
194	event_loop = asyncio.get_event_loop()	194	event_loop = asyncio.get_event_loop()
195	self.pregenerated_tests += [	195	self.pregenerated_tests += [
196	event_loop.run_until_complete(self.testfactory.generate())	196	event_loop.run_until_complete(self.testfactory.generate())
	@@ -13,6 +13,12 @@ import re		@@ -13,6 +13,12 @@ import re
13	from typing import Any, Dict, NewType	13	from typing import Any, Dict, NewType
14	import uuid	14	import uuid
15		15
		16	+
		17	+from urllib.error import HTTPError
		18	+import json
		19	+import http.client
		20	+
		21	+
16	# this project	22	# this project
17	from perguntations.tools import run_script, run_script_async	23	from perguntations.tools import run_script, run_script_async
18		24
	@@ -584,6 +590,130 @@ class QuestionTextArea(Question):		@@ -584,6 +590,130 @@ class QuestionTextArea(Question):
584		590
585		591
586	# ============================================================================	592	# ============================================================================
		593	+class QuestionCode(Question):
		594	+ '''An instance of QuestionCode will always have the keys:
		595	+ type (str)
		596	+ text (str)
		597	+ correct (str with script to run)
		598	+ answer (None or an actual answer)
		599	+ '''
		600	+
		601	+ _outcomes = {
		602	+ 0: 'JOBE outcome: Successful run',
		603	+ 11: 'JOBE outcome: Compile error',
		604	+ 12: 'JOBE outcome: Runtime error',
		605	+ 13: 'JOBE outcome: Time limit exceeded',
		606	+ 15: 'JOBE outcome: Successful run',
		607	+ 17: 'JOBE outcome: Memory limit exceeded',
		608	+ 19: 'JOBE outcome: Illegal system call',
		609	+ 20: 'JOBE outcome: Internal error, please report',
		610	+ 21: 'JOBE outcome: Server overload',
		611	+ }
		612	+
		613	+ # ------------------------------------------------------------------------
		614	+ def __init__(self, q: QDict) -> None:
		615	+ super().__init__(q)
		616	+
		617	+ self.set_defaults(QDict({
		618	+ 'text': '',
		619	+ 'timeout': 5, # seconds
		620	+ 'server': '127.0.0.1', # JOBE server
		621	+ 'language': 'c',
		622	+ 'correct': [{'stdin': '', 'stdout': '', 'stderr': '', 'args': ''}],
		623	+ }))
		624	+
		625	+ # ------------------------------------------------------------------------
		626	+ def correct(self) -> None:
		627	+ super().correct()
		628	+
		629	+ if self['answer'] is None:
		630	+ return
		631	+
		632	+ # submit answer to JOBE server
		633	+ resource = '/jobe/index.php/restapi/runs/'
		634	+ headers = {"Content-type": "application/json; charset=utf-8",
		635	+ "Accept": "application/json"}
		636	+
		637	+ for expected in self['correct']:
		638	+ data_json = json.dumps({
		639	+ 'run_spec' : {
		640	+ 'language_id': self['language'],
		641	+ 'sourcecode': self['answer'],
		642	+ 'input': expected.get('stdin', ''),
		643	+ },
		644	+ })
		645	+
		646	+ try:
		647	+ connect = http.client.HTTPConnection(self['server'])
		648	+ connect.request(
		649	+ method='POST',
		650	+ url=resource,
		651	+ body=data_json,
		652	+ headers=headers
		653	+ )
		654	+ response = connect.getresponse()
		655	+ logger.debug('JOBE response status %d', response.status)
		656	+ if response.status != 204:
		657	+ content = response.read().decode('utf8')
		658	+ if content:
		659	+ result = json.loads(content)
		660	+ connect.close()
		661	+
		662	+ except (HTTPError, ValueError):
		663	+ logger.error('HTTPError while connecting to JOBE server')
		664	+
		665	+ try:
		666	+ outcome = result['outcome']
		667	+ except (NameError, TypeError, KeyError):
		668	+ logger.error('Bad result returned from JOBE server: %s', result)
		669	+ return
		670	+ logger.debug(self._outcomes[outcome])
		671	+
		672	+ if result['cmpinfo']: # compiler errors and warnings
		673	+ self['comments'] = f'Erros de compilação:\n{result["cmpinfo"]}'
		674	+ self['grade'] = 0.0
		675	+ return
		676	+
		677	+ if result['stdout'] != expected['stdout']:
		678	+ self['comments'] = 'O output gerado é diferente do esperado.'
		679	+ self['grade'] = 0.0
		680	+ return
		681	+
		682	+ self['comments'] = 'Ok!'
		683	+ self['grade'] = 1.0
		684	+
		685	+
		686	+ # ------------------------------------------------------------------------
		687	+ async def correct_async(self) -> None:
		688	+ self.correct()
		689	+
		690	+
		691	+ # out = run_script(
		692	+ # script=self['correct'],
		693	+ # args=self['args'],
		694	+ # stdin=self['answer'],
		695	+ # timeout=self['timeout']
		696	+ # )
		697	+
		698	+ # if out is None:
		699	+ # logger.warning('No grade after running "%s".', self["correct"])
		700	+ # self['comments'] = 'O programa de correcção abortou...'
		701	+ # self['grade'] = 0.0
		702	+ # elif isinstance(out, dict):
		703	+ # self['comments'] = out.get('comments', '')
		704	+ # try:
		705	+ # self['grade'] = float(out['grade'])
		706	+ # except ValueError:
		707	+ # logger.error('Output error in "%s".', self["correct"])
		708	+ # except KeyError:
		709	+ # logger.error('No grade in "%s".', self["correct"])
		710	+ # else:
		711	+ # try:
		712	+ # self['grade'] = float(out)
		713	+ # except (TypeError, ValueError):
		714	+ # logger.error('Invalid grade in "%s".', self["correct"])
		715	+
		716	+# ============================================================================
587	class QuestionInformation(Question):	717	class QuestionInformation(Question):
588	'''	718	'''
589	Not really a question, just an information panel.	719	Not really a question, just an information panel.
	@@ -645,6 +775,7 @@ class QFactory():		@@ -645,6 +775,7 @@ class QFactory():
645	'text-regex': QuestionTextRegex,	775	'text-regex': QuestionTextRegex,
646	'numeric-interval': QuestionNumericInterval,	776	'numeric-interval': QuestionNumericInterval,
647	'textarea': QuestionTextArea,	777	'textarea': QuestionTextArea,
		778	+ 'code': QuestionCode,
648	# -- informative panels --	779	# -- informative panels --
649	'information': QuestionInformation,	780	'information': QuestionInformation,
650	'success': QuestionInformation,	781	'success': QuestionInformation,