From bfedb0e29e181a6dfece6a7cc0bd52fa9535da30 Mon Sep 17 00:00:00 2001 From: Miguel Barão Date: Tue, 17 Nov 2020 16:21:48 +0000 Subject: [PATCH] initial support for submitting code to a JOBE server --- demo/demo.yaml | 2 ++ demo/questions/questions-tutorial.yaml | 46 ++++++++++++++++++++++++++++++++++++++++++++++ perguntations/__init__.py | 2 +- perguntations/app.py | 2 +- perguntations/questions.py | 131 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ perguntations/serve.py | 4 +++- perguntations/templates/review-question.html | 8 ++++---- 7 files changed, 188 insertions(+), 7 deletions(-) diff --git a/demo/demo.yaml b/demo/demo.yaml index 2391207..0bfe4ef 100644 --- a/demo/demo.yaml +++ b/demo/demo.yaml @@ -70,6 +70,8 @@ questions: - [tut-alert1, tut-alert2] - tut-generator - tut-yamllint + - tut-code + # test: # - ref1 diff --git a/demo/questions/questions-tutorial.yaml b/demo/questions/questions-tutorial.yaml index 876c1e5..fbfe559 100644 --- a/demo/questions/questions-tutorial.yaml +++ b/demo/questions/questions-tutorial.yaml @@ -609,3 +609,49 @@ generate-question | yamllint - correct-answer | yamllint - ``` + +# ---------------------------------------------------------------------------- +- type: code + ref: tut-code + title: Submissão de código (JOBE) + text: | + É possível enviar código para ser compilado e executado por um servidor + JOBE instalado separadamente, ver [JOBE](https://github.com/trampgeek/jobe). + + ```yaml + - type: code + ref: tut-code + title: Submissão de código (JOBE) + text: | + Escreva um programa em C que recebe uma string no standard input e + mostra a mensagem `hello ` seguida da string. + Por exemplo, se o input for `Maria`, o output deverá ser `hello Maria`. + server: 127.0.0.1 # replace by appropriate address + language: c + correct: + - stdin: 'Maria' + stdout: 'hello Maria' + - stdin: 'xyz' + stdout: 'hello xyz' + ``` + + Existem várias linguagens suportadas pelo servidor JOBE (C, C++, Java, + Python2, Python3, Octave, Pascal, PHP). + O campo `correct` deverá ser uma lista de casos a testar. + Se um caso incluir `stdin`, este será enviado para o programa e o `stdout` + obtido será comparado com o declarado. A pergunta é considerada correcta se + todos os outputs coincidirem. + answer: | + #include + int main() { + char name[20]; + scanf("%s", name); + printf("hello %s", name); + } + server: 192.168.1.141 + language: c + correct: + - stdin: 'Maria' + stdout: 'hello Maria' + - stdin: 'xyz' + stdout: 'hello xyz' diff --git a/perguntations/__init__.py b/perguntations/__init__.py index 11f2850..97c5244 100644 --- a/perguntations/__init__.py +++ b/perguntations/__init__.py @@ -32,7 +32,7 @@ proof of submission and for review. ''' APP_NAME = 'perguntations' -APP_VERSION = '2020.11.dev2' +APP_VERSION = '2020.11.dev3' APP_DESCRIPTION = __doc__ __author__ = 'Miguel Barão' diff --git a/perguntations/app.py b/perguntations/app.py index f020652..a66d33b 100644 --- a/perguntations/app.py +++ b/perguntations/app.py @@ -190,7 +190,7 @@ class App(): raise AppException('Failed to create test factory!') from exc # ------------------------------------------------------------------------ - def _pregenerate_tests(self, num): + def _pregenerate_tests(self, num): # TODO needs improvement event_loop = asyncio.get_event_loop() self.pregenerated_tests += [ event_loop.run_until_complete(self.testfactory.generate()) diff --git a/perguntations/questions.py b/perguntations/questions.py index e193cc0..6338f35 100644 --- a/perguntations/questions.py +++ b/perguntations/questions.py @@ -13,6 +13,12 @@ import re from typing import Any, Dict, NewType import uuid + +from urllib.error import HTTPError +import json +import http.client + + # this project from perguntations.tools import run_script, run_script_async @@ -584,6 +590,130 @@ class QuestionTextArea(Question): # ============================================================================ +class QuestionCode(Question): + '''An instance of QuestionCode will always have the keys: + type (str) + text (str) + correct (str with script to run) + answer (None or an actual answer) + ''' + + _outcomes = { + 0: 'JOBE outcome: Successful run', + 11: 'JOBE outcome: Compile error', + 12: 'JOBE outcome: Runtime error', + 13: 'JOBE outcome: Time limit exceeded', + 15: 'JOBE outcome: Successful run', + 17: 'JOBE outcome: Memory limit exceeded', + 19: 'JOBE outcome: Illegal system call', + 20: 'JOBE outcome: Internal error, please report', + 21: 'JOBE outcome: Server overload', + } + + # ------------------------------------------------------------------------ + def __init__(self, q: QDict) -> None: + super().__init__(q) + + self.set_defaults(QDict({ + 'text': '', + 'timeout': 5, # seconds + 'server': '127.0.0.1', # JOBE server + 'language': 'c', + 'correct': [{'stdin': '', 'stdout': '', 'stderr': '', 'args': ''}], + })) + + # ------------------------------------------------------------------------ + def correct(self) -> None: + super().correct() + + if self['answer'] is None: + return + + # submit answer to JOBE server + resource = '/jobe/index.php/restapi/runs/' + headers = {"Content-type": "application/json; charset=utf-8", + "Accept": "application/json"} + + for expected in self['correct']: + data_json = json.dumps({ + 'run_spec' : { + 'language_id': self['language'], + 'sourcecode': self['answer'], + 'input': expected.get('stdin', ''), + }, + }) + + try: + connect = http.client.HTTPConnection(self['server']) + connect.request( + method='POST', + url=resource, + body=data_json, + headers=headers + ) + response = connect.getresponse() + logger.debug('JOBE response status %d', response.status) + if response.status != 204: + content = response.read().decode('utf8') + if content: + result = json.loads(content) + connect.close() + + except (HTTPError, ValueError): + logger.error('HTTPError while connecting to JOBE server') + + try: + outcome = result['outcome'] + except (NameError, TypeError, KeyError): + logger.error('Bad result returned from JOBE server: %s', result) + return + logger.debug(self._outcomes[outcome]) + + if result['cmpinfo']: # compiler errors and warnings + self['comments'] = f'Erros de compilação:\n{result["cmpinfo"]}' + self['grade'] = 0.0 + return + + if result['stdout'] != expected['stdout']: + self['comments'] = 'O output gerado é diferente do esperado.' + self['grade'] = 0.0 + return + + self['comments'] = 'Ok!' + self['grade'] = 1.0 + + + # ------------------------------------------------------------------------ + async def correct_async(self) -> None: + self.correct() + + + # out = run_script( + # script=self['correct'], + # args=self['args'], + # stdin=self['answer'], + # timeout=self['timeout'] + # ) + + # if out is None: + # logger.warning('No grade after running "%s".', self["correct"]) + # self['comments'] = 'O programa de correcção abortou...' + # self['grade'] = 0.0 + # elif isinstance(out, dict): + # self['comments'] = out.get('comments', '') + # try: + # self['grade'] = float(out['grade']) + # except ValueError: + # logger.error('Output error in "%s".', self["correct"]) + # except KeyError: + # logger.error('No grade in "%s".', self["correct"]) + # else: + # try: + # self['grade'] = float(out) + # except (TypeError, ValueError): + # logger.error('Invalid grade in "%s".', self["correct"]) + +# ============================================================================ class QuestionInformation(Question): ''' Not really a question, just an information panel. @@ -645,6 +775,7 @@ class QFactory(): 'text-regex': QuestionTextRegex, 'numeric-interval': QuestionNumericInterval, 'textarea': QuestionTextArea, + 'code': QuestionCode, # -- informative panels -- 'information': QuestionInformation, 'success': QuestionInformation, diff --git a/perguntations/serve.py b/perguntations/serve.py index 1f6e521..3870b9c 100644 --- a/perguntations/serve.py +++ b/perguntations/serve.py @@ -227,6 +227,7 @@ class RootHandler(BaseHandler): 'text-regex': 'question-text.html', 'numeric-interval': 'question-text.html', 'textarea': 'question-textarea.html', + 'code': 'question-textarea.html', # -- information panels -- 'information': 'question-information.html', 'success': 'question-information.html', @@ -289,7 +290,7 @@ class RootHandler(BaseHandler): else: ans[i] = ans[i][0] elif question['type'] in ('text', 'text-regex', 'textarea', - 'numeric-interval'): + 'numeric-interval', 'code'): ans[i] = ans[i][0] # correct answered questions and logout @@ -477,6 +478,7 @@ class ReviewHandler(BaseHandler): 'text-regex': 'review-question-text.html', 'numeric-interval': 'review-question-text.html', 'textarea': 'review-question-text.html', + 'code': 'review-question-text.html', # -- information panels -- 'information': 'review-question-information.html', 'success': 'review-question-information.html', diff --git a/perguntations/templates/review-question.html b/perguntations/templates/review-question.html index 802b3a3..305c068 100644 --- a/perguntations/templates/review-question.html +++ b/perguntations/templates/review-question.html @@ -47,7 +47,7 @@ pontos

{{ q['comments'] }}

- {% if q.get('solution', '') %} + {% if q['solution'] %}
{{ md('**Solução:** \n\n' + q['solution']) }} {% end %} @@ -57,8 +57,8 @@ {{ round(q['grade'] * q['points'], 2) }} pontos

-

{{ q['comments'] }}

- {% if q.get('solution', '') %} + 

{{ q['comments'] }}

+ {% if q['solution'] %}
{{ md('**Solução:** \n\n' + q['solution']) }} {% end %} @@ -102,7 +102,7 @@ {{ round(q['grade'] * q['points'], 2) }} pontos
{{ q['comments'] }} - {% if q.get('solution', '') %} + {% if q['solution'] %}
{{ md('**Solução:** \n\n' + q['solution']) }} {% end %} -- libgit2 0.21.2