- changes how database is updated after correction of a test

- fix get_questions_csv when there are questions with duplicate refs - modified database model to include question number and comment in the questions table. Also removed student_id so that it is now normalized.

- changes how database is updated after correction of a test
- fix get_questions_csv when there are questions with duplicate refs - modified database model to include question number and comment in the questions table. Also removed student_id so that it is now normalized.
Miguel Barão
1 parent aab6a036
Showing 4 changed files with 107 additions and 99 deletions Show diff stats
BUGS.md
perguntations/app.py
perguntations/initdb.py
perguntations/models.py
 # BUGS
-- perguntas repetidas (mesma ref) dao asneira, porque a referencia é usada como chave em varios sitios e as chaves nao podem ser dupplicadas.
-  da asneira pelo menos na funcao get_questions_csv. na base de dados tem de estar registado tb o numero da pergunta, caso contrario é impossível saber a qual corresponde.
+- esta a corrigir código JOBE mesmo que nao tenha respondido???
 - show-ref nao esta a funcionar na correccao (pelo menos)
-- algumas submissões ficaram em duplicado na base de dados. a base de dados deveria ter como chave do teste um id que fosse único desse teste particular (não um auto counter, nem ref do teste)
-- guardar nota final grade truncado em zero e sem ser truncado (quando é necessário fazer correcções à mão às perguntas, é necessário o valor não truncado)
-- internal server error quando em --review, download csv detalhado.
+- algumas vezes a base de dados guarda o mesmo teste em duplicado. ver se dois submits dao origem a duas correcções.
+talvez a base de dados devesse ter como chave do teste um id que fosse único desse teste particular (não um auto counter, nem ref do teste)
 - JOBE correct async
 - em caso de timeout na submissão (e.g. JOBE ou script nao responde) a correcção não termina e o teste não é guardado.
 - QuestionCode falta reportar nos comments os vários erros que podem ocorrer (timeout, etc)
-- permitir remover alunos que estão online para poderem comecar de novo.
 - grade gives internal server error??
 - reload do teste recomeça a contagem no inicio do tempo.
 - em admin, quando scale_max não é 20, as cores das barras continuam a reflectir a escala 0,20. a tabela teste na DB não tem a escala desse teste.
@@ -24,6 +21,8 @@
 # TODO
+- permitir remover alunos que estão online para poderem comecar de novo.
+- guardar nota final grade truncado em zero e sem ser truncado (quando é necessário fazer correcções à mão às perguntas, é necessário o valor não truncado)
 - stress tests. use https://locust.io
 - wait for admin to start test. (students can be allowed earlier)
 - impedir os eventos copy/paste. alunos usam isso para trazer codigo ja feito nos computadores. Obrigar a fazer reset? fazer um copy automaticamente?
@@ -72,6 +71,9 @@ ou usar push (websockets?)
 # FIXED
+- internal server error quando em --review, download csv detalhado.
+- perguntas repetidas (mesma ref) dao asneira, porque a referencia é usada como chave em varios sitios e as chaves nao podem ser dupplicadas.
+  da asneira pelo menos na funcao get_questions_csv. na base de dados tem de estar registado tb o numero da pergunta, caso contrario é impossível saber a qual corresponde.
 - mostrar unfocus e window area em /admin
 - CRITICAL se answer for `i<n` a revisão de provas mostra apenas i (interpreta `<` como tag?)
 - botao de autorizar desliga-se, fazer debounce.
@@ -258,61 +258,66 @@ class App():
         fname = '--'.join(fields) + '.json'
         fpath = path.join(test['answers_dir'], fname)
         with open(path.expanduser(fpath), 'w') as file:
-            # default=str required for datetime objects
             json.dump(test, file, indent=2, default=str)
+            # option default=str is required for datetime objects
+
         logger.info('"%s" saved JSON.', uid)
         # --- insert test and questions into database
+        test_row = Test(
+            ref=test['ref'],
+            title=test['title'],
+            grade=test['grade'],
+            state=test['state'],
+            comment='',
+            starttime=str(test['start_time']),
+            finishtime=str(test['finish_time']),
+            filename=fpath,
+            student_id=uid)
+        test_row.questions = [Question(
+            number=n,
+            ref=q['ref'],
+            grade=q['grade'],
+            comment=q.get('comment', ''),
+            starttime=str(test['start_time']),
+            finishtime=str(test['finish_time']),
+            test_id=test['ref'])
+                for n, q in enumerate(test['questions'])
+                if 'grade' in q
+            ]
         with self._db_session() as sess:
-            sess.add(Test(
-                ref=test['ref'],
-                title=test['title'],
-                grade=test['grade'],
-                starttime=str(test['start_time']),
-                finishtime=str(test['finish_time']),
-                filename=fpath,
-                student_id=uid,
-                state=test['state'],
-                comment=''))
-            sess.add_all([Question(
-                ref=q['ref'],
-                grade=q['grade'],
-                starttime=str(test['start_time']),
-                finishtime=str(test['finish_time']),
-                student_id=uid,
-                test_id=test['ref'])
-                          for q in test['questions'] if 'grade' in q])
+            sess.add(test_row)
         logger.info('"%s" database updated.', uid)
         return grade
     # ------------------------------------------------------------------------
-    def giveup_test(self, uid):
-        '''giveup test - not used??'''
-        test = self.online[uid]['test']
-        test.giveup()
-
-        # save JSON with the test
-        fields = (test['student']['number'], test['ref'],
-                  str(test['finish_time']))
-        fname = '--'.join(fields) + '.json'
-        fpath = path.join(test['answers_dir'], fname)
-        test.save_json(fpath)
-
-        # insert test into database
-        with self._db_session() as sess:
-            sess.add(Test(ref=test['ref'],
-                          title=test['title'],
-                          grade=test['grade'],
-                          starttime=str(test['start_time']),
-                          finishtime=str(test['finish_time']),
-                          filename=fpath,
-                          student_id=test['student']['number'],
-                          state=test['state'],
-                          comment=''))
-
-        logger.info('"%s" gave up.', uid)
-        return test
+    # def giveup_test(self, uid):
+    #     '''giveup test - not used??'''
+    #     test = self.online[uid]['test']
+    #     test.giveup()
+
+    #     # save JSON with the test
+    #     fields = (test['student']['number'], test['ref'],
+    #               str(test['finish_time']))
+    #     fname = '--'.join(fields) + '.json'
+    #     fpath = path.join(test['answers_dir'], fname)
+    #     test.save_json(fpath)
+
+    #     # insert test into database
+    #     with self._db_session() as sess:
+    #         sess.add(Test(ref=test['ref'],
+    #                       title=test['title'],
+    #                       grade=test['grade'],
+    #                       starttime=str(test['start_time']),
+    #                       finishtime=str(test['finish_time']),
+    #                       filename=fpath,
+    #                       # student_id=test['student']['number'],
+    #                       state=test['state'],
+    #                       comment=''))
+
+    #     logger.info('"%s" gave up.', uid)
+    #     return test
     # ------------------------------------------------------------------------
     def event_test(self, uid, cmd, value):
@@ -334,55 +339,56 @@ class App():
     def get_questions_csv(self):
         '''generates a CSV with the grades of the test'''
-        test_id = self.testfactory['ref']
-
+        test_ref = self.testfactory['ref']
         with self._db_session() as sess:
-            grades = sess.query(Question.student_id, Question.starttime,
-                                Question.ref, Question.grade)\
-                         .filter(Question.test_id == test_id)\
-                         .order_by(Question.student_id)\
-                         .all()
-
-        # first row of the csv will include student_id, date/time, and refs for
-        # all questions declared in the test
-        cols = ['Aluno', 'Início'] + \
-               [r for question in self.testfactory['questions']
-                for r in question['ref']]
-
-        # tests is a dict of dicts:  FIXME what about duplicate qrefs???
-        # { (student_id, time): {qref: grade, ...} }
-        # the key is a tuple (student_id, time) because the same student can
-        # repeat the same test multiple times (if the professor allows).
-        tests = {}
-        for question in grades:
-            student, qref, qgrade = question[:2], *question[2:]
-            tests.setdefault(student, {})[qref] = qgrade
-
-        rows = [{'Aluno': test[0], 'Início': test[1], **q}
-                for test, q in tests.items()]
+            questions = sess.query(Test.id, Test.student_id, Test.starttime,
+                                   Question.number, Question.grade)\
+                            .join(Question)\
+                            .filter(Test.ref == test_ref)\
+                            .all()
+
+            qnums = set()  # keeps track of all the questions in the test
+            tests = {}      # {test_id: {student_id, starttime, 0: grade, ...}}
+            for question in questions:
+                test_id, student_id, starttime, num, grade = question
+                default_test_id = {'Aluno': student_id, 'Início': starttime}
+                tests.setdefault(test_id, default_test_id)[num] = grade
+                qnums.add(num)
+
+        if not tests:
+            logger.warning('Empty CSV: there are no tests!')
+            return test_ref, ''
+
+        cols = ['Aluno', 'Início'] + list(qnums)
         csvstr = io.StringIO()
         writer = csv.DictWriter(csvstr, fieldnames=cols, restval=None,
                                 delimiter=';', quoting=csv.QUOTE_ALL)
         writer.writeheader()
-        writer.writerows(rows)
-        return test_id, csvstr.getvalue()
-
+        writer.writerows(tests.values())
+        return test_ref, csvstr.getvalue()
     def get_test_csv(self):
-        '''generates a CSV with the grades of the test'''
+        '''generates a CSV with the grades of the test currently running'''
+        test_ref = self.testfactory['ref']
         with self._db_session() as sess:
-            grades = sess.query(Test.student_id, Test.grade,
+            tests = sess.query(Test.student_id,
+                                Test.grade,
                                 Test.starttime, Test.finishtime)\
-                         .filter(Test.ref == self.testfactory['ref'])\
+                         .filter(Test.ref == test_ref)\
                          .order_by(Test.student_id)\
                          .all()
+        if not tests:
+            logger.warning('Empty CSV: there are no tests!')
+            return test_ref, ''
+
         csvstr = io.StringIO()
         writer = csv.writer(csvstr, delimiter=';', quoting=csv.QUOTE_ALL)
         writer.writerow(('Aluno', 'Nota', 'Início', 'Fim'))
-        writer.writerows(grades)
-        return self.testfactory['ref'], csvstr.getvalue()
+        writer.writerows(tests)
+
+        return test_ref, csvstr.getvalue()
     # ------------------------------------------------------------------------
     def get_student_grades_from_all_tests(self, uid):
@@ -19,7 +19,7 @@ import sqlalchemy as sa
 from perguntations.models import Base, Student
-# ===========================================================================
+# ============================================================================
 def parse_commandline_arguments():
     '''Parse command line options'''
     parser = argparse.ArgumentParser(
@@ -68,7 +68,7 @@ def parse_commandline_arguments():
     return parser.parse_args()
-# ===========================================================================
+# ============================================================================
 def get_students_from_csv(filename):
     '''
     SIIUE names have alien strings like "(TE)" and are sometimes capitalized
@@ -97,7 +97,7 @@ def get_students_from_csv(filename):
     return students
-# ===========================================================================
+# ============================================================================
 def hashpw(student, password=None):
     '''replace password by hash for a single student'''
     print('.', end='', flush=True)
@@ -108,7 +108,7 @@ def hashpw(student, password=None):
                                       bcrypt.gensalt())
-# ===========================================================================
+# ============================================================================
 def insert_students_into_db(session, students):
     '''insert list of students into the database'''
     try:
@@ -25,9 +25,8 @@ class Student(Base):
     # ---
     tests = relationship('Test', back_populates='student')
-    questions = relationship('Question', back_populates='student')
-    def __repr__(self):
+    def __str__(self):
         return (f'Student:\n'
                 f'  id:         "{self.id}"\n'
                 f'  name:       "{self.name}"\n'
@@ -53,12 +52,12 @@ class Test(Base):
     student = relationship('Student', back_populates='tests')
     questions = relationship('Question', back_populates='test')
-    def __repr__(self):
+    def __str__(self):
         return (f'Test:\n'
-                f'  id:         "{self.id}"\n'
+                f'  id:         {self.id}\n'
                 f'  ref:        "{self.ref}"\n'
                 f'  title:      "{self.title}"\n'
-                f'  grade:      "{self.grade}"\n'
+                f'  grade:      {self.grade}\n'
                 f'  state:      "{self.state}"\n'
                 f'  comment:    "{self.comment}"\n'
                 f'  starttime:  "{self.starttime}"\n'
@@ -72,23 +71,24 @@ class Question(Base):
     '''Question table'''
     __tablename__ = 'questions'
     id = Column(Integer, primary_key=True)  # auto_increment
+    number = Column(Integer)    # question number (ref may be not be unique)
     ref = Column(String)
     grade = Column(Float)
+    comment = Column(String)
     starttime = Column(String)
     finishtime = Column(String)
-    student_id = Column(String, ForeignKey('students.id'))
     test_id = Column(String, ForeignKey('tests.id'))
     # ---
-    student = relationship('Student', back_populates='questions')
     test = relationship('Test', back_populates='questions')
-    def __repr__(self):
+    def __str__(self):
         return (f'Question:\n'
-                f'  id:         "{self.id}"\n'
+                f'  id:         {self.id}\n'
+                f'  number:     {self.number}\n'
                 f'  ref:        "{self.ref}"\n'
-                f'  grade:      "{self.grade}"\n'
+                f'  grade:      {self.grade}\n'
+                f'  comment:    "{self.comment}"\n'
                 f'  starttime:  "{self.starttime}"\n'
                 f'  finishtime: "{self.finishtime}"\n'
-                f'  student_id: "{self.student_id}"\n'  # FIXME normal form
                 f'  test_id:    "{self.test_id}"\n')
	@@ -19,7 +19,7 @@ import sqlalchemy as sa		@@ -19,7 +19,7 @@ import sqlalchemy as sa
19	from perguntations.models import Base, Student	19	from perguntations.models import Base, Student
20		20
21		21
22	-# ===========================================================================	22	+# ============================================================================
23	def parse_commandline_arguments():	23	def parse_commandline_arguments():
24	'''Parse command line options'''	24	'''Parse command line options'''
25	parser = argparse.ArgumentParser(	25	parser = argparse.ArgumentParser(
	@@ -68,7 +68,7 @@ def parse_commandline_arguments():		@@ -68,7 +68,7 @@ def parse_commandline_arguments():
68	return parser.parse_args()	68	return parser.parse_args()
69		69
70		70
71	-# ===========================================================================	71	+# ============================================================================
72	def get_students_from_csv(filename):	72	def get_students_from_csv(filename):
73	'''	73	'''
74	SIIUE names have alien strings like "(TE)" and are sometimes capitalized	74	SIIUE names have alien strings like "(TE)" and are sometimes capitalized
	@@ -97,7 +97,7 @@ def get_students_from_csv(filename):		@@ -97,7 +97,7 @@ def get_students_from_csv(filename):
97	return students	97	return students
98		98
99		99
100	-# ===========================================================================	100	+# ============================================================================
101	def hashpw(student, password=None):	101	def hashpw(student, password=None):
102	'''replace password by hash for a single student'''	102	'''replace password by hash for a single student'''
103	print('.', end='', flush=True)	103	print('.', end='', flush=True)
	@@ -108,7 +108,7 @@ def hashpw(student, password=None):		@@ -108,7 +108,7 @@ def hashpw(student, password=None):
108	bcrypt.gensalt())	108	bcrypt.gensalt())
109		109
110		110
111	-# ===========================================================================	111	+# ============================================================================
112	def insert_students_into_db(session, students):	112	def insert_students_into_db(session, students):
113	'''insert list of students into the database'''	113	'''insert list of students into the database'''
114	try:	114	try: