Commit 1edf739ffb890e486abf956410c0cfc1bc5c373a
1 parent
a5c2ee14
Exists in
master
and in
1 other branch
- nova versão do initdb, igual à do aprendizations
Showing
2 changed files
with
161 additions
and
67 deletions
Show diff stats
BUGS.md
| 1 | 1 | |
| 2 | 2 | # BUGS |
| 3 | 3 | |
| 4 | +- o eventloop está a bloquear. correção do teste é blocking. usar threadpoolexecutor? | |
| 4 | 5 | - se submissão falhar (aluno desconectado da rede) nao pode sair da página para nao perder o teste. possiveis solucoes: |
| 5 | 6 | - botao submeter valida se esta online com um post willing_to_submit, se estiver online, mostra mensagem de confirmacao, caso contrario avisa que nao esta online. |
| 6 | 7 | - periodicamente, cada user faz um post keep-alive. warning nos logs se offline. | ... | ... |
initdb.py
| 1 | -#!/usr/bin/env python3.6 | |
| 2 | -# -*- coding: utf-8 -*- | |
| 1 | +#!/usr/bin/env python3 | |
| 3 | 2 | |
| 3 | +# base | |
| 4 | 4 | import csv |
| 5 | 5 | import argparse |
| 6 | 6 | import re |
| 7 | 7 | import string |
| 8 | -import sys | |
| 9 | -from sqlalchemy import create_engine | |
| 10 | -from sqlalchemy.orm import sessionmaker | |
| 8 | +from concurrent.futures import ThreadPoolExecutor | |
| 11 | 9 | |
| 10 | +# installed packages | |
| 11 | +import bcrypt | |
| 12 | +import sqlalchemy as sa | |
| 13 | + | |
| 14 | +# this project | |
| 12 | 15 | from models import Base, Student, Test, Question |
| 13 | 16 | |
| 17 | + | |
| 18 | +# =========================================================================== | |
| 19 | +# Parse command line options | |
| 20 | +def parse_commandline_arguments(): | |
| 21 | + argparser = argparse.ArgumentParser( | |
| 22 | + formatter_class=argparse.ArgumentDefaultsHelpFormatter, | |
| 23 | + description='Insert new users into a database. Users can be imported from CSV files in the SIIUE format or defined in the command line. If the database does not exist, a new one is created.') | |
| 24 | + | |
| 25 | + argparser.add_argument('csvfile', | |
| 26 | + nargs='*', | |
| 27 | + type=str, | |
| 28 | + default='', | |
| 29 | + help='CSV file to import (SIIUE)') | |
| 30 | + | |
| 31 | + argparser.add_argument('--db', | |
| 32 | + default='students.db', | |
| 33 | + type=str, | |
| 34 | + help='database file') | |
| 35 | + | |
| 36 | + argparser.add_argument('-A', '--admin', | |
| 37 | + action='store_true', | |
| 38 | + help='insert the admin user') | |
| 39 | + | |
| 40 | + argparser.add_argument('-a', '--add', | |
| 41 | + nargs=2, | |
| 42 | + action='append', | |
| 43 | + metavar=('uid', 'name'), | |
| 44 | + help='add new user') | |
| 45 | + | |
| 46 | + argparser.add_argument('-u', '--update', | |
| 47 | + nargs='+', | |
| 48 | + metavar='uid', | |
| 49 | + default=[], | |
| 50 | + help='users to update') | |
| 51 | + | |
| 52 | + argparser.add_argument('--pw', | |
| 53 | + default=None, | |
| 54 | + type=str, | |
| 55 | + help='set password for new and updated users') | |
| 56 | + | |
| 57 | + argparser.add_argument('-V', '--verbose', | |
| 58 | + action='store_true', | |
| 59 | + help='show all students in database') | |
| 60 | + | |
| 61 | + return argparser.parse_args() | |
| 62 | + | |
| 63 | + | |
| 64 | +# =========================================================================== | |
| 14 | 65 | # SIIUE names have alien strings like "(TE)" and are sometimes capitalized |
| 15 | 66 | # We remove them so that students dont keep asking what it means |
| 16 | -def fix(name): | |
| 17 | - return string.capwords(re.sub('\(.*\)', '', name).strip()) | |
| 67 | +def get_students_from_csv(filename): | |
| 68 | + try: | |
| 69 | + csvreader = csv.DictReader(open(filename, encoding='iso-8859-1'), delimiter=';', quotechar='"', skipinitialspace=True) | |
| 70 | + except EnvironmentError: | |
| 71 | + print(f'!!! Error. File "{filename}" not found !!!') | |
| 72 | + students = [] | |
| 73 | + else: | |
| 74 | + students = [{ | |
| 75 | + 'uid': s['N.º'], | |
| 76 | + 'name': string.capwords(re.sub('\(.*\)', '', s['Nome']).strip()) | |
| 77 | + } for s in csvreader] | |
| 78 | + | |
| 79 | + return students | |
| 80 | + | |
| 18 | 81 | |
| 19 | 82 | # =========================================================================== |
| 20 | -# Parse command line options | |
| 21 | -argparser = argparse.ArgumentParser(description='Create new database from a CSV file (SIIUE format)') | |
| 22 | -argparser.add_argument('--db', default='students.db', type=str, help='database filename') | |
| 23 | -argparser.add_argument('--demo', action='store_true', help='initialize database with a few fake students') | |
| 24 | -argparser.add_argument('csvfile', nargs='?', type=str, default='', help='CSV filename') | |
| 25 | -args = argparser.parse_args() | |
| 83 | +# replace password by hash for a single student | |
| 84 | +def hashpw(student, pw=None): | |
| 85 | + print('.', end='', flush=True) | |
| 86 | + pw = (pw or student.get('pw', None) or student['uid']).encode('utf-8') | |
| 87 | + student['pw'] = bcrypt.hashpw(pw, bcrypt.gensalt()) | |
| 88 | + | |
| 26 | 89 | |
| 27 | 90 | # =========================================================================== |
| 28 | -engine = create_engine('sqlite:///{}'.format(args.db), echo=False) | |
| 91 | +def insert_students_into_db(session, students): | |
| 92 | + try: | |
| 93 | + # --- start db session --- | |
| 94 | + session.add_all([Student(id=s['uid'], name=s['name'], password=s['pw']) | |
| 95 | + for s in students]) | |
| 29 | 96 | |
| 30 | -# Criate schema if needed | |
| 31 | -Base.metadata.create_all(engine) | |
| 97 | + session.commit() | |
| 32 | 98 | |
| 33 | -Session = sessionmaker(bind=engine) | |
| 99 | + except sa.exc.IntegrityError: | |
| 100 | + print('!!! Integrity error. User(s) already in database. None inserted !!!\n') | |
| 101 | + session.rollback() | |
| 34 | 102 | |
| 35 | -# --- start session --- | |
| 36 | -try: | |
| 37 | - session = Session() | |
| 38 | 103 | |
| 39 | - # add administrator | |
| 40 | - session.add(Student(id='0', name='Professor', password='')) | |
| 41 | - | |
| 42 | - # add students | |
| 43 | - if args.csvfile: | |
| 44 | - # from csv file if available | |
| 45 | - try: | |
| 46 | - csvreader = csv.DictReader(open(args.csvfile, encoding='iso-8859-1'), delimiter=';', quotechar='"', skipinitialspace=True) | |
| 47 | - except OSError: | |
| 48 | - print(f'Error: Can\'t open CSV file "{args.csvfile}".') | |
| 49 | - session.rollback() | |
| 50 | - sys.exit(1) | |
| 104 | +# =========================================================================== | |
| 105 | +def show_students_in_database(session, verbose=False): | |
| 106 | + try: | |
| 107 | + users = session.query(Student).order_by(Student.id).all() | |
| 108 | + except: | |
| 109 | + raise | |
| 110 | + else: | |
| 111 | + n = len(users) | |
| 112 | + print(f'Registered users:') | |
| 113 | + if n == 0: | |
| 114 | + print(' -- none --') | |
| 51 | 115 | else: |
| 52 | - session.add_all([Student(id=r['N.º'], name=fix(r['Nome']), password='') for r in csvreader]) | |
| 53 | - elif args.demo: | |
| 54 | - # add a few fake students | |
| 55 | - fakes = [ | |
| 56 | - ['1915', 'Alan Turing'], | |
| 57 | - ['1938', 'Donald Knuth'], | |
| 58 | - ['1815', 'Ada Lovelace'], | |
| 59 | - ['1969', 'Linus Torvalds'], | |
| 60 | - ['1955', 'Tim Burners-Lee'], | |
| 61 | - ['1916', 'Claude Shannon'], | |
| 62 | - ['1903', 'John von Neumann'], | |
| 63 | - ] | |
| 64 | - session.add_all([Student(id=i, name=name, password='') for i,name in fakes]) | |
| 65 | - | |
| 66 | - session.commit() | |
| 67 | - | |
| 68 | -except Exception: | |
| 69 | - print('Error: Database already exists.') | |
| 70 | - session.rollback() | |
| 71 | - sys.exit(1) | |
| 72 | - | |
| 73 | -else: | |
| 74 | - n = session.query(Student).count() | |
| 75 | - print(f'New database created: {args.db}\n{n} user(s) inserted:') | |
| 76 | - | |
| 77 | - users = session.query(Student).order_by(Student.id).all() | |
| 78 | - print(f' {users[0].id:8} - {users[0].name} (administrator)') | |
| 79 | - if n > 1: | |
| 80 | - print(f' {users[1].id:8} - {users[1].name}') | |
| 81 | - if n > 3: | |
| 82 | - print(' ... ...') | |
| 83 | - if n > 2: | |
| 84 | - print(f' {users[-1].id:8} - {users[-1].name}') | |
| 85 | - | |
| 86 | -# --- end session --- | |
| 116 | + if verbose: | |
| 117 | + for u in users: | |
| 118 | + print(f'{u.id:>12} {u.name}') | |
| 119 | + else: | |
| 120 | + print(f'{users[0].id:>12} {users[0].name}') | |
| 121 | + if n > 1: | |
| 122 | + print(f'{users[1].id:>12} {users[1].name}') | |
| 123 | + if n > 3: | |
| 124 | + print(' | |') | |
| 125 | + if n > 2: | |
| 126 | + print(f'{users[-1].id:>12} {users[-1].name}') | |
| 127 | + print(f'Total: {n}.') | |
| 128 | + | |
| 129 | + | |
| 130 | +# =========================================================================== | |
| 131 | +if __name__=='__main__': | |
| 132 | + args = parse_commandline_arguments() | |
| 133 | + | |
| 134 | + # --- make list of students to insert/update | |
| 135 | + students = [] | |
| 136 | + | |
| 137 | + for csvfile in args.csvfile: | |
| 138 | + print('Adding users from:', csvfile) | |
| 139 | + students.extend(get_students_from_csv(csvfile)) | |
| 140 | + | |
| 141 | + if args.admin: | |
| 142 | + print('Adding user: 0, Admin.') | |
| 143 | + students.append({'uid': '0', 'name': 'Admin'}) | |
| 144 | + | |
| 145 | + if args.add: | |
| 146 | + for uid, name in args.add: | |
| 147 | + print(f'Adding user: {uid}, {name}.') | |
| 148 | + students.append({'uid': uid, 'name': name}) | |
| 149 | + | |
| 150 | + # --- password hashing | |
| 151 | + if students: | |
| 152 | + print(f'Generating password hashes (bcrypt).') | |
| 153 | + hash_func = lambda s: hashpw(s, args.pw) | |
| 154 | + with ThreadPoolExecutor() as executor: | |
| 155 | + executor.map(hash_func, students) # hashing in parallel | |
| 156 | + | |
| 157 | + print() | |
| 158 | + | |
| 159 | + # --- database stuff | |
| 160 | + print(f'Using database: ', args.db) | |
| 161 | + engine = sa.create_engine(f'sqlite:///{args.db}', echo=False) | |
| 162 | + Base.metadata.create_all(engine) # Criate schema if needed | |
| 163 | + Session = sa.orm.sessionmaker(bind=engine) | |
| 164 | + session = Session() | |
| 165 | + | |
| 166 | + if students: | |
| 167 | + print(f'Inserting {len(students)}') | |
| 168 | + insert_students_into_db(session, students) | |
| 169 | + | |
| 170 | + for s in args.update: | |
| 171 | + print(f'Updating password of: {s}') | |
| 172 | + u = session.query(Student).get(s) | |
| 173 | + pw =(args.pw or s).encode('utf-8') | |
| 174 | + u.password = bcrypt.hashpw(pw, bcrypt.gensalt()) | |
| 175 | + session.commit() | |
| 176 | + | |
| 177 | + show_students_in_database(session, args.verbose) | |
| 178 | + | |
| 179 | + session.close() | ... | ... |