Commit 1edf739ffb890e486abf956410c0cfc1bc5c373a
1 parent
a5c2ee14
Exists in
master
and in
1 other branch
- nova versão do initdb, igual à do aprendizations
Showing
2 changed files
with
161 additions
and
67 deletions
Show diff stats
BUGS.md
| 1 | 1 | ||
| 2 | # BUGS | 2 | # BUGS |
| 3 | 3 | ||
| 4 | +- o eventloop está a bloquear. correção do teste é blocking. usar threadpoolexecutor? | ||
| 4 | - se submissão falhar (aluno desconectado da rede) nao pode sair da página para nao perder o teste. possiveis solucoes: | 5 | - se submissão falhar (aluno desconectado da rede) nao pode sair da página para nao perder o teste. possiveis solucoes: |
| 5 | - botao submeter valida se esta online com um post willing_to_submit, se estiver online, mostra mensagem de confirmacao, caso contrario avisa que nao esta online. | 6 | - botao submeter valida se esta online com um post willing_to_submit, se estiver online, mostra mensagem de confirmacao, caso contrario avisa que nao esta online. |
| 6 | - periodicamente, cada user faz um post keep-alive. warning nos logs se offline. | 7 | - periodicamente, cada user faz um post keep-alive. warning nos logs se offline. |
initdb.py
| 1 | -#!/usr/bin/env python3.6 | ||
| 2 | -# -*- coding: utf-8 -*- | 1 | +#!/usr/bin/env python3 |
| 3 | 2 | ||
| 3 | +# base | ||
| 4 | import csv | 4 | import csv |
| 5 | import argparse | 5 | import argparse |
| 6 | import re | 6 | import re |
| 7 | import string | 7 | import string |
| 8 | -import sys | ||
| 9 | -from sqlalchemy import create_engine | ||
| 10 | -from sqlalchemy.orm import sessionmaker | 8 | +from concurrent.futures import ThreadPoolExecutor |
| 11 | 9 | ||
| 10 | +# installed packages | ||
| 11 | +import bcrypt | ||
| 12 | +import sqlalchemy as sa | ||
| 13 | + | ||
| 14 | +# this project | ||
| 12 | from models import Base, Student, Test, Question | 15 | from models import Base, Student, Test, Question |
| 13 | 16 | ||
| 17 | + | ||
| 18 | +# =========================================================================== | ||
| 19 | +# Parse command line options | ||
| 20 | +def parse_commandline_arguments(): | ||
| 21 | + argparser = argparse.ArgumentParser( | ||
| 22 | + formatter_class=argparse.ArgumentDefaultsHelpFormatter, | ||
| 23 | + description='Insert new users into a database. Users can be imported from CSV files in the SIIUE format or defined in the command line. If the database does not exist, a new one is created.') | ||
| 24 | + | ||
| 25 | + argparser.add_argument('csvfile', | ||
| 26 | + nargs='*', | ||
| 27 | + type=str, | ||
| 28 | + default='', | ||
| 29 | + help='CSV file to import (SIIUE)') | ||
| 30 | + | ||
| 31 | + argparser.add_argument('--db', | ||
| 32 | + default='students.db', | ||
| 33 | + type=str, | ||
| 34 | + help='database file') | ||
| 35 | + | ||
| 36 | + argparser.add_argument('-A', '--admin', | ||
| 37 | + action='store_true', | ||
| 38 | + help='insert the admin user') | ||
| 39 | + | ||
| 40 | + argparser.add_argument('-a', '--add', | ||
| 41 | + nargs=2, | ||
| 42 | + action='append', | ||
| 43 | + metavar=('uid', 'name'), | ||
| 44 | + help='add new user') | ||
| 45 | + | ||
| 46 | + argparser.add_argument('-u', '--update', | ||
| 47 | + nargs='+', | ||
| 48 | + metavar='uid', | ||
| 49 | + default=[], | ||
| 50 | + help='users to update') | ||
| 51 | + | ||
| 52 | + argparser.add_argument('--pw', | ||
| 53 | + default=None, | ||
| 54 | + type=str, | ||
| 55 | + help='set password for new and updated users') | ||
| 56 | + | ||
| 57 | + argparser.add_argument('-V', '--verbose', | ||
| 58 | + action='store_true', | ||
| 59 | + help='show all students in database') | ||
| 60 | + | ||
| 61 | + return argparser.parse_args() | ||
| 62 | + | ||
| 63 | + | ||
| 64 | +# =========================================================================== | ||
| 14 | # SIIUE names have alien strings like "(TE)" and are sometimes capitalized | 65 | # SIIUE names have alien strings like "(TE)" and are sometimes capitalized |
| 15 | # We remove them so that students dont keep asking what it means | 66 | # We remove them so that students dont keep asking what it means |
| 16 | -def fix(name): | ||
| 17 | - return string.capwords(re.sub('\(.*\)', '', name).strip()) | 67 | +def get_students_from_csv(filename): |
| 68 | + try: | ||
| 69 | + csvreader = csv.DictReader(open(filename, encoding='iso-8859-1'), delimiter=';', quotechar='"', skipinitialspace=True) | ||
| 70 | + except EnvironmentError: | ||
| 71 | + print(f'!!! Error. File "{filename}" not found !!!') | ||
| 72 | + students = [] | ||
| 73 | + else: | ||
| 74 | + students = [{ | ||
| 75 | + 'uid': s['N.º'], | ||
| 76 | + 'name': string.capwords(re.sub('\(.*\)', '', s['Nome']).strip()) | ||
| 77 | + } for s in csvreader] | ||
| 78 | + | ||
| 79 | + return students | ||
| 80 | + | ||
| 18 | 81 | ||
| 19 | # =========================================================================== | 82 | # =========================================================================== |
| 20 | -# Parse command line options | ||
| 21 | -argparser = argparse.ArgumentParser(description='Create new database from a CSV file (SIIUE format)') | ||
| 22 | -argparser.add_argument('--db', default='students.db', type=str, help='database filename') | ||
| 23 | -argparser.add_argument('--demo', action='store_true', help='initialize database with a few fake students') | ||
| 24 | -argparser.add_argument('csvfile', nargs='?', type=str, default='', help='CSV filename') | ||
| 25 | -args = argparser.parse_args() | 83 | +# replace password by hash for a single student |
| 84 | +def hashpw(student, pw=None): | ||
| 85 | + print('.', end='', flush=True) | ||
| 86 | + pw = (pw or student.get('pw', None) or student['uid']).encode('utf-8') | ||
| 87 | + student['pw'] = bcrypt.hashpw(pw, bcrypt.gensalt()) | ||
| 88 | + | ||
| 26 | 89 | ||
| 27 | # =========================================================================== | 90 | # =========================================================================== |
| 28 | -engine = create_engine('sqlite:///{}'.format(args.db), echo=False) | 91 | +def insert_students_into_db(session, students): |
| 92 | + try: | ||
| 93 | + # --- start db session --- | ||
| 94 | + session.add_all([Student(id=s['uid'], name=s['name'], password=s['pw']) | ||
| 95 | + for s in students]) | ||
| 29 | 96 | ||
| 30 | -# Criate schema if needed | ||
| 31 | -Base.metadata.create_all(engine) | 97 | + session.commit() |
| 32 | 98 | ||
| 33 | -Session = sessionmaker(bind=engine) | 99 | + except sa.exc.IntegrityError: |
| 100 | + print('!!! Integrity error. User(s) already in database. None inserted !!!\n') | ||
| 101 | + session.rollback() | ||
| 34 | 102 | ||
| 35 | -# --- start session --- | ||
| 36 | -try: | ||
| 37 | - session = Session() | ||
| 38 | 103 | ||
| 39 | - # add administrator | ||
| 40 | - session.add(Student(id='0', name='Professor', password='')) | ||
| 41 | - | ||
| 42 | - # add students | ||
| 43 | - if args.csvfile: | ||
| 44 | - # from csv file if available | ||
| 45 | - try: | ||
| 46 | - csvreader = csv.DictReader(open(args.csvfile, encoding='iso-8859-1'), delimiter=';', quotechar='"', skipinitialspace=True) | ||
| 47 | - except OSError: | ||
| 48 | - print(f'Error: Can\'t open CSV file "{args.csvfile}".') | ||
| 49 | - session.rollback() | ||
| 50 | - sys.exit(1) | 104 | +# =========================================================================== |
| 105 | +def show_students_in_database(session, verbose=False): | ||
| 106 | + try: | ||
| 107 | + users = session.query(Student).order_by(Student.id).all() | ||
| 108 | + except: | ||
| 109 | + raise | ||
| 110 | + else: | ||
| 111 | + n = len(users) | ||
| 112 | + print(f'Registered users:') | ||
| 113 | + if n == 0: | ||
| 114 | + print(' -- none --') | ||
| 51 | else: | 115 | else: |
| 52 | - session.add_all([Student(id=r['N.º'], name=fix(r['Nome']), password='') for r in csvreader]) | ||
| 53 | - elif args.demo: | ||
| 54 | - # add a few fake students | ||
| 55 | - fakes = [ | ||
| 56 | - ['1915', 'Alan Turing'], | ||
| 57 | - ['1938', 'Donald Knuth'], | ||
| 58 | - ['1815', 'Ada Lovelace'], | ||
| 59 | - ['1969', 'Linus Torvalds'], | ||
| 60 | - ['1955', 'Tim Burners-Lee'], | ||
| 61 | - ['1916', 'Claude Shannon'], | ||
| 62 | - ['1903', 'John von Neumann'], | ||
| 63 | - ] | ||
| 64 | - session.add_all([Student(id=i, name=name, password='') for i,name in fakes]) | ||
| 65 | - | ||
| 66 | - session.commit() | ||
| 67 | - | ||
| 68 | -except Exception: | ||
| 69 | - print('Error: Database already exists.') | ||
| 70 | - session.rollback() | ||
| 71 | - sys.exit(1) | ||
| 72 | - | ||
| 73 | -else: | ||
| 74 | - n = session.query(Student).count() | ||
| 75 | - print(f'New database created: {args.db}\n{n} user(s) inserted:') | ||
| 76 | - | ||
| 77 | - users = session.query(Student).order_by(Student.id).all() | ||
| 78 | - print(f' {users[0].id:8} - {users[0].name} (administrator)') | ||
| 79 | - if n > 1: | ||
| 80 | - print(f' {users[1].id:8} - {users[1].name}') | ||
| 81 | - if n > 3: | ||
| 82 | - print(' ... ...') | ||
| 83 | - if n > 2: | ||
| 84 | - print(f' {users[-1].id:8} - {users[-1].name}') | ||
| 85 | - | ||
| 86 | -# --- end session --- | 116 | + if verbose: |
| 117 | + for u in users: | ||
| 118 | + print(f'{u.id:>12} {u.name}') | ||
| 119 | + else: | ||
| 120 | + print(f'{users[0].id:>12} {users[0].name}') | ||
| 121 | + if n > 1: | ||
| 122 | + print(f'{users[1].id:>12} {users[1].name}') | ||
| 123 | + if n > 3: | ||
| 124 | + print(' | |') | ||
| 125 | + if n > 2: | ||
| 126 | + print(f'{users[-1].id:>12} {users[-1].name}') | ||
| 127 | + print(f'Total: {n}.') | ||
| 128 | + | ||
| 129 | + | ||
| 130 | +# =========================================================================== | ||
| 131 | +if __name__=='__main__': | ||
| 132 | + args = parse_commandline_arguments() | ||
| 133 | + | ||
| 134 | + # --- make list of students to insert/update | ||
| 135 | + students = [] | ||
| 136 | + | ||
| 137 | + for csvfile in args.csvfile: | ||
| 138 | + print('Adding users from:', csvfile) | ||
| 139 | + students.extend(get_students_from_csv(csvfile)) | ||
| 140 | + | ||
| 141 | + if args.admin: | ||
| 142 | + print('Adding user: 0, Admin.') | ||
| 143 | + students.append({'uid': '0', 'name': 'Admin'}) | ||
| 144 | + | ||
| 145 | + if args.add: | ||
| 146 | + for uid, name in args.add: | ||
| 147 | + print(f'Adding user: {uid}, {name}.') | ||
| 148 | + students.append({'uid': uid, 'name': name}) | ||
| 149 | + | ||
| 150 | + # --- password hashing | ||
| 151 | + if students: | ||
| 152 | + print(f'Generating password hashes (bcrypt).') | ||
| 153 | + hash_func = lambda s: hashpw(s, args.pw) | ||
| 154 | + with ThreadPoolExecutor() as executor: | ||
| 155 | + executor.map(hash_func, students) # hashing in parallel | ||
| 156 | + | ||
| 157 | + print() | ||
| 158 | + | ||
| 159 | + # --- database stuff | ||
| 160 | + print(f'Using database: ', args.db) | ||
| 161 | + engine = sa.create_engine(f'sqlite:///{args.db}', echo=False) | ||
| 162 | + Base.metadata.create_all(engine) # Criate schema if needed | ||
| 163 | + Session = sa.orm.sessionmaker(bind=engine) | ||
| 164 | + session = Session() | ||
| 165 | + | ||
| 166 | + if students: | ||
| 167 | + print(f'Inserting {len(students)}') | ||
| 168 | + insert_students_into_db(session, students) | ||
| 169 | + | ||
| 170 | + for s in args.update: | ||
| 171 | + print(f'Updating password of: {s}') | ||
| 172 | + u = session.query(Student).get(s) | ||
| 173 | + pw =(args.pw or s).encode('utf-8') | ||
| 174 | + u.password = bcrypt.hashpw(pw, bcrypt.gensalt()) | ||
| 175 | + session.commit() | ||
| 176 | + | ||
| 177 | + show_students_in_database(session, args.verbose) | ||
| 178 | + | ||
| 179 | + session.close() |