Upgrade to new sqlalchemy 1.4/2.0

- models.py -> done - initdb.py -> done

Upgrade to new sqlalchemy 1.4/2.0
- models.py -> done - initdb.py -> done
Miguel Barão
1 parent 5c47ea62
Showing 5 changed files with 83 additions and 96 deletions Show diff stats
aprendizations/initdb.py
aprendizations/models.py
mypy.ini
package-lock.json
setup.py
@@ -9,13 +9,12 @@ import csv
 import argparse
 import re
 from string import capwords
-from concurrent.futures import ThreadPoolExecutor
  
 # third party libraries
 import bcrypt
-import sqlalchemy as sa
-import sqlalchemy.orm as orm
-from sqlalchemy.exc import IntegrityError
+from sqlalchemy import create_engine, select
+from sqlalchemy.orm import Session
+from sqlalchemy.exc import IntegrityError, NoResultFound
  
 # this project
 from aprendizations.models import Base, Student
@@ -103,24 +102,14 @@ def get_students_from_csv(filename):
  
  
 # ===========================================================================
-def hashpw(student, passw=None):
-    '''replace password by hash for a single student'''
-    print('.', end='', flush=True)
-    passw = (passw or student.get('pw', None) or student['uid']).encode('utf-8')
-    student['pw'] = bcrypt.hashpw(passw, bcrypt.gensalt())
-
-
-# ===========================================================================
 def show_students_in_database(session, verbose=False):
-    '''print students that are in the database'''
-    users = session.query(Student).all()
+    '''shows students in the database'''
+    users = session.execute(select(Student)).scalars().all()
     total = len(users)
  
     print('\nRegistered users:')
-    if total == 0:
-        print('  -- none --')
-    else:
-        users.sort(key=lambda u: f'{u.id:>12}')  # sort by number
+    if users:
+        users.sort(key=lambda u: f'{u.id:>12}')  # sort by right aligned string
         if verbose:
             for user in users:
                 print(f'{user.id:>12}   {user.name}')
@@ -142,65 +131,64 @@ def main():
     args = parse_commandline_arguments()
  
     # --- database stuff
-    print(f'Using database: {args.db}')
-    engine = sa.create_engine(f'sqlite:///{args.db}', echo=False)
+    print(f'Database: {args.db}')
+    engine = create_engine(f'sqlite:///{args.db}', echo=False, future=True)
     Base.metadata.create_all(engine)  # Creates schema if needed
-    session = orm.sessionmaker(bind=engine)()
+    session = Session(engine)
  
     # --- build list of students to insert/update
     students = []
  
     for csvfile in args.csvfile:
-        # print('Adding users from:', csvfile)
-        students.extend(get_students_from_csv(csvfile))
+        students += get_students_from_csv(csvfile)
  
     if args.admin:
-        # print('Adding user: 0, Admin.')
         students.append({'uid': '0', 'name': 'Admin'})
  
     if args.add:
         for uid, name in args.add:
-            # print(f'Adding user: {uid}, {name}.')
             students.append({'uid': uid, 'name': name})
  
     # --- only insert students that are not yet in the database
-    db_students = {user.id for user in session.query(Student).all()}
-    new_students = list(filter(lambda s: s['uid'] not in db_students, students))
-
-    if new_students:
-        # --- password hashing
-        print('Generating password hashes', end='')
-        with ThreadPoolExecutor() as executor:
-            executor.map(lambda s: hashpw(s, args.pw), new_students)
-
-        print('\nAdding students:')
-        for student in new_students:
-            print(f'  + {student["uid"]}, {student["name"]}')
-
-        try:
-            session.add_all([Student(id=s['uid'],
-                                     name=s['name'],
-                                     password=s['pw'])
-                            for s in new_students])
-            session.commit()
-        except IntegrityError:
-            print('!!! Integrity error. Aborted !!!\n')
-            session.rollback()
-
-        print(f'Inserted {len(new_students)} new student(s).')
+    print('\nInserting new students:')
+
+    db_students = set(session.execute(select(Student.id)).scalars().all())
+    new_students = (s for s in students if s['uid'] not in db_students)
+    count = 0
+    for s in new_students:
+        print(f'  {s["uid"]}, {s["name"]}')
+
+        pw = args.pw or s['uid']
+        hashed_pw = bcrypt.hashpw(pw.encode('utf-8'), bcrypt.gensalt())
+
+        session.add(Student(id=s['uid'], name=s['name'], password=hashed_pw))
+        count += 1
+
+    try:
+        session.commit()
+    except IntegrityError:
+        print('!!! Integrity error. Aborted !!!\n')
+        session.rollback()
     else:
-        print('There are no new students to add.')
+        print(f'Total {count} new student(s).')
  
     # --- update data for student in the database
-    for student_id in args.update:
-        print(f'Updating password of: {student_id}')
-        student = session.query(Student).get(student_id)
-        if student is not None:
-            passw = (args.pw or student_id).encode('utf-8')
-            student.password = bcrypt.hashpw(passw, bcrypt.gensalt())
-            session.commit()
-        else:
-            print(f'!!! Student {student_id} does not exist. Skipped!!!')
+    if args.update:
+        print('\nUpdating passwords of students:')
+        count = 0
+        for sid in args.update:
+            try:
+                s = session.execute(select(Student).filter_by(id=sid)).scalar_one()
+            except NoResultFound:
+                print(f'  -> student {sid} does not exist!')
+                continue
+            else:
+                print(f'  {sid}, {s.name}')
+                count += 1
+            pw = args.pw or sid
+            s.password = bcrypt.hashpw(pw.encode('utf-8'), bcrypt.gensalt())
+        session.commit()
+        print(f'Total {count} password(s) updated.')
  
     show_students_in_database(session, args.verbose)
  
  
 # python standard library
-from typing import Any
+# from typing import Any
  
 # third party libraries
 from sqlalchemy import Column, ForeignKey, Integer, Float, String
-from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy.orm import relationship
+from sqlalchemy.orm import declarative_base, relationship
  
  
 # ===========================================================================
 #   Declare ORM
 # FIXME Any is a workaround for mypy static type checking (see https://github.com/python/mypy/issues/6372)
-Base: Any = declarative_base()
+# Base: Any = declarative_base()
+Base = declarative_base()
  
  
 # ---------------------------------------------------------------------------
@@ -27,11 +27,11 @@ class StudentTopic(Base):
     topic = relationship('Topic', back_populates='students')
  
     def __repr__(self):
-        return f'''StudentTopic:
-            student_id: "{self.student_id}"
-            topic_id:   "{self.topic_id}"
-            level:      "{self.level}"
-            date:       "{self.date}"'''
+        return ('StudentTopic('
+                f'student_id={self.student_id!r}, '
+                f'topic_id={self.topic_id!r}, '
+                f'level={self.level!r}, '
+                f'date={self.date!r})')
  
  
 # ---------------------------------------------------------------------------
@@ -48,10 +48,10 @@ class Student(Base):
     topics = relationship('StudentTopic', back_populates='student')
  
     def __repr__(self):
-        return f'''Student:
-            id:         "{self.id}"
-            name:       "{self.name}"
-            password:   "{self.password}"'''
+        return ('Student('
+            f'id={self.id!r}, '
+            f'name={self.name!r}, '
+            f'password={self.password!r})')
  
  
 # ---------------------------------------------------------------------------
@@ -72,14 +72,14 @@ class Answer(Base):
     topic = relationship('Topic', back_populates='answers')
  
     def __repr__(self):
-        return f'''Question:
-            id:         "{self.id}"
-            ref:        "{self.ref}"
-            grade:      "{self.grade}"
-            starttime:  "{self.starttime}"
-            finishtime: "{self.finishtime}"
-            student_id: "{self.student_id}"
-            topic_id:   "{self.topic_id}"'''
+        return ('Question('
+            f'id={self.id!r}, '
+            f'ref={self.ref!r}, '
+            f'grade={self.grade!r}, '
+            f'starttime={self.starttime!r}, '
+            f'finishtime={self.finishtime!r}, '
+            f'student_id={self.student_id!r}, '
+            f'topic_id={self.topic_id!r})')
  
  
 # ---------------------------------------------------------------------------
@@ -94,5 +94,4 @@ class Topic(Base):
     answers = relationship('Answer', back_populates='topic')
  
     def __repr__(self):
-        return f'''Topic:
-            id:         "{self.id}"'''
+        return f'Topic(id={self.id!r})'
 [mypy]
-python_version = 3.8
-warn_return_any = True
-warn_unused_configs = True
+python_version = 3.9
+plugins = sqlalchemy.ext.mypy.plugin
  
-[mypy-sqlalchemy.*]
-ignore_missing_imports = True
  
 [mypy-pygments.*]
 ignore_missing_imports = True
@@ -5,8 +5,8 @@
     "packages": {
         "": {
             "dependencies": {
-                "@fortawesome/fontawesome-free": "^5.15.2",
-                "codemirror": "^5.59.1",
+                "@fortawesome/fontawesome-free": "^5.15.3",
+                "codemirror": "^5.59.4",
                 "mdbootstrap": "^4.19.2"
             }
         },
@@ -20,9 +20,9 @@
             }
         },
         "node_modules/codemirror": {
-            "version": "5.61.1",
-            "resolved": "https://registry.npmjs.org/codemirror/-/codemirror-5.61.1.tgz",
-            "integrity": "sha512-+D1NZjAucuzE93vJGbAaXzvoBHwp9nJZWWWF9utjv25+5AZUiah6CIlfb4ikG4MoDsFsCG8niiJH5++OO2LgIQ=="
+            "version": "5.62.0",
+            "resolved": "https://registry.npmjs.org/codemirror/-/codemirror-5.62.0.tgz",
+            "integrity": "sha512-Xnl3304iCc8nyVZuRkzDVVwc794uc9QNX0UcPGeNic1fbzkSrO4l4GVXho9tRNKBgPYZXgocUqXyfIv3BILhCQ=="
         },
         "node_modules/mdbootstrap": {
             "version": "4.19.2",
@@ -32,11 +32,14 @@
     },
     "dependencies": {
         "@fortawesome/fontawesome-free": {
+            "version": "5.15.3",
+            "resolved": "https://registry.npmjs.org/@fortawesome/fontawesome-free/-/fontawesome-free-5.15.3.tgz",
+            "integrity": "sha512-rFnSUN/QOtnOAgqFRooTA3H57JLDm0QEG/jPdk+tLQNL/eWd+Aok8g3qCI+Q1xuDPWpGW/i9JySpJVsq8Q0s9w=="
         },
         "codemirror": {
-            "version": "5.61.1",
-            "resolved": "https://registry.npmjs.org/codemirror/-/codemirror-5.61.1.tgz",
-            "integrity": "sha512-+D1NZjAucuzE93vJGbAaXzvoBHwp9nJZWWWF9utjv25+5AZUiah6CIlfb4ikG4MoDsFsCG8niiJH5++OO2LgIQ=="
+            "version": "5.62.0",
+            "resolved": "https://registry.npmjs.org/codemirror/-/codemirror-5.62.0.tgz",
+            "integrity": "sha512-Xnl3304iCc8nyVZuRkzDVVwc794uc9QNX0UcPGeNic1fbzkSrO4l4GVXho9tRNKBgPYZXgocUqXyfIv3BILhCQ=="
         },
         "mdbootstrap": {
             "version": "4.19.2",
@@ -24,7 +24,7 @@ setup(
         'mistune',
         'pyyaml>=5.1',
         'pygments',
-        'sqlalchemy<1.4',
+        'sqlalchemy>=1.4',
         'bcrypt>=3.1',
         'networkx>=2.4'
         ],
...	...	@@ -9,13 +9,12 @@ import csv
9	9	import argparse
10	10	import re
11	11	from string import capwords
12		-from concurrent.futures import ThreadPoolExecutor
13	12
14	13	# third party libraries
15	14	import bcrypt
16		-import sqlalchemy as sa
17		-import sqlalchemy.orm as orm
18		-from sqlalchemy.exc import IntegrityError
	15	+from sqlalchemy import create_engine, select
	16	+from sqlalchemy.orm import Session
	17	+from sqlalchemy.exc import IntegrityError, NoResultFound
19	18
20	19	# this project
21	20	from aprendizations.models import Base, Student
...	...	@@ -103,24 +102,14 @@ def get_students_from_csv(filename):
103	102
104	103
105	104	# ===========================================================================
106		-def hashpw(student, passw=None):
107		- '''replace password by hash for a single student'''
108		- print('.', end='', flush=True)
109		- passw = (passw or student.get('pw', None) or student['uid']).encode('utf-8')
110		- student['pw'] = bcrypt.hashpw(passw, bcrypt.gensalt())
111		-
112		-
113		-# ===========================================================================
114	105	def show_students_in_database(session, verbose=False):
115		- '''print students that are in the database'''
116		- users = session.query(Student).all()
	106	+ '''shows students in the database'''
	107	+ users = session.execute(select(Student)).scalars().all()
117	108	total = len(users)
118	109
119	110	print('\nRegistered users:')
120		- if total == 0:
121		- print(' -- none --')
122		- else:
123		- users.sort(key=lambda u: f'{u.id:>12}') # sort by number
	111	+ if users:
	112	+ users.sort(key=lambda u: f'{u.id:>12}') # sort by right aligned string
124	113	if verbose:
125	114	for user in users:
126	115	print(f'{user.id:>12} {user.name}')
...	...	@@ -142,65 +131,64 @@ def main():
142	131	args = parse_commandline_arguments()
143	132
144	133	# --- database stuff
145		- print(f'Using database: {args.db}')
146		- engine = sa.create_engine(f'sqlite:///{args.db}', echo=False)
	134	+ print(f'Database: {args.db}')
	135	+ engine = create_engine(f'sqlite:///{args.db}', echo=False, future=True)
147	136	Base.metadata.create_all(engine) # Creates schema if needed
148		- session = orm.sessionmaker(bind=engine)()
	137	+ session = Session(engine)
149	138
150	139	# --- build list of students to insert/update
151	140	students = []
152	141
153	142	for csvfile in args.csvfile:
154		- # print('Adding users from:', csvfile)
155		- students.extend(get_students_from_csv(csvfile))
	143	+ students += get_students_from_csv(csvfile)
156	144
157	145	if args.admin:
158		- # print('Adding user: 0, Admin.')
159	146	students.append({'uid': '0', 'name': 'Admin'})
160	147
161	148	if args.add:
162	149	for uid, name in args.add:
163		- # print(f'Adding user: {uid}, {name}.')
164	150	students.append({'uid': uid, 'name': name})
165	151
166	152	# --- only insert students that are not yet in the database
167		- db_students = {user.id for user in session.query(Student).all()}
168		- new_students = list(filter(lambda s: s['uid'] not in db_students, students))
169		-
170		- if new_students:
171		- # --- password hashing
172		- print('Generating password hashes', end='')
173		- with ThreadPoolExecutor() as executor:
174		- executor.map(lambda s: hashpw(s, args.pw), new_students)
175		-
176		- print('\nAdding students:')
177		- for student in new_students:
178		- print(f' + {student["uid"]}, {student["name"]}')
179		-
180		- try:
181		- session.add_all([Student(id=s['uid'],
182		- name=s['name'],
183		- password=s['pw'])
184		- for s in new_students])
185		- session.commit()
186		- except IntegrityError:
187		- print('!!! Integrity error. Aborted !!!\n')
188		- session.rollback()
189		-
190		- print(f'Inserted {len(new_students)} new student(s).')
	153	+ print('\nInserting new students:')
	154	+
	155	+ db_students = set(session.execute(select(Student.id)).scalars().all())
	156	+ new_students = (s for s in students if s['uid'] not in db_students)
	157	+ count = 0
	158	+ for s in new_students:
	159	+ print(f' {s["uid"]}, {s["name"]}')
	160	+
	161	+ pw = args.pw or s['uid']
	162	+ hashed_pw = bcrypt.hashpw(pw.encode('utf-8'), bcrypt.gensalt())
	163	+
	164	+ session.add(Student(id=s['uid'], name=s['name'], password=hashed_pw))
	165	+ count += 1
	166	+
	167	+ try:
	168	+ session.commit()
	169	+ except IntegrityError:
	170	+ print('!!! Integrity error. Aborted !!!\n')
	171	+ session.rollback()
191	172	else:
192		- print('There are no new students to add.')
	173	+ print(f'Total {count} new student(s).')
193	174
194	175	# --- update data for student in the database
195		- for student_id in args.update:
196		- print(f'Updating password of: {student_id}')
197		- student = session.query(Student).get(student_id)
198		- if student is not None:
199		- passw = (args.pw or student_id).encode('utf-8')
200		- student.password = bcrypt.hashpw(passw, bcrypt.gensalt())
201		- session.commit()
202		- else:
203		- print(f'!!! Student {student_id} does not exist. Skipped!!!')
	176	+ if args.update:
	177	+ print('\nUpdating passwords of students:')
	178	+ count = 0
	179	+ for sid in args.update:
	180	+ try:
	181	+ s = session.execute(select(Student).filter_by(id=sid)).scalar_one()
	182	+ except NoResultFound:
	183	+ print(f' -> student {sid} does not exist!')
	184	+ continue
	185	+ else:
	186	+ print(f' {sid}, {s.name}')
	187	+ count += 1
	188	+ pw = args.pw or sid
	189	+ s.password = bcrypt.hashpw(pw.encode('utf-8'), bcrypt.gensalt())
	190	+ session.commit()
	191	+ print(f'Total {count} password(s) updated.')
204	192
205	193	show_students_in_database(session, args.verbose)
206	194
...	...
1	1
2	2	# python standard library
3		-from typing import Any
	3	+# from typing import Any
4	4
5	5	# third party libraries
6	6	from sqlalchemy import Column, ForeignKey, Integer, Float, String
7		-from sqlalchemy.ext.declarative import declarative_base
8		-from sqlalchemy.orm import relationship
	7	+from sqlalchemy.orm import declarative_base, relationship
9	8
10	9
11	10	# ===========================================================================
12	11	# Declare ORM
13	12	# FIXME Any is a workaround for mypy static type checking (see https://github.com/python/mypy/issues/6372)
14		-Base: Any = declarative_base()
	13	+# Base: Any = declarative_base()
	14	+Base = declarative_base()
15	15
16	16
17	17	# ---------------------------------------------------------------------------
...	...	@@ -27,11 +27,11 @@ class StudentTopic(Base):
27	27	topic = relationship('Topic', back_populates='students')
28	28
29	29	def __repr__(self):
30		- return f'''StudentTopic:
31		- student_id: "{self.student_id}"
32		- topic_id: "{self.topic_id}"
33		- level: "{self.level}"
34		- date: "{self.date}"'''
	30	+ return ('StudentTopic('
	31	+ f'student_id={self.student_id!r}, '
	32	+ f'topic_id={self.topic_id!r}, '
	33	+ f'level={self.level!r}, '
	34	+ f'date={self.date!r})')
35	35
36	36
37	37	# ---------------------------------------------------------------------------
...	...	@@ -48,10 +48,10 @@ class Student(Base):
48	48	topics = relationship('StudentTopic', back_populates='student')
49	49
50	50	def __repr__(self):
51		- return f'''Student:
52		- id: "{self.id}"
53		- name: "{self.name}"
54		- password: "{self.password}"'''
	51	+ return ('Student('
	52	+ f'id={self.id!r}, '
	53	+ f'name={self.name!r}, '
	54	+ f'password={self.password!r})')
55	55
56	56
57	57	# ---------------------------------------------------------------------------
...	...	@@ -72,14 +72,14 @@ class Answer(Base):
72	72	topic = relationship('Topic', back_populates='answers')
73	73
74	74	def __repr__(self):
75		- return f'''Question:
76		- id: "{self.id}"
77		- ref: "{self.ref}"
78		- grade: "{self.grade}"
79		- starttime: "{self.starttime}"
80		- finishtime: "{self.finishtime}"
81		- student_id: "{self.student_id}"
82		- topic_id: "{self.topic_id}"'''
	75	+ return ('Question('
	76	+ f'id={self.id!r}, '
	77	+ f'ref={self.ref!r}, '
	78	+ f'grade={self.grade!r}, '
	79	+ f'starttime={self.starttime!r}, '
	80	+ f'finishtime={self.finishtime!r}, '
	81	+ f'student_id={self.student_id!r}, '
	82	+ f'topic_id={self.topic_id!r})')
83	83
84	84
85	85	# ---------------------------------------------------------------------------
...	...	@@ -94,5 +94,4 @@ class Topic(Base):
94	94	answers = relationship('Answer', back_populates='topic')
95	95
96	96	def __repr__(self):
97		- return f'''Topic:
98		- id: "{self.id}"'''
	97	+ return f'Topic(id={self.id!r})'
...	...
1	1	[mypy]
2		-python_version = 3.8
3		-warn_return_any = True
4		-warn_unused_configs = True
	2	+python_version = 3.9
	3	+plugins = sqlalchemy.ext.mypy.plugin
5	4
6		-[mypy-sqlalchemy.*]
7		-ignore_missing_imports = True
8	5
9	6	[mypy-pygments.*]
10	7	ignore_missing_imports = True
...	...
...	...	@@ -5,8 +5,8 @@
5	5	"packages": {
6	6	"": {
7	7	"dependencies": {
8		- "@fortawesome/fontawesome-free": "^5.15.2",
9		- "codemirror": "^5.59.1",
	8	+ "@fortawesome/fontawesome-free": "^5.15.3",
	9	+ "codemirror": "^5.59.4",
10	10	"mdbootstrap": "^4.19.2"
11	11	}
12	12	},
...	...	@@ -20,9 +20,9 @@
20	20	}
21	21	},
22	22	"node_modules/codemirror": {
23		- "version": "5.61.1",
24		- "resolved": "https://registry.npmjs.org/codemirror/-/codemirror-5.61.1.tgz",
25		- "integrity": "sha512-+D1NZjAucuzE93vJGbAaXzvoBHwp9nJZWWWF9utjv25+5AZUiah6CIlfb4ikG4MoDsFsCG8niiJH5++OO2LgIQ=="
	23	+ "version": "5.62.0",
	24	+ "resolved": "https://registry.npmjs.org/codemirror/-/codemirror-5.62.0.tgz",
	25	+ "integrity": "sha512-Xnl3304iCc8nyVZuRkzDVVwc794uc9QNX0UcPGeNic1fbzkSrO4l4GVXho9tRNKBgPYZXgocUqXyfIv3BILhCQ=="
26	26	},
27	27	"node_modules/mdbootstrap": {
28	28	"version": "4.19.2",
...	...	@@ -32,11 +32,14 @@
32	32	},
33	33	"dependencies": {
34	34	"@fortawesome/fontawesome-free": {
	35	+ "version": "5.15.3",
	36	+ "resolved": "https://registry.npmjs.org/@fortawesome/fontawesome-free/-/fontawesome-free-5.15.3.tgz",
	37	+ "integrity": "sha512-rFnSUN/QOtnOAgqFRooTA3H57JLDm0QEG/jPdk+tLQNL/eWd+Aok8g3qCI+Q1xuDPWpGW/i9JySpJVsq8Q0s9w=="
35	38	},
36	39	"codemirror": {
37		- "version": "5.61.1",
38		- "resolved": "https://registry.npmjs.org/codemirror/-/codemirror-5.61.1.tgz",
39		- "integrity": "sha512-+D1NZjAucuzE93vJGbAaXzvoBHwp9nJZWWWF9utjv25+5AZUiah6CIlfb4ikG4MoDsFsCG8niiJH5++OO2LgIQ=="
	40	+ "version": "5.62.0",
	41	+ "resolved": "https://registry.npmjs.org/codemirror/-/codemirror-5.62.0.tgz",
	42	+ "integrity": "sha512-Xnl3304iCc8nyVZuRkzDVVwc794uc9QNX0UcPGeNic1fbzkSrO4l4GVXho9tRNKBgPYZXgocUqXyfIv3BILhCQ=="
40	43	},
41	44	"mdbootstrap": {
42	45	"version": "4.19.2",
...	...
...	...	@@ -24,7 +24,7 @@ setup(
24	24	'mistune',
25	25	'pyyaml>=5.1',
26	26	'pygments',
27		- 'sqlalchemy<1.4',
	27	+ 'sqlalchemy>=1.4',
28	28	'bcrypt>=3.1',
29	29	'networkx>=2.4'
30	30	],
...	...