From a203d3ccc52a4cd6d46aeff2efc28ebd161b1f87 Mon Sep 17 00:00:00 2001 From: Miguel Barao Date: Thu, 29 Nov 2018 15:30:01 +0000 Subject: [PATCH] - new http server redirects all traffic to a given (https) server. - documentation updates to python3.7. - started to make changes to allow multiple configuration files. --- README.md | 34 ++++++++++++++++++---------------- demo/demo.yaml | 8 ++++---- factory.py | 12 ++++++++++-- http-redirect.py | 87 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ knowledge.py | 44 +++++++++++++------------------------------- learnapp.py | 169 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++------------------------------------------------------------------------- 6 files changed, 228 insertions(+), 126 deletions(-) create mode 100755 http-redirect.py diff --git a/README.md b/README.md index 3d96adb..3f4f855 100644 --- a/README.md +++ b/README.md @@ -5,23 +5,23 @@ Before installing the server, we will need to install python with some additional packages. -### Install python3.6 with sqlite3 support +### Install python3.7 with sqlite3 support This can be done using the system package management, downloaded from [http://www.python.org](), or compiled from sources. - Installing from the system package manager: ```sh - sudo port install python36 # MacOS - sudo pkg install python36 py36-sqlite3 # FreeBSD + sudo port install python37 # MacOS + sudo pkg install python37 py37-sqlite3 # FreeBSD sudo apt install ?not available yet? # Linux, install from source ``` - Installing from source: Download from [http://www.python.org]() and ```sh - unxz Python-3.6.tar.xz - tar xvf Python-3.6.tar - cd Python-3.6 + unxz Python-3.7.tar.xz + tar xvf Python-3.7.tar + cd Python-3.7 ./configure --prefix=$HOME/.local/bin make && make install ``` @@ -33,7 +33,7 @@ This can be done using the system package management, downloaded from [http://ww If the `pip` command is not yet installed, run ```sh -python3.6 -m ensurepip --user +python3.7 -m ensurepip --user ``` This will install pip in your user account under `~/.local/bin`. @@ -54,15 +54,15 @@ pip3 install --user \ sqlalchemy \ pyyaml \ pygments \ - markdown \ + mistune \ bcrypt \ networkx ``` These are usually installed under -- `~/.local/lib/python3.6/site-packages/` in Linux/FreeBSD. -- `~/Library/python/3.6/lib/python/site-packages/` in MacOS. +- `~/.local/lib/python3.7/site-packages/` in Linux/FreeBSD. +- `~/Library/python/3.7/lib/python/site-packages/` in MacOS. ## Installation @@ -82,18 +82,19 @@ The user data is maintained in a sqlite3 database file. We first need to create ```sh cd aprendizations -./initdb.py --help # for the available options ./initdb.py # show current database or initialize empty if nonexisting ./initdb.py inscricoes.csv # add students from CSV, passwords are the numbers -./initdb.py --add 1184 "Ana Bola" # add new user (password=1184) -./initdb.py --update 1184 --pw alibaba # update password +./initdb.py --add 1184 "Aladino da Silva" # add new user (default password=1184) +./initdb.py --update 1184 --pw alibaba # update password of given student +./initdb.py --help # for the available options ``` ### SSL Certificates We need certificates for https. Certificates can be self-signed or validated by a trusted authority. -Self-signed can be used for testing, but browsers will complain. LetsEncrypt issues trusted and free certificates, but the server must have a fixed IP and a registered domain name in the DNS (dynamic DNS does not work). +Self-signed can be used locally for development and testing, but browsers will complain. +LetsEncrypt issues trusted and free certificates, but the server must have a registered publicly accessible domain name. #### Selfsigned @@ -109,7 +110,7 @@ openssl req -x509 -newkey rsa:4096 -keyout privkey.pem -out cert.pem -days 365 - sudo pkg install py27-certbot # FreeBSD ``` -Shutdown the firewall and any server running. Then run the script to generate the certificate: +Shutdown the firewall and any web server that might be running. Then run the script to generate the certificate: ```sh sudo service pf stop # disable pf firewall (FreeBSD) @@ -144,7 +145,7 @@ The application includes a small example in `demo/demo.yaml`. Run it with ./serve.py demo/demo.yaml ``` -and open a browser at [https://127.0.0.1:8443](). If it everything looks good, check at the correct address `https://www.example.com` (requires port forward in the firewall). +and open a browser at [https://127.0.0.1:8443](). If it everything looks good, check at the correct address `https://www.example.com` (requires port forward in the firewall). The option `--debug` provides more verbose logging and might be useful during testing. ### Firewall configuration @@ -178,6 +179,7 @@ pflog_logfile="/var/log/pflog" Reboot or `sudo service pf start`. + ## Troubleshooting To help with troubleshooting, use the option `--debug` when running the server. This will increase logs in the terminal and will present the python exception errors in the browser. diff --git a/demo/demo.yaml b/demo/demo.yaml index 54a9455..1f87bc8 100644 --- a/demo/demo.yaml +++ b/demo/demo.yaml @@ -1,13 +1,13 @@ title: Example database: students.db +path: ./demo -# default global values +# values applie to each topic, if undefined there +# default values are: file=question.yaml, shuffle=True, choose: all file: questions.yaml shuffle: True choose: 6 - -# path prefix applied to the topics -path: ./demo +forgetting_factor: 0.99 # Representation of the edges of the dependency graph. # Example: A depends on B and C diff --git a/factory.py b/factory.py index e0f2c0f..06d3998 100644 --- a/factory.py +++ b/factory.py @@ -38,6 +38,14 @@ from questions import QuestionInformation, QuestionRadio, QuestionCheckbox, Ques logger = logging.getLogger(__name__) +# Topic Factory +# Generates a list +# class RunningTopic(object): +# def __init__(self, graph, topic): + + + + # =========================================================================== # Question Factory @@ -59,7 +67,7 @@ class QFactory(object): 'success' : QuestionInformation, } - def __init__(self, question_dict): + def __init__(self, question_dict={}): self.question = question_dict # ----------------------------------------------------------------------- @@ -86,7 +94,7 @@ class QFactory(object): try: qinstance = self._types[q['type']](q) # instance with correct class except KeyError as e: - logger.error(f'Failed to generate question "{q["ref"]}"') + logger.error(f'Unknown type "{q["type"]}" in question "{q["ref"]}"') raise e else: return qinstance diff --git a/http-redirect.py b/http-redirect.py new file mode 100755 index 0000000..ae1fbeb --- /dev/null +++ b/http-redirect.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 + +# python standard library +from os import path +import sys +import argparse +import signal + +# user installed libraries +from tornado import ioloop, web, httpserver + + +# ============================================================================ +# WebApplication - Tornado Web Server +# ============================================================================ +class WebRedirectApplication(web.Application): + def __init__(self, target='https://localhost'): + handlers = [ + (r'/*', RootHandler), # redirect to https + ] + super().__init__(handlers) + self.target = target + +# ============================================================================ +# Handlers +# ============================================================================ +class RootHandler(web.RequestHandler): + SUPPORTED_METHODS = ['GET'] + + def get(self): + print('Redirecting...') + self.redirect(self.application.target) + + +# ------------------------------------------------------------------------- +def signal_handler(signal, frame): + r = input(' --> Stop webserver? (yes/no) ').lower() + if r == 'yes': + ioloop.IOLoop.current().stop() + print('Webserver stopped.') + sys.exit(0) + + +# ------------------------------------------------------------------------- +# Tornado web server +# ---------------------------------------------------------------------------- +def main(): + # --- Commandline argument parsing + argparser = argparse.ArgumentParser( + description='Redirection server. Any request to http will be redirected to a target server provided in the command line argument.') + argparser.add_argument('target', type=str, + help='Target https server address, e.g. https://www.example.com.') + argparser.add_argument('--port', type=int, default=8080, + help='Port to which this server will listen to, e.g. 8080') + arg = argparser.parse_args() + + # --- create web redirect application + try: + redirectapp = WebRedirectApplication(target=arg.target) + except Exception as e: + print('Failed to start web redirect application.') + raise e + + # --- create webserver + try: + http_server = httpserver.HTTPServer(redirectapp) + except Exception as e: + print('Failed to create HTTP server.') + raise e + else: + http_server.listen(8080) + + # --- run webserver + print(f'Redirecting port {arg.port} to {arg.target} (Ctrl-C to stop)') + signal.signal(signal.SIGINT, signal_handler) + + try: + ioloop.IOLoop.current().start() # running... + except Exception as e: + print('Redirection stopped!') + ioloop.IOLoop.current().stop() + raise e + + +# ---------------------------------------------------------------------------- +if __name__ == "__main__": + main() diff --git a/knowledge.py b/knowledge.py index 021f326..e2cb796 100644 --- a/knowledge.py +++ b/knowledge.py @@ -17,7 +17,7 @@ logger = logging.getLogger(__name__) # kowledge state of each student....?? # Contains: # state - dict of topics with state of unlocked topics -# deps - dependency graph +# deps - access to dependency graph shared between students # topic_sequence - list with the order of recommended topics # ---------------------------------------------------------------------------- class StudentKnowledge(object): @@ -28,12 +28,11 @@ class StudentKnowledge(object): self.deps = deps # dependency graph shared among students self.state = state # {'topic': {'level':0.5, 'date': datetime}, ...} - self.update_topic_levels() # forgetting factor + self.update_topic_levels() # applies forgetting factor self.topic_sequence = self.recommend_topic_sequence() # ['a', 'b', ...] - self.unlock_topics() # whose dependencies have been done + self.unlock_topics() # whose dependencies have been completed self.current_topic = None - print(deps.graph) - self.MAX_QUESTIONS = deps.graph['config'].get('choose', 10) + # self.MAX_QUESTIONS = deps.graph['config'].get('choose', None) # ------------------------------------------------------------------------ # Updates the proficiency levels of the topics, with forgetting factor @@ -67,18 +66,13 @@ class StudentKnowledge(object): # ------------------------------------------------------------------------ - # Start a new topic. If not provided, gets a recommendation. + # Start a new topic. # questions: list of generated questions to do in the topic # current_question: the current question to be presented # ------------------------------------------------------------------------ - def init_topic(self, topic=''): + def init_topic(self, topic): logger.debug(f'StudentKnowledge.init_topic({topic})') - # maybe get topic recommendation - if not topic: - topic = self.get_recommended_topic() - logger.debug(f'Recommended topic is {topic}') - # do not allow locked topics if self.is_locked(topic): logger.debug(f'Topic {topic} is locked') @@ -105,23 +99,6 @@ class StudentKnowledge(object): # get first question self.next_question() - - # def init_learning(self, topic=''): - # logger.debug(f'StudentKnowledge.init_learning({topic})') - - # if self.is_locked(topic): - # logger.debug(f'Topic {topic} is locked') - # return False - - # self.current_topic = topic - # factory = self.deps.node[topic]['factory'] - # lesson = self.deps.node[topic]['lesson'] - - # self.questions = [factory[qref].generate() for qref in lesson] - # logger.debug(f'Total: {len(self.questions)} questions') - - # self.next_question_in_lesson() - # ------------------------------------------------------------------------ # The topic has finished and there are no more questions. # The topic level is updated in state and unlocks are performed. @@ -204,6 +181,11 @@ class StudentKnowledge(object): # pure functions of the state (no side effects) # ======================================================================== + def get_max_questions(self, topic=None): + if topic is not None: + node = self.deps.nodes[topic] + max_questions = node.get('choose', len(node['questions'])) + # ------------------------------------------------------------------------ # compute recommended sequence of topics ['a', 'b', ...] @@ -251,6 +233,6 @@ class StudentKnowledge(object): # ------------------------------------------------------------------------ # Recommends a topic to practice/learn from the state. # ------------------------------------------------------------------------ - def get_recommended_topic(self): # FIXME untested - return min(self.state.items(), key=lambda x: x[1]['level'])[0] + # def get_recommended_topic(self): # FIXME untested + # return min(self.state.items(), key=lambda x: x[1]['level'])[0] diff --git a/learnapp.py b/learnapp.py index fc01a1a..6296332 100644 --- a/learnapp.py +++ b/learnapp.py @@ -57,12 +57,14 @@ class LearnApp(object): session.close() # ------------------------------------------------------------------------ - def __init__(self, config_file): - # state of online students - self.online = {} - config = load_yaml(config_file[0]) - self.db_setup(config['database']) # setup and check students - self.deps = build_dependency_graph(config) + def __init__(self, config_files): + self.db_setup() # setup database and check students + self.online = dict() # online students + + self.deps = nx.DiGraph() + for c in config_files: + self.populate_graph(c) + self.db_add_missing_topics(self.deps.nodes()) # ------------------------------------------------------------------------ @@ -190,7 +192,6 @@ class LearnApp(object): loop = asyncio.get_running_loop() try: await loop.run_in_executor(None, self.online[uid]['state'].init_topic, topic) - # self.online[uid]['state'].init_topic(topic) except KeyError as e: logger.warning(f'User "{uid}" tried to open nonexistent topic: "{topic}"') raise e @@ -200,10 +201,10 @@ class LearnApp(object): # ------------------------------------------------------------------------ # Fill db table 'Topic' with topics from the graph if not already there. # ------------------------------------------------------------------------ - def db_add_missing_topics(self, nn): + def db_add_missing_topics(self, topics): with self.db_session() as s: - tt = [t[0] for t in s.query(Topic.id)] # db list of topics - missing_topics = [Topic(id=n) for n in nn if n not in tt] + dbtopics = [t[0] for t in s.query(Topic.id)] # get topics from DB + missing_topics = [Topic(id=t) for t in topics if t not in dbtopics] if missing_topics: s.add_all(missing_topics) logger.info(f'Added {len(missing_topics)} new topics to the database') @@ -211,7 +212,7 @@ class LearnApp(object): # ------------------------------------------------------------------------ # setup and check database # ------------------------------------------------------------------------ - def db_setup(self, db): + def db_setup(self, db='students.db'): logger.info(f'Checking database "{db}":') engine = create_engine(f'sqlite:///{db}', echo=False) self.Session = sessionmaker(bind=engine) @@ -260,7 +261,7 @@ class LearnApp(object): # ------------------------------------------------------------------------ def get_title(self): - return self.deps.graph['title'] + return self.deps.graph.get('title', '') # FIXME # ------------------------------------------------------------------------ def get_topic_name(self, ref): @@ -278,64 +279,86 @@ class LearnApp(object): -# ============================================================================ -# Builds a digraph. -# -# First, topics such as `computer/mips/exceptions` are added as nodes -# together with dependencies. Then, questions are loaded to a factory. -# -# g.graph['path'] base path where topic directories are located -# g.graph['title'] title defined in the configuration YAML -# g.graph['database'] sqlite3 database file to use -# -# Nodes are the topic references e.g. 'my/topic' -# g.node['my/topic']['name'] name of the topic -# g.node['my/topic']['questions'] list of question refs defined in YAML -# g.node['my/topic']['factory'] dict with question factories -# -# Edges are obtained from the deps defined in the YAML file for each topic. -# ---------------------------------------------------------------------------- -def build_dependency_graph(config={}): - logger.info('Building graph and loading questions:') - - # create graph - prefix = config.get('path', '.') - title = config.get('title', '') - database = config.get('database', 'students.db') - g = nx.DiGraph(path=prefix, title=title, database=database, config=config) - - # iterate over topics and build graph - topics = config.get('topics', {}) - tcount = qcount = 0 # topic and question counters - for ref,attr in topics.items(): - g.add_node(ref) - tnode = g.node[ref] # current node (topic) - - if isinstance(attr, dict): - tnode['type'] = attr.get('type', 'topic') - tnode['name'] = attr.get('name', ref) - tnode['questions'] = attr.get('questions', []) - g.add_edges_from((d,ref) for d in attr.get('deps', [])) - - fullpath = path.expanduser(path.join(prefix, ref)) - - # load questions - filename = path.join(fullpath, 'questions.yaml') - loaded_questions = load_yaml(filename, default=[]) # list - if not tnode['questions']: - # if questions not in configuration then load all, preserving order - tnode['questions'] = [q.setdefault('ref', f'{ref}:{i}') for i,q in enumerate(loaded_questions)] - - # make questions factory (without repeating same question) - tnode['factory'] = {} - for q in loaded_questions: - if q['ref'] in tnode['questions']: - q['path'] = fullpath # fullpath added to each question - tnode['factory'][q['ref']] = QFactory(q) - - logger.info(f'{len(tnode["questions"]):6} {ref}') - qcount += len(tnode["questions"]) # count total questions - tcount += 1 - - logger.info(f'Total loaded: {tcount} topics, {qcount} questions') - return g + # ============================================================================ + # Populates a digraph. + # + # First, topics such as `computer/mips/exceptions` are added as nodes + # together with dependencies. Then, questions are loaded to a factory. + # + # g.graph['path'] base path where topic directories are located + # g.graph['title'] title defined in the configuration YAML + # g.graph['database'] sqlite3 database file to use + # + # Nodes are the topic references e.g. 'my/topic' + # g.node['my/topic']['name'] name of the topic + # g.node['my/topic']['questions'] list of question refs defined in YAML + # g.node['my/topic']['factory'] dict with question factories + # + # Edges are obtained from the deps defined in the YAML file for each topic. + # ---------------------------------------------------------------------------- + def populate_graph(self, conffile): + logger.info(f'Loading {conffile} and populating graph:') + g = self.deps # the graph + config = load_yaml(conffile) # course configuration + + # set attributes of the graph + prefix = path.expanduser(config.get('path', '.')) + # title = config.get('title', '') + # database = config.get('database', 'students.db') + + # default attributes that apply to the topics + default_file = config.get('file', 'questions.yaml') + default_shuffle = config.get('shuffle', True) + default_choose = config.get('choose', 9999) + default_forgetting_factor = config.get('forgetting_factor', 1.0) + + # create graph + # g = nx.DiGraph(path=prefix, title=title, database=database, config=config) + + + # iterate over topics and populate graph + topics = config.get('topics', {}) + tcount = qcount = 0 # topic and question counters + for tref, attr in topics.items(): + if tref in g: + logger.error(f'--> Topic {tref} already exists. Skipped.') + continue + + # add topic to the graph + g.add_node(tref) + t = g.node[tref] # current topic node + + topicpath = path.join(prefix, tref) + + t['type'] = attr.get('type', 'topic') + t['name'] = attr.get('name', tref) + t['path'] = topicpath + t['file'] = attr.get('file', default_file) + t['shuffle'] = attr.get('shuffle', default_shuffle) + t['forgetting_factor'] = attr.get('forgetting_factor', default_forgetting_factor) + g.add_edges_from((d,tref) for d in attr.get('deps', [])) + + # load questions as list of dicts + questions = load_yaml(path.join(topicpath, t['file']), default=[]) + + # if questions are left undefined, include all. + # refs undefined in questions.yaml are set to topic:n + t['questions'] = attr.get('questions', + [q.setdefault('ref', f'{tref}:{i}') for i, q in enumerate(questions)]) + + # topic will generate a certain amount of questions + t['choose'] = min(attr.get('choose', default_choose), len(t['questions'])) + + # make questions factory (without repeating same question) FIXME move to somewhere else? + t['factory'] = {} + for q in questions: + if q['ref'] in t['questions']: + q['path'] = topicpath # fullpath added to each question + t['factory'][q['ref']] = QFactory(q) + + logger.info(f'{len(t["questions"]):6} {tref}') + qcount += len(t["questions"]) # count total questions + tcount += 1 + + logger.info(f'Total loaded: {tcount} topics, {qcount} questions') + -- libgit2 0.21.2