Blame view

aprendizations/tools.py 8.18 KB
592bd11e   Miguel Barão   - added tools.py
1

bd2ad2b7   Miguel Barão   - fix references ...
2
# python standard library
f9bea841   Miguel Barão   Use asyncio subpr...
3
import asyncio
592bd11e   Miguel Barão   - added tools.py
4
import logging
f9bea841   Miguel Barão   Use asyncio subpr...
5
from os import path
3aa7e128   Miguel Barão   - usable version!
6
import re
f9bea841   Miguel Barão   Use asyncio subpr...
7
import subprocess
a333dc72   Miguel Barão   Add type annotati...
8
from typing import Any, List
3aa7e128   Miguel Barão   - usable version!
9

3259fc7c   Miguel Barão   - Modified login ...
10
# third party libraries
3aa7e128   Miguel Barão   - usable version!
11
12
13
14
import mistune
from pygments import highlight
from pygments.lexers import get_lexer_by_name
from pygments.formatters import HtmlFormatter
f9bea841   Miguel Barão   Use asyncio subpr...
15
import yaml
592bd11e   Miguel Barão   - added tools.py
16

ae59481e   Miguel Barão   fix regression of...
17

592bd11e   Miguel Barão   - added tools.py
18
19
20
# setup logger for this module
logger = logging.getLogger(__name__)

3aa7e128   Miguel Barão   - usable version!
21

5476b345   Miguel Barão   - Ask yes/no to c...
22
# -------------------------------------------------------------------------
3aa7e128   Miguel Barão   - usable version!
23
# Markdown to HTML renderer with support for LaTeX equations
bff14879   Miguel Barão   - add support for...
24
25
26
27
28
29
# -------------------------------------------------------------------------


# -------------------------------------------------------------------------
# Block math:
#   $$x$$ or \begin{equation}x\end{equation}
5476b345   Miguel Barão   - Ask yes/no to c...
30
# -------------------------------------------------------------------------
3aa7e128   Miguel Barão   - usable version!
31
class MathBlockGrammar(mistune.BlockGrammar):
1af49693   Miguel Barão   - fix 'info' vs '...
32
    block_math = re.compile(r'^\$\$(.*?)\$\$', re.DOTALL)
8e601953   Miguel Barão   fix mostly flake8...
33
34
    latex_environment = re.compile(r'^\\begin\{([a-z]*\*?)\}(.*?)\\end\{\1\}',
                                   re.DOTALL)
3aa7e128   Miguel Barão   - usable version!
35
36
37


class MathBlockLexer(mistune.BlockLexer):
8e601953   Miguel Barão   fix mostly flake8...
38
39
    default_rules = ['block_math', 'latex_environment'] \
                    + mistune.BlockLexer.default_rules
3aa7e128   Miguel Barão   - usable version!
40
41
42
43
44
45
46

    def __init__(self, rules=None, **kwargs):
        if rules is None:
            rules = MathBlockGrammar()
        super().__init__(rules, **kwargs)

    def parse_block_math(self, m):
1af49693   Miguel Barão   - fix 'info' vs '...
47
        '''Parse a $$math$$ block'''
3aa7e128   Miguel Barão   - usable version!
48
49
50
51
52
53
        self.tokens.append({
            'type': 'block_math',
            'text': m.group(1)
        })

    def parse_latex_environment(self, m):
bff14879   Miguel Barão   - add support for...
54
        r'''Parse an environment \begin{name}text\end{name}'''
3aa7e128   Miguel Barão   - usable version!
55
56
57
58
59
60
61
        self.tokens.append({
            'type': 'latex_environment',
            'name': m.group(1),
            'text': m.group(2)
        })


bff14879   Miguel Barão   - add support for...
62
63
64
# -------------------------------------------------------------------------
# Inline math: $x$
# -------------------------------------------------------------------------
3aa7e128   Miguel Barão   - usable version!
65
class MathInlineGrammar(mistune.InlineGrammar):
1af49693   Miguel Barão   - fix 'info' vs '...
66
67
    math = re.compile(r'^\$(.+?)\$', re.DOTALL)
    block_math = re.compile(r'^\$\$(.+?)\$\$', re.DOTALL)
3aa7e128   Miguel Barão   - usable version!
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
    text = re.compile(r'^[\s\S]+?(?=[\\<!\[_*`~$]|https?://| {2,}\n|$)')


class MathInlineLexer(mistune.InlineLexer):
    default_rules = ['block_math', 'math'] + mistune.InlineLexer.default_rules

    def __init__(self, renderer, rules=None, **kwargs):
        if rules is None:
            rules = MathInlineGrammar()
        super().__init__(renderer, rules, **kwargs)

    def output_math(self, m):
        return self.renderer.inline_math(m.group(1))

    def output_block_math(self, m):
        return self.renderer.block_math(m.group(1))


class MarkdownWithMath(mistune.Markdown):
    def __init__(self, renderer, **kwargs):
        if 'inline' not in kwargs:
            kwargs['inline'] = MathInlineLexer
        if 'block' not in kwargs:
            kwargs['block'] = MathBlockLexer
        super().__init__(renderer, **kwargs)

    def output_block_math(self):
        return self.renderer.block_math(self.token['text'])

    def output_latex_environment(self):
8e601953   Miguel Barão   fix mostly flake8...
98
99
        return self.renderer.latex_environment(self.token['name'],
                                               self.token['text'])
3aa7e128   Miguel Barão   - usable version!
100
101
102
103
104


class HighlightRenderer(mistune.Renderer):
    def block_code(self, code, lang='text'):
        try:
6975951c   Miguel Barão   - fixed indentati...
105
            lexer = get_lexer_by_name(lang, stripall=False)
8e601953   Miguel Barão   fix mostly flake8...
106
        except Exception:
6975951c   Miguel Barão   - fixed indentati...
107
            lexer = get_lexer_by_name('text', stripall=False)
3aa7e128   Miguel Barão   - usable version!
108
109

        formatter = HtmlFormatter()
3aa7e128   Miguel Barão   - usable version!
110
111
        return highlight(code, lexer, formatter)

08a26dd3   Miguel Barão   - tables renderin...
112
    def table(self, header, body):
f4809e6f   Miguel Barão   - code cleaning q...
113
        return ('<table class="table table-sm"><thead class="thead-light">'
a6b50da0   Miguel Barão   fixes error where...
114
                f'{header}</thead><tbody>{body}</tbody></table>')
3aa7e128   Miguel Barão   - usable version!
115

6deb4a4f   Miguel Barão   - file support ad...
116
117
118
    def image(self, src, title, alt):
        alt = mistune.escape(alt, quote=True)
        title = mistune.escape(title or '', quote=True)
f4809e6f   Miguel Barão   - code cleaning q...
119
        return (f'<img src="/file/{src}" alt="{alt}" title="{title}"'
a6b50da0   Miguel Barão   fixes error where...
120
                f'class="img-fluid">')  # class="img-fluid mx-auto d-block"
3aa7e128   Miguel Barão   - usable version!
121
122
123

    # Pass math through unaltered - mathjax does the rendering in the browser
    def block_math(self, text):
bff14879   Miguel Barão   - add support for...
124
        return fr'\[ {text} \]'
3aa7e128   Miguel Barão   - usable version!
125
126
127
128
129

    def latex_environment(self, name, text):
        return fr'\begin{{{name}}} {text} \end{{{name}}}'

    def inline_math(self, text):
bff14879   Miguel Barão   - add support for...
130
        return fr'\( {text} \)'
3aa7e128   Miguel Barão   - usable version!
131
132


8e601953   Miguel Barão   fix mostly flake8...
133
134
135
# hard_wrap=True to insert <br> on newline
markdown = MarkdownWithMath(HighlightRenderer(escape=True))

3aa7e128   Miguel Barão   - usable version!
136

a333dc72   Miguel Barão   Add type annotati...
137
def md_to_html(text: str, strip_p_tag: bool = False) -> str:
1a7cc17b   Miguel Barão   more type annotat...
138
    md: str = markdown(text)
443a1eea   Miguel Barão   Update to latest ...
139
    if strip_p_tag and md.startswith('<p>') and md.endswith('</p>'):
bd2ad2b7   Miguel Barão   - fix references ...
140
141
142
        return md[3:-5]
    else:
        return md
3aa7e128   Miguel Barão   - usable version!
143

8e601953   Miguel Barão   fix mostly flake8...
144

592bd11e   Miguel Barão   - added tools.py
145
146
147
# ---------------------------------------------------------------------------
# load data from yaml file
# ---------------------------------------------------------------------------
a333dc72   Miguel Barão   Add type annotati...
148
def load_yaml(filename: str, default: Any = None) -> Any:
691ee097   Miguel Barão   fixed: now logs a...
149
    filename = path.expanduser(filename)
592bd11e   Miguel Barão   - added tools.py
150
    try:
691ee097   Miguel Barão   fixed: now logs a...
151
        f = open(filename, 'r', encoding='utf-8')
a6b50da0   Miguel Barão   fixes error where...
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
    except Exception as e:
        logger.error(e)
        if default is not None:
            return default
        else:
            raise

    with f:
        try:
            return yaml.safe_load(f)
        except yaml.YAMLError as e:
            logger.error(str(e).replace('\n', ' '))
            if default is not None:
                return default
            else:
                raise
592bd11e   Miguel Barão   - added tools.py
168

8e601953   Miguel Barão   fix mostly flake8...
169

592bd11e   Miguel Barão   - added tools.py
170
171
# ---------------------------------------------------------------------------
# Runs a script and returns its stdout parsed as yaml, or None on error.
ff655aef   Miguel Barão   - large rewrite o...
172
173
# The script is run in another process but this function blocks waiting
# for its termination.
592bd11e   Miguel Barão   - added tools.py
174
# ---------------------------------------------------------------------------
a333dc72   Miguel Barão   Add type annotati...
175
176
177
def run_script(script: str,
               args: List[str] = [],
               stdin: str = '',
1ab9d6ba   Miguel Barão   Async question ge...
178
               timeout: int = 2) -> Any:
a333dc72   Miguel Barão   Add type annotati...
179

3aa7e128   Miguel Barão   - usable version!
180
    script = path.expanduser(script)
592bd11e   Miguel Barão   - added tools.py
181
    try:
fa091c84   Miguel Barão   Allow generator t...
182
183
        cmd = [script] + [str(a) for a in args]
        p = subprocess.run(cmd,
8e601953   Miguel Barão   fix mostly flake8...
184
185
186
                           input=stdin,
                           stdout=subprocess.PIPE,
                           stderr=subprocess.STDOUT,
b2444bac   Miguel Barão   changed transitio...
187
                           text=True,  # same as universal_newlines=True
8e601953   Miguel Barão   fix mostly flake8...
188
189
                           timeout=timeout,
                           )
592bd11e   Miguel Barão   - added tools.py
190
    except FileNotFoundError:
8e601953   Miguel Barão   fix mostly flake8...
191
        logger.error(f'Can not execute script "{script}": not found.')
592bd11e   Miguel Barão   - added tools.py
192
    except PermissionError:
8e601953   Miguel Barão   fix mostly flake8...
193
        logger.error(f'Can not execute script "{script}": wrong permissions.')
3aa7e128   Miguel Barão   - usable version!
194
    except OSError:
8e601953   Miguel Barão   fix mostly flake8...
195
        logger.error(f'Can not execute script "{script}": unknown reason.')
592bd11e   Miguel Barão   - added tools.py
196
    except subprocess.TimeoutExpired:
8e601953   Miguel Barão   fix mostly flake8...
197
        logger.error(f'Timeout {timeout}s exceeded while running "{script}".')
fa091c84   Miguel Barão   Allow generator t...
198
199
    except Exception:
        logger.error(f'An Exception ocurred running {script}.')
592bd11e   Miguel Barão   - added tools.py
200
201
    else:
        if p.returncode != 0:
8e601953   Miguel Barão   fix mostly flake8...
202
            logger.error(f'Return code {p.returncode} running "{script}".')
592bd11e   Miguel Barão   - added tools.py
203
204
        else:
            try:
a51bcefd   Miguel Barão   Many changes:
205
                output = yaml.safe_load(p.stdout)
8e601953   Miguel Barão   fix mostly flake8...
206
            except Exception:
3aa7e128   Miguel Barão   - usable version!
207
                logger.error(f'Error parsing yaml output of "{script}"')
592bd11e   Miguel Barão   - added tools.py
208
209
            else:
                return output
f9bea841   Miguel Barão   Use asyncio subpr...
210
211
212
213
214
215
216
217


# ----------------------------------------------------------------------------
# Same as above, but asynchronous
# ----------------------------------------------------------------------------
async def run_script_async(script: str,
                           args: List[str] = [],
                           stdin: str = '',
1ab9d6ba   Miguel Barão   Async question ge...
218
                           timeout: int = 2) -> Any:
f9bea841   Miguel Barão   Use asyncio subpr...
219
220

    script = path.expanduser(script)
1ab9d6ba   Miguel Barão   Async question ge...
221
    args = [str(a) for a in args]
f9bea841   Miguel Barão   Use asyncio subpr...
222

1ab9d6ba   Miguel Barão   Async question ge...
223
224
    p = await asyncio.create_subprocess_exec(
        script, *args,
f9bea841   Miguel Barão   Use asyncio subpr...
225
226
227
228
229
230
        stdin=asyncio.subprocess.PIPE,
        stdout=asyncio.subprocess.PIPE,
        stderr=asyncio.subprocess.DEVNULL,
        )

    try:
443a1eea   Miguel Barão   Update to latest ...
231
        stdout, _ = await asyncio.wait_for(
f9bea841   Miguel Barão   Use asyncio subpr...
232
233
234
235
236
237
238
239
240
241
242
            p.communicate(input=stdin.encode('utf-8')),
            timeout=timeout
            )
    except asyncio.TimeoutError:
        logger.warning(f'Timeout {timeout}s running script "{script}".')
        return

    if p.returncode != 0:
        logger.error(f'Return code {p.returncode} running "{script}".')
    else:
        try:
1ab9d6ba   Miguel Barão   Async question ge...
243
            output = yaml.safe_load(stdout.decode('utf-8', 'ignore'))
f9bea841   Miguel Barão   Use asyncio subpr...
244
245
246
247
        except Exception:
            logger.error(f'Error parsing yaml output of "{script}"')
        else:
            return output