592bd11e
Miguel Barão
- added tools.py
|
1
|
|
bd2ad2b7
Miguel Barão
- fix references ...
|
2
|
# python standard library
|
f9bea841
Miguel Barão
Use asyncio subpr...
|
3
|
import asyncio
|
592bd11e
Miguel Barão
- added tools.py
|
4
|
import logging
|
f9bea841
Miguel Barão
Use asyncio subpr...
|
5
|
from os import path
|
3aa7e128
Miguel Barão
- usable version!
|
6
|
import re
|
f9bea841
Miguel Barão
Use asyncio subpr...
|
7
|
import subprocess
|
a333dc72
Miguel Barão
Add type annotati...
|
8
|
from typing import Any, List
|
3aa7e128
Miguel Barão
- usable version!
|
9
|
|
3259fc7c
Miguel Barão
- Modified login ...
|
10
|
# third party libraries
|
3aa7e128
Miguel Barão
- usable version!
|
11
12
13
14
|
import mistune
from pygments import highlight
from pygments.lexers import get_lexer_by_name
from pygments.formatters import HtmlFormatter
|
f9bea841
Miguel Barão
Use asyncio subpr...
|
15
|
import yaml
|
592bd11e
Miguel Barão
- added tools.py
|
16
|
|
ae59481e
Miguel Barão
fix regression of...
|
17
|
|
592bd11e
Miguel Barão
- added tools.py
|
18
19
20
|
# setup logger for this module
logger = logging.getLogger(__name__)
|
3aa7e128
Miguel Barão
- usable version!
|
21
|
|
5476b345
Miguel Barão
- Ask yes/no to c...
|
22
|
# -------------------------------------------------------------------------
|
3aa7e128
Miguel Barão
- usable version!
|
23
|
# Markdown to HTML renderer with support for LaTeX equations
|
bff14879
Miguel Barão
- add support for...
|
24
25
26
27
28
29
|
# -------------------------------------------------------------------------
# -------------------------------------------------------------------------
# Block math:
# $$x$$ or \begin{equation}x\end{equation}
|
5476b345
Miguel Barão
- Ask yes/no to c...
|
30
|
# -------------------------------------------------------------------------
|
3aa7e128
Miguel Barão
- usable version!
|
31
|
class MathBlockGrammar(mistune.BlockGrammar):
|
1af49693
Miguel Barão
- fix 'info' vs '...
|
32
|
block_math = re.compile(r'^\$\$(.*?)\$\$', re.DOTALL)
|
8e601953
Miguel Barão
fix mostly flake8...
|
33
34
|
latex_environment = re.compile(r'^\\begin\{([a-z]*\*?)\}(.*?)\\end\{\1\}',
re.DOTALL)
|
3aa7e128
Miguel Barão
- usable version!
|
35
36
37
|
class MathBlockLexer(mistune.BlockLexer):
|
8e601953
Miguel Barão
fix mostly flake8...
|
38
39
|
default_rules = ['block_math', 'latex_environment'] \
+ mistune.BlockLexer.default_rules
|
3aa7e128
Miguel Barão
- usable version!
|
40
41
42
43
44
45
46
|
def __init__(self, rules=None, **kwargs):
if rules is None:
rules = MathBlockGrammar()
super().__init__(rules, **kwargs)
def parse_block_math(self, m):
|
1af49693
Miguel Barão
- fix 'info' vs '...
|
47
|
'''Parse a $$math$$ block'''
|
3aa7e128
Miguel Barão
- usable version!
|
48
49
50
51
52
53
|
self.tokens.append({
'type': 'block_math',
'text': m.group(1)
})
def parse_latex_environment(self, m):
|
bff14879
Miguel Barão
- add support for...
|
54
|
r'''Parse an environment \begin{name}text\end{name}'''
|
3aa7e128
Miguel Barão
- usable version!
|
55
56
57
58
59
60
61
|
self.tokens.append({
'type': 'latex_environment',
'name': m.group(1),
'text': m.group(2)
})
|
bff14879
Miguel Barão
- add support for...
|
62
63
64
|
# -------------------------------------------------------------------------
# Inline math: $x$
# -------------------------------------------------------------------------
|
3aa7e128
Miguel Barão
- usable version!
|
65
|
class MathInlineGrammar(mistune.InlineGrammar):
|
1af49693
Miguel Barão
- fix 'info' vs '...
|
66
67
|
math = re.compile(r'^\$(.+?)\$', re.DOTALL)
block_math = re.compile(r'^\$\$(.+?)\$\$', re.DOTALL)
|
3aa7e128
Miguel Barão
- usable version!
|
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
|
text = re.compile(r'^[\s\S]+?(?=[\\<!\[_*`~$]|https?://| {2,}\n|$)')
class MathInlineLexer(mistune.InlineLexer):
default_rules = ['block_math', 'math'] + mistune.InlineLexer.default_rules
def __init__(self, renderer, rules=None, **kwargs):
if rules is None:
rules = MathInlineGrammar()
super().__init__(renderer, rules, **kwargs)
def output_math(self, m):
return self.renderer.inline_math(m.group(1))
def output_block_math(self, m):
return self.renderer.block_math(m.group(1))
class MarkdownWithMath(mistune.Markdown):
def __init__(self, renderer, **kwargs):
if 'inline' not in kwargs:
kwargs['inline'] = MathInlineLexer
if 'block' not in kwargs:
kwargs['block'] = MathBlockLexer
super().__init__(renderer, **kwargs)
def output_block_math(self):
return self.renderer.block_math(self.token['text'])
def output_latex_environment(self):
|
8e601953
Miguel Barão
fix mostly flake8...
|
98
99
|
return self.renderer.latex_environment(self.token['name'],
self.token['text'])
|
3aa7e128
Miguel Barão
- usable version!
|
100
101
102
103
104
|
class HighlightRenderer(mistune.Renderer):
def block_code(self, code, lang='text'):
try:
|
6975951c
Miguel Barão
- fixed indentati...
|
105
|
lexer = get_lexer_by_name(lang, stripall=False)
|
8e601953
Miguel Barão
fix mostly flake8...
|
106
|
except Exception:
|
6975951c
Miguel Barão
- fixed indentati...
|
107
|
lexer = get_lexer_by_name('text', stripall=False)
|
3aa7e128
Miguel Barão
- usable version!
|
108
109
|
formatter = HtmlFormatter()
|
3aa7e128
Miguel Barão
- usable version!
|
110
111
|
return highlight(code, lexer, formatter)
|
08a26dd3
Miguel Barão
- tables renderin...
|
112
|
def table(self, header, body):
|
f4809e6f
Miguel Barão
- code cleaning q...
|
113
|
return ('<table class="table table-sm"><thead class="thead-light">'
|
a6b50da0
Miguel Barão
fixes error where...
|
114
|
f'{header}</thead><tbody>{body}</tbody></table>')
|
3aa7e128
Miguel Barão
- usable version!
|
115
|
|
6deb4a4f
Miguel Barão
- file support ad...
|
116
117
118
|
def image(self, src, title, alt):
alt = mistune.escape(alt, quote=True)
title = mistune.escape(title or '', quote=True)
|
f4809e6f
Miguel Barão
- code cleaning q...
|
119
|
return (f'<img src="/file/{src}" alt="{alt}" title="{title}"'
|
a6b50da0
Miguel Barão
fixes error where...
|
120
|
f'class="img-fluid">') # class="img-fluid mx-auto d-block"
|
3aa7e128
Miguel Barão
- usable version!
|
121
122
123
|
# Pass math through unaltered - mathjax does the rendering in the browser
def block_math(self, text):
|
bff14879
Miguel Barão
- add support for...
|
124
|
return fr'\[ {text} \]'
|
3aa7e128
Miguel Barão
- usable version!
|
125
126
127
128
129
|
def latex_environment(self, name, text):
return fr'\begin{{{name}}} {text} \end{{{name}}}'
def inline_math(self, text):
|
bff14879
Miguel Barão
- add support for...
|
130
|
return fr'\( {text} \)'
|
3aa7e128
Miguel Barão
- usable version!
|
131
132
|
|
8e601953
Miguel Barão
fix mostly flake8...
|
133
134
135
|
# hard_wrap=True to insert <br> on newline
markdown = MarkdownWithMath(HighlightRenderer(escape=True))
|
3aa7e128
Miguel Barão
- usable version!
|
136
|
|
a333dc72
Miguel Barão
Add type annotati...
|
137
|
def md_to_html(text: str, strip_p_tag: bool = False) -> str:
|
1a7cc17b
Miguel Barão
more type annotat...
|
138
|
md: str = markdown(text)
|
443a1eea
Miguel Barão
Update to latest ...
|
139
|
if strip_p_tag and md.startswith('<p>') and md.endswith('</p>'):
|
bd2ad2b7
Miguel Barão
- fix references ...
|
140
141
142
|
return md[3:-5]
else:
return md
|
3aa7e128
Miguel Barão
- usable version!
|
143
|
|
8e601953
Miguel Barão
fix mostly flake8...
|
144
|
|
592bd11e
Miguel Barão
- added tools.py
|
145
146
147
|
# ---------------------------------------------------------------------------
# load data from yaml file
# ---------------------------------------------------------------------------
|
a333dc72
Miguel Barão
Add type annotati...
|
148
|
def load_yaml(filename: str, default: Any = None) -> Any:
|
691ee097
Miguel Barão
fixed: now logs a...
|
149
|
filename = path.expanduser(filename)
|
592bd11e
Miguel Barão
- added tools.py
|
150
|
try:
|
691ee097
Miguel Barão
fixed: now logs a...
|
151
|
f = open(filename, 'r', encoding='utf-8')
|
a6b50da0
Miguel Barão
fixes error where...
|
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
|
except Exception as e:
logger.error(e)
if default is not None:
return default
else:
raise
with f:
try:
return yaml.safe_load(f)
except yaml.YAMLError as e:
logger.error(str(e).replace('\n', ' '))
if default is not None:
return default
else:
raise
|
592bd11e
Miguel Barão
- added tools.py
|
168
|
|
8e601953
Miguel Barão
fix mostly flake8...
|
169
|
|
592bd11e
Miguel Barão
- added tools.py
|
170
171
|
# ---------------------------------------------------------------------------
# Runs a script and returns its stdout parsed as yaml, or None on error.
|
ff655aef
Miguel Barão
- large rewrite o...
|
172
173
|
# The script is run in another process but this function blocks waiting
# for its termination.
|
592bd11e
Miguel Barão
- added tools.py
|
174
|
# ---------------------------------------------------------------------------
|
a333dc72
Miguel Barão
Add type annotati...
|
175
176
177
|
def run_script(script: str,
args: List[str] = [],
stdin: str = '',
|
1ab9d6ba
Miguel Barão
Async question ge...
|
178
|
timeout: int = 2) -> Any:
|
a333dc72
Miguel Barão
Add type annotati...
|
179
|
|
3aa7e128
Miguel Barão
- usable version!
|
180
|
script = path.expanduser(script)
|
592bd11e
Miguel Barão
- added tools.py
|
181
|
try:
|
fa091c84
Miguel Barão
Allow generator t...
|
182
183
|
cmd = [script] + [str(a) for a in args]
p = subprocess.run(cmd,
|
8e601953
Miguel Barão
fix mostly flake8...
|
184
185
186
|
input=stdin,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
|
b2444bac
Miguel Barão
changed transitio...
|
187
|
text=True, # same as universal_newlines=True
|
8e601953
Miguel Barão
fix mostly flake8...
|
188
189
|
timeout=timeout,
)
|
592bd11e
Miguel Barão
- added tools.py
|
190
|
except FileNotFoundError:
|
8e601953
Miguel Barão
fix mostly flake8...
|
191
|
logger.error(f'Can not execute script "{script}": not found.')
|
592bd11e
Miguel Barão
- added tools.py
|
192
|
except PermissionError:
|
8e601953
Miguel Barão
fix mostly flake8...
|
193
|
logger.error(f'Can not execute script "{script}": wrong permissions.')
|
3aa7e128
Miguel Barão
- usable version!
|
194
|
except OSError:
|
8e601953
Miguel Barão
fix mostly flake8...
|
195
|
logger.error(f'Can not execute script "{script}": unknown reason.')
|
592bd11e
Miguel Barão
- added tools.py
|
196
|
except subprocess.TimeoutExpired:
|
8e601953
Miguel Barão
fix mostly flake8...
|
197
|
logger.error(f'Timeout {timeout}s exceeded while running "{script}".')
|
fa091c84
Miguel Barão
Allow generator t...
|
198
199
|
except Exception:
logger.error(f'An Exception ocurred running {script}.')
|
592bd11e
Miguel Barão
- added tools.py
|
200
201
|
else:
if p.returncode != 0:
|
8e601953
Miguel Barão
fix mostly flake8...
|
202
|
logger.error(f'Return code {p.returncode} running "{script}".')
|
592bd11e
Miguel Barão
- added tools.py
|
203
204
|
else:
try:
|
a51bcefd
Miguel Barão
Many changes:
|
205
|
output = yaml.safe_load(p.stdout)
|
8e601953
Miguel Barão
fix mostly flake8...
|
206
|
except Exception:
|
3aa7e128
Miguel Barão
- usable version!
|
207
|
logger.error(f'Error parsing yaml output of "{script}"')
|
592bd11e
Miguel Barão
- added tools.py
|
208
209
|
else:
return output
|
f9bea841
Miguel Barão
Use asyncio subpr...
|
210
211
212
213
214
215
216
217
|
# ----------------------------------------------------------------------------
# Same as above, but asynchronous
# ----------------------------------------------------------------------------
async def run_script_async(script: str,
args: List[str] = [],
stdin: str = '',
|
1ab9d6ba
Miguel Barão
Async question ge...
|
218
|
timeout: int = 2) -> Any:
|
f9bea841
Miguel Barão
Use asyncio subpr...
|
219
220
|
script = path.expanduser(script)
|
1ab9d6ba
Miguel Barão
Async question ge...
|
221
|
args = [str(a) for a in args]
|
f9bea841
Miguel Barão
Use asyncio subpr...
|
222
|
|
1ab9d6ba
Miguel Barão
Async question ge...
|
223
224
|
p = await asyncio.create_subprocess_exec(
script, *args,
|
f9bea841
Miguel Barão
Use asyncio subpr...
|
225
226
227
228
229
230
|
stdin=asyncio.subprocess.PIPE,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.DEVNULL,
)
try:
|
443a1eea
Miguel Barão
Update to latest ...
|
231
|
stdout, _ = await asyncio.wait_for(
|
f9bea841
Miguel Barão
Use asyncio subpr...
|
232
233
234
235
236
237
238
239
240
241
242
|
p.communicate(input=stdin.encode('utf-8')),
timeout=timeout
)
except asyncio.TimeoutError:
logger.warning(f'Timeout {timeout}s running script "{script}".')
return
if p.returncode != 0:
logger.error(f'Return code {p.returncode} running "{script}".')
else:
try:
|
1ab9d6ba
Miguel Barão
Async question ge...
|
243
|
output = yaml.safe_load(stdout.decode('utf-8', 'ignore'))
|
f9bea841
Miguel Barão
Use asyncio subpr...
|
244
245
246
247
|
except Exception:
logger.error(f'Error parsing yaml output of "{script}"')
else:
return output
|