-
-
Notifications
You must be signed in to change notification settings - Fork 837
Expand file tree
/
Copy pathsage-preparse
More file actions
executable file
·302 lines (260 loc) · 10.4 KB
/
sage-preparse
File metadata and controls
executable file
·302 lines (260 loc) · 10.4 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
#!/usr/bin/env python3
"""
Preparse .sage files and save the result to .sage.py files.
AUTHOR:
-- William Stein (2005): first version
-- William Stein (2008): fix trac #2391 and document the code.
-- Dan Drake (2009): fix trac #5052
-- Dan Drake (2010-12-08): fix trac #10440
-- Johan S. R. Nielsen (2015-11-06): fix trac #17019
"""
import os
import re
import sys
from sage.misc.temporary_file import atomic_write
from sage.repl.preparse import preparse_file
# The spkg/bin/sage script passes the files to be preparsed as
# arguments (but remove sys.argv[0]).
files = sys.argv[1:]
# There must be at least 1 file or we display an error/usage message
# and exit
if not files:
print("""Usage: {} <file1.sage> <file2.sage>...
Creates files file1.sage.py, file2.sage.py ... that are the Sage
preparsed versions of file1.sage, file2.sage ...
If a non-autogenerated .sage.py file with the same name exists, you will
receive an error and the file will not be overwritten.""".format(sys.argv[0]))
sys.exit(1)
# The module-scope variable contains a list of all files we
# have seen while preparsing a given file. The point of this
# is that we want to avoid preparsing a file we have already
# seen, since then infinite loops would result from mutual
# recursive includes.
files_so_far = []
# This message is inserted in autogenerated files so that the reader
# will know, and so we know it is safe to overwrite them.
AUTOGEN_MSG = "# This file was *autogenerated* from the file "
# We use this regexp to parse lines with load or attach statements.
# Here's what it looks for:
#
# A (possibly empty) sequence of whitespace at the beginning of the
# line, saved as a group named 'lws';
#
# followed by
#
# the word "load" or "attach";
#
# followed by
#
# a nonempty sequence of whitespace;
#
# followed by
#
# whatever else is on the line, saved as a group named 'files'.
#
# We want to save the leading white space so that we can maintain
# correct indentation in the preparsed file.
load_or_attach = re.compile(r"^(?P<lws>\s*)(load|attach)\s+(?P<files>.*)$")
def do_preparse(f, files_before=[]):
"""
Preparse the file f and write the result out to a filename
with extension .sage.py.
INPUT:
- ``f`` -- string: the name of a file
- ``files_before`` -- list of strings of previous filenames loaded (to avoid circular loops)
OUTPUT: none (writes a file with extension .sage.py to disk)
"""
if f in files_so_far:
return
files_so_far.append(f)
if not os.path.exists(f):
print("{}: File '{}' is missing".format(sys.argv[0], f))
return
if f.endswith('.py'):
return
if not f.endswith('.sage'):
print("{}: Unknown file type {}".format(sys.argv[0], f))
sys.exit(1)
fname = f + ".py"
if os.path.exists(fname):
with open(fname) as fin:
if AUTOGEN_MSG not in fin.read():
print("Refusing to overwrite existing non-autogenerated file {!r}."
.format(os.path.abspath(fname)))
print("Please delete or move this file manually.")
sys.exit(1)
# TODO:
# I am commenting this "intelligence" out, since, e.g., if I change
# the preparser between versions this can cause problems. This
# is an optimization that definitely isn't needed at present, since
# preparsing is so fast.
# Idea: I could introduce version numbers, though....
#if os.path.exists(fname) and os.path.getmtime(fname) >= os.path.getmtime(f):
# return
# Finally open the file
with open(f) as fin:
F = fin.read()
# Check to see if a coding is specified in the .sage file. If it is,
# then we want to copy it over to the new file and not include it in
# the preprocessing. If both the first and second line have an
# encoding declaration, the second line's encoding will get used.
lines = F.splitlines()
coding = ''
for num, line in enumerate(lines[:2]):
if re.search(r"coding[:=]\s*([-\w.]+)", line):
coding = line + '\n'
F = '\n'.join(lines[:num] + lines[(num+1):])
# It is ** critical ** that all the preparser-stuff we put into
# the file are put after the module docstring, since
# otherwise the docstring will not be understood by Python.
i = find_position_right_after_module_docstring(F)
header, body = F[:i] , F[i:]
# Preparse the body
body = preparse_file(body)
# Check for "from __future__ import ..." statements. Those
# statements need to come at the top of the file (after the
# module-level docstring is okay), so we separate them from the
# body.
#
# Note: this will convert invalid Python to valid, because it will
# move all "from __future__ import ..." to the top of the file,
# even if they were not at the top originally.
future_imports, body = find_future_imports(body)
# Check for load/attach commands.
body = do_load_and_attach(body, f, files_before)
# The Sage library include line along with a autogen message
sage_incl = '%s%s\nfrom sage.all_cmdline import * # import sage library\n'%(AUTOGEN_MSG, f)
# Finally, write out the result. We use atomic_write to avoid
# race conditions (for example, the file will never be half written).
with atomic_write(fname) as f:
f.write(coding)
f.write(header)
f.write('\n')
f.write(future_imports)
f.write('\n')
f.write(sage_incl)
f.write('\n')
f.write(body)
f.write('\n')
def find_position_right_after_module_docstring(G):
"""
Return first position right after the module docstring of G, if it
has one. Otherwise return 0.
INPUT:
G -- a string
OUTPUT:
an integer -- the index into G so that G[i] is right after
the module docstring of G, if G has one.
"""
# The basic idea below is that we look at each line first ignoring
# all empty lines and commented out lines. Then we check to see
# if the next line is a docstring. If so, we find where that
# docstring ends.
v = G.splitlines()
i = 0
while i < len(v):
s = v[i].strip()
if s and s[0] != '#':
break
i += 1
if i >= len(v):
# No module docstring --- entire file is commented out
return 0
# Now v[i] contains the first line of the first statement in the file.
# Is it a docstring?
n = v[i].lstrip()
if not (n[0] in ['"',"'"] or n[0:2] in ['r"',"r'"]):
# not a docstring
return 0
# First line is the module docstring. Where does it end?
def pos_after_line(k):
return sum(len(v[j])+1 for j in range(k+1))
n = n.lstrip('r') # strip leading r if there is one
if n[:3] in ["'''", '"""']:
quotes = n[:3]
# possibly multiline
if quotes in n[3:]:
return pos_after_line(i)
j = i+1
while j < len(v) and quotes not in v[j]:
j += 1
return pos_after_line(j)
else:
# it must be a single line; so add up the lengths of all lines
# including this one and return that
return pos_after_line(i)
def find_future_imports(G):
"""
Parse a file G as a string, looking for "from __future__ import ...".
Return a tuple: (the import statements, the file G with those
statements removed)
INPUT:
G -- a string; the contents of a file
This can only handle "from __future__ import ..." statements which
are completely on a single line: nothing of the form ::
from __future__ import \
print_function
or ::
from __future__ import (print_function,
division)
This function will raise an error if it detects lines of these forms.
"""
import_statements = []
# "from __future__ import ..." should not be indented.
m = re.search("^(from __future__ import .*)$", G, re.MULTILINE)
while m:
statement = m.group(0)
# If the import statement ends in a line continuation marker
# or if it contains a left parenthesis but not a right one,
# then the statement is not complete, so raise an error. (This
# is not a perfect check and some bad cases may slip through,
# like two left parentheses and only one right parenthesis,
# but they should be rare.)
if (statement[-1] == '\\' or
(statement.find('(') > -1 and statement.find(')') == -1)):
raise NotImplementedError('the Sage preparser can only preparse "from __future__ import ..." statements which are on a single line')
import_statements.append(statement)
G = G[:m.start()] + G[m.end():]
m = re.search("^(from __future__ import .*)$", G, re.MULTILINE)
return ('\n'.join(import_statements), G)
def do_load_and_attach(G, file, files_before):
"""
Parse a file G and replace load and attach statements with the
corresponding execfile() statements.
INPUT:
G -- a string; a file loaded in from disk
file -- the name of the file that contains the non-preparsed
version of G.
files_before -- list of files seen so far (don't recurse into
infinite loop)
OUTPUT:
string -- result of parsing load/attach statements in G, i.e.
modified version of G with execfiles.
"""
s = ''
for t in G.split('\n'):
z = load_or_attach.match(t)
if z:
files = z.group('files').split()
lws = z.group('lws')
for w in files:
name = w.replace(',', '').replace('"', '').replace("'", "")
if name in files_before:
print("WARNING: not loading {} (in {}) again since would cause circular loop"
.format(name, file))
continue
if name.endswith('.sage'):
do_preparse(name, files_before + [file])
s += lws + "exec(compile(open({0}.py).read(), {0}.py, \
'exec'))\n".format(name)
elif name.endswith('.py'):
s += lws + "exec(compile(open({0}).read(), {0}, \
'exec'))\n".format(name)
else:
s += t + '\n'
return s
# Here we do the actual work. We iterate over ever
# file in the input args and create the corresponding
# output file.
for f in files:
do_preparse(f)