Skip to content

Commit a14f856

Browse files
youknowoneyouknowone
committed
fmt
1 parent bdfe2cf commit a14f856

File tree

2 files changed

+48
-40
lines changed

2 files changed

+48
-40
lines changed

‎crates/stdlib/src/_tokenize.rs‎

Lines changed: 46 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -13,14 +13,14 @@ mod _tokenize {
1313
types::{Constructor, IterNext, Iterable, SelfIter},
1414
},
1515
};
16+
use core::fmt;
1617
use ruff_python_ast::PySourceType;
1718
use ruff_python_ast::token::{Token, TokenKind};
1819
use ruff_python_parser::{
1920
LexicalErrorType, ParseError, ParseErrorType, parse_unchecked_source,
2021
};
2122
use ruff_source_file::{LineIndex, LineRanges};
2223
use ruff_text_size::{Ranged, TextSize};
23-
use core::fmt;
2424

2525
const TOKEN_ENDMARKER: u8 = 0;
2626
const TOKEN_DEDENT: u8 = 6;
@@ -114,8 +114,7 @@ mod _tokenize {
114114
let line = zelf.readline(vm)?;
115115
if line.is_empty() {
116116
let accumulated = core::mem::take(source);
117-
let parsed =
118-
parse_unchecked_source(&accumulated, PySourceType::Python);
117+
let parsed = parse_unchecked_source(&accumulated, PySourceType::Python);
119118
let tokens: Vec<Token> = parsed.tokens().iter().copied().collect();
120119
let errors: Vec<ParseError> = parsed.errors().to_vec();
121120
let line_index = LineIndex::from_source_text(&accumulated);
@@ -135,8 +134,7 @@ mod _tokenize {
135134
}
136135
}
137136
TokenizerPhase::Yielding { .. } => {
138-
let result =
139-
emit_next_token(&mut state, zelf.extra_tokens, vm)?;
137+
let result = emit_next_token(&mut state, zelf.extra_tokens, vm)?;
140138
*zelf.state.write() = state;
141139
return Ok(result);
142140
}
@@ -191,10 +189,16 @@ mod _tokenize {
191189
.take(sl.saturating_sub(1))
192190
.map(|l| l.len() + 1)
193191
.sum();
194-
let full_line =
195-
source.full_line_str(TextSize::from(offset.min(source.len()) as u32));
192+
let full_line = source.full_line_str(TextSize::from(offset.min(source.len()) as u32));
196193
return Ok(PyIterReturn::Return(make_token_tuple(
197-
vm, tok_type, &tok_str, sl, sc as isize, el, ec as isize, full_line,
194+
vm,
195+
tok_type,
196+
&tok_str,
197+
sl,
198+
sc as isize,
199+
el,
200+
ec as isize,
201+
full_line,
198202
)));
199203
}
200204

@@ -218,9 +222,7 @@ mod _tokenize {
218222
) {
219223
continue;
220224
}
221-
if err.location.start() <= range.start()
222-
&& range.start() < err.location.end()
223-
{
225+
if err.location.start() <= range.start() && range.start() < err.location.end() {
224226
return Err(raise_indentation_error(vm, err, source, line_index));
225227
}
226228
}
@@ -230,15 +232,12 @@ mod _tokenize {
230232
continue;
231233
}
232234

233-
if !extra_tokens
234-
&& matches!(kind, TokenKind::Comment | TokenKind::NonLogicalNewline)
235-
{
235+
if !extra_tokens && matches!(kind, TokenKind::Comment | TokenKind::NonLogicalNewline) {
236236
continue;
237237
}
238238

239239
let raw_type = token_kind_value(kind);
240-
let token_type = if extra_tokens && raw_type > TOKEN_DEDENT && raw_type < TOKEN_OP
241-
{
240+
let token_type = if extra_tokens && raw_type > TOKEN_DEDENT && raw_type < TOKEN_OP {
242241
TOKEN_OP
243242
} else {
244243
raw_type
@@ -294,15 +293,21 @@ mod _tokenize {
294293
&& (token_str.contains("{{") || token_str.contains("}}"))
295294
{
296295
let mut parts =
297-
split_fstring_middle(token_str, token_type, start_line, start_col)
298-
.into_iter();
296+
split_fstring_middle(token_str, token_type, start_line, start_col).into_iter();
299297
let (tt, ts, sl, sc, el, ec) = parts.next().unwrap();
300298
let rest: Vec<_> = parts.collect();
301299
for p in rest.into_iter().rev() {
302300
pending_fstring_parts.push(p);
303301
}
304302
return Ok(PyIterReturn::Return(make_token_tuple(
305-
vm, tt, &ts, sl, sc as isize, el, ec as isize, line_str,
303+
vm,
304+
tt,
305+
&ts,
306+
sl,
307+
sc as isize,
308+
el,
309+
ec as isize,
310+
line_str,
306311
)));
307312
}
308313

@@ -315,17 +320,19 @@ mod _tokenize {
315320
.is_some_and(|t| t.kind() == TokenKind::Rbrace)
316321
{
317322
let mid_type = find_fstring_middle_type(tokens, *index);
318-
*pending_empty_fstring_middle = Some((
319-
mid_type,
320-
end_line,
321-
end_col,
322-
line_str.to_string(),
323-
));
323+
*pending_empty_fstring_middle =
324+
Some((mid_type, end_line, end_col, line_str.to_string()));
324325
}
325326

326327
return Ok(PyIterReturn::Return(make_token_tuple(
327-
vm, token_type, token_str, start_line, start_col as isize, end_line,
328-
end_col as isize, line_str,
328+
vm,
329+
token_type,
330+
token_str,
331+
start_line,
332+
start_col as isize,
333+
end_line,
334+
end_col as isize,
335+
line_str,
329336
)));
330337
}
331338

@@ -380,14 +387,20 @@ mod _tokenize {
380387
let (em_line, em_col, em_line_str): (usize, isize, &str) = if extra_tokens {
381388
(last_line + 1, 0, "")
382389
} else {
383-
let last_line_text = source.full_line_str(TextSize::from(
384-
source.len().saturating_sub(1) as u32,
385-
));
390+
let last_line_text =
391+
source.full_line_str(TextSize::from(source.len().saturating_sub(1) as u32));
386392
(last_line, -1, last_line_text)
387393
};
388394

389395
let result = make_token_tuple(
390-
vm, TOKEN_ENDMARKER, "", em_line, em_col, em_line, em_col, em_line_str,
396+
vm,
397+
TOKEN_ENDMARKER,
398+
"",
399+
em_line,
400+
em_col,
401+
em_line,
402+
em_col,
403+
em_line_str,
391404
);
392405
state.phase = TokenizerPhase::Done;
393406
Ok(PyIterReturn::Return(result))
@@ -448,10 +461,7 @@ mod _tokenize {
448461
lineno: usize,
449462
offset: usize,
450463
) -> rustpython_vm::builtins::PyBaseExceptionRef {
451-
let exc = vm.new_exception_msg(
452-
vm.ctx.exceptions.syntax_error.to_owned(),
453-
msg.into(),
454-
);
464+
let exc = vm.new_exception_msg(vm.ctx.exceptions.syntax_error.to_owned(), msg.into());
455465
let obj = exc.as_object();
456466
let _ = obj.set_attr("msg", vm.ctx.new_str(msg), vm);
457467
let _ = obj.set_attr("lineno", vm.ctx.new_int(lineno), vm);
@@ -739,9 +749,7 @@ mod _tokenize {
739749
TokenKind::TStringStart => 62,
740750
TokenKind::TStringMiddle => 63,
741751
TokenKind::TStringEnd => 64,
742-
TokenKind::IpyEscapeCommand
743-
| TokenKind::Question
744-
| TokenKind::Unknown => 67, // ERRORTOKEN
752+
TokenKind::IpyEscapeCommand | TokenKind::Question | TokenKind::Unknown => 67, // ERRORTOKEN
745753
}
746754
}
747755
}

‎crates/stdlib/src/lib.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,8 @@ mod json;
4141
mod locale;
4242

4343
mod _opcode;
44+
#[path = "_tokenize.rs"]
45+
mod _tokenize;
4446
mod math;
4547
#[cfg(all(feature = "host_env", any(unix, windows)))]
4648
mod mmap;
@@ -49,8 +51,6 @@ mod pystruct;
4951
mod random;
5052
mod statistics;
5153
mod suggestions;
52-
#[path = "_tokenize.rs"]
53-
mod _tokenize;
5454
// TODO: maybe make this an extension module, if we ever get those
5555
// mod re;
5656
#[cfg(all(feature = "host_env", not(target_arch = "wasm32")))]

0 commit comments

Comments
 (0)