Skip to content

Commit e725fd0

Browse files
youknowoneyouknowone
committed
numerous refactoring
- Split parser core and compiler core. Fix RustPython#14 - AST int type to `u32` - Updated asdl_rs.py and update_asdl.sh fix RustPython#6 - Use `ruff_python_ast::SourceLocation` for Python source location. Deleted our own Location. - Renamed ast::Located to ast::Attributed to distinguish terms for TextSize and SourceLocation - `ast::<Node>`s for TextSize located ast. `ast::located::<Node>` for Python source located ast. - And also strictly renaming `located` to refer only python location related interfaces. - `SourceLocator` to convert locations. - New `source-code` features of to disable python locations when unnecessary. - Also including fully merging astral-sh/RustPython#4 closes RustPython#9
1 parent 09a6afd commit e725fd0

29 files changed

+9699
-11962
lines changed

‎Cargo.toml

+5-4
Original file line numberDiff line numberDiff line change
@@ -16,11 +16,12 @@ members = [
1616
]
1717

1818
[workspace.dependencies]
19+
rustpython-ast = { path = "ast", version = "0.2.0" }
20+
rustpython-parser-core = { path = "core", version = "0.2.0" }
21+
rustpython-literal = { path = "literal", version = "0.2.0" }
22+
1923
ahash = "0.7.6"
2024
anyhow = "1.0.45"
21-
ascii = "1.0"
22-
bitflags = "1.3.2"
23-
bstr = "0.2.17"
2425
cfg-if = "1.0"
2526
insta = "1.14.0"
2627
itertools = "0.10.3"
@@ -32,7 +33,7 @@ rand = "0.8.5"
3233
serde = "1.0"
3334
static_assertions = "1.1"
3435
unicode_names2 = { version = "0.6.0", git = "https://github.com/youknowone/unicode_names2.git", rev = "4ce16aa85cbcdd9cc830410f1a72ef9a235f2fde" }
35-
ruff_python_ast = { git = "https://github.com/youknowone/ruff.git", rev = "583df5c1fa43b2732896219f8ab425116c140c80" }
36+
ruff_python_ast = { git = "https://github.com/youknowone/ruff.git", rev = "088958e8fda2f74f1ebf315c75db13c232409b13" }
3637
# ruff_python_ast = { path = "../ruff/crates/ruff_python_ast" }
3738

3839
[profile.dev.package."*"]

‎ast/Cargo.toml

+4-4
Original file line numberDiff line numberDiff line change
@@ -8,14 +8,14 @@ repository = "https://github.com/RustPython/RustPython"
88
license = "MIT"
99

1010
[features]
11-
default = ["constant-optimization", "fold", "location"]
11+
default = ["constant-optimization", "fold", "source-code"]
1212
constant-optimization = ["fold"]
13-
location = []
13+
source-code = ["fold"]
1414
fold = []
1515
unparse = ["rustpython-literal"]
1616

1717
[dependencies]
18-
rustpython-compiler-core = { path = "../core", version = "0.2.0" }
19-
rustpython-literal = { path = "../literal", version = "0.2.0", optional = true }
18+
rustpython-parser-core = { workspace = true }
19+
rustpython-literal = { workspace = true, optional = true }
2020

2121
num-bigint = { workspace = true }

‎ast/asdl_rs.py

+23-9
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@
88
from argparse import ArgumentParser
99
from pathlib import Path
1010
from typing import Optional, Dict
11-
from attr import dataclass
1211

1312
import asdl
1413

@@ -18,7 +17,7 @@
1817
builtin_type_mapping = {
1918
"identifier": "Ident",
2019
"string": "String",
21-
"int": "usize",
20+
"int": "u32",
2221
"constant": "Constant",
2322
}
2423
assert builtin_type_mapping.keys() == asdl.builtin_types
@@ -391,7 +390,18 @@ def visitModule(self, mod, depth):
391390
depth + 1,
392391
)
393392
self.emit(
394-
"fn map_located<T>(&mut self, located: Attributed<T, U>) -> Result<Attributed<T, Self::TargetU>, Self::Error> { let custom = self.map_user(located.custom)?; Ok(Attributed { range: located.range, custom, node: located.node }) }",
393+
"""
394+
fn map_located<T>(&mut self, located: Attributed<T, U>) -> Result<Attributed<T, Self::TargetU>, Self::Error> {
395+
let custom = self.map_user(located.custom)?;
396+
Ok(Attributed { range: located.range, custom, node: located.node })
397+
}""",
398+
depth + 1,
399+
)
400+
self.emit(
401+
"""
402+
fn fold<X: Foldable<U, Self::TargetU>>(&mut self, node: X) -> Result<X::Mapped, Self::Error> {
403+
node.fold(self)
404+
}""",
395405
depth + 1,
396406
)
397407
for dfn in mod.dfns:
@@ -761,11 +771,15 @@ def gen_construction(self, cons_path, cons, name, depth):
761771
def extract_location(self, typename, depth):
762772
row = self.decode_field(asdl.Field("int", "lineno"), typename)
763773
column = self.decode_field(asdl.Field("int", "col_offset"), typename)
764-
self.emit(f"""let _location = {{
765-
let row = try_location_field({row}, _vm)?;
766-
let column = try_location_field({column}, _vm)?;
767-
SourceLocation {{ row, column }}
768-
}};""", depth)
774+
self.emit(
775+
f"""
776+
let _location = {{
777+
let row = {row};
778+
let column = {column};
779+
try_location(row, column, _vm)?
780+
}};""",
781+
depth,
782+
)
769783

770784
def decode_field(self, field, typename):
771785
name = json.dumps(field.name)
@@ -805,7 +819,7 @@ def write_located_def(typeinfo, f):
805819
f.write(
806820
textwrap.dedent(
807821
"""
808-
use crate::location::SourceRange;
822+
use rustpython_parser_core::source_code::SourceRange;
809823
810824
pub type Located<T> = super::generic::Attributed<T, SourceRange>;
811825
"""

‎ast/src/attributed.rs

+4-2
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
1-
use crate::location::{SourceLocation, SourceRange};
2-
use rustpython_compiler_core::text_size::{TextRange, TextSize};
1+
use rustpython_parser_core::{
2+
source_code::{SourceLocation, SourceRange},
3+
text_size::{TextRange, TextSize},
4+
};
35

46
#[derive(Clone, Debug, PartialEq)]
57
pub struct Attributed<T, U = ()> {

‎ast/src/constant.rs

+1-2
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
use num_bigint::BigInt;
2-
pub use rustpython_compiler_core::ConversionFlag;
32

43
#[derive(Clone, Debug, PartialEq)]
54
pub enum Constant {
@@ -137,7 +136,7 @@ impl<U> crate::fold::Fold<U> for ConstantOptimizer {
137136
#[cfg(test)]
138137
mod tests {
139138
use super::*;
140-
use rustpython_compiler_core::text_size::TextRange;
139+
use rustpython_parser_core::text_size::TextRange;
141140

142141
#[cfg(feature = "constant-optimization")]
143142
#[test]

‎ast/src/fold_helpers.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -62,4 +62,4 @@ macro_rules! simple_fold {
6262
};
6363
}
6464

65-
simple_fold!(usize, String, bool, constant::Constant);
65+
simple_fold!(u32, String, bool, constant::Constant);

‎ast/src/gen/generic.rs

+13-5
Original file line numberDiff line numberDiff line change
@@ -158,7 +158,7 @@ pub struct StmtAnnAssign<U = ()> {
158158
pub target: Box<Expr<U>>,
159159
pub annotation: Box<Expr<U>>,
160160
pub value: Option<Box<Expr<U>>>,
161-
pub simple: usize,
161+
pub simple: u32,
162162
}
163163

164164
impl<U> From<StmtAnnAssign<U>> for StmtKind<U> {
@@ -328,7 +328,7 @@ impl<U> From<StmtImport<U>> for StmtKind<U> {
328328
pub struct StmtImportFrom<U = ()> {
329329
pub module: Option<Ident>,
330330
pub names: Vec<Alias<U>>,
331-
pub level: Option<usize>,
331+
pub level: Option<u32>,
332332
}
333333

334334
impl<U> From<StmtImportFrom<U>> for StmtKind<U> {
@@ -610,7 +610,7 @@ impl<U> From<ExprCall<U>> for ExprKind<U> {
610610
#[derive(Clone, Debug, PartialEq)]
611611
pub struct ExprFormattedValue<U = ()> {
612612
pub value: Box<Expr<U>>,
613-
pub conversion: usize,
613+
pub conversion: u32,
614614
pub format_spec: Option<Box<Expr<U>>>,
615615
}
616616

@@ -819,7 +819,7 @@ pub struct Comprehension<U = ()> {
819819
pub target: Expr<U>,
820820
pub iter: Expr<U>,
821821
pub ifs: Vec<Expr<U>>,
822-
pub is_async: usize,
822+
pub is_async: u32,
823823
}
824824

825825
#[derive(Clone, Debug, PartialEq)]
@@ -996,7 +996,7 @@ pub type Pattern<U = ()> = Attributed<PatternKind<U>, U>;
996996

997997
#[derive(Clone, Debug, PartialEq)]
998998
pub struct TypeIgnoreTypeIgnore {
999-
pub lineno: usize,
999+
pub lineno: u32,
10001000
pub tag: String,
10011001
}
10021002

@@ -1019,6 +1019,7 @@ pub mod fold {
10191019
type TargetU;
10201020
type Error;
10211021
fn map_user(&mut self, user: U) -> Result<Self::TargetU, Self::Error>;
1022+
10221023
fn map_located<T>(
10231024
&mut self,
10241025
located: Attributed<T, U>,
@@ -1030,6 +1031,13 @@ pub mod fold {
10301031
node: located.node,
10311032
})
10321033
}
1034+
1035+
fn fold<X: Foldable<U, Self::TargetU>>(
1036+
&mut self,
1037+
node: X,
1038+
) -> Result<X::Mapped, Self::Error> {
1039+
node.fold(self)
1040+
}
10331041
fn fold_mod(&mut self, node: Mod<U>) -> Result<Mod<Self::TargetU>, Self::Error> {
10341042
fold_mod(self, node)
10351043
}

‎ast/src/gen/located.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
// File automatically generated by ast/asdl_rs.py.
22

3-
use crate::location::SourceRange;
3+
use rustpython_parser_core::source_code::SourceRange;
44

55
pub type Located<T> = super::generic::Attributed<T, SourceRange>;
66
pub type Mod = super::generic::Mod<SourceRange>;

‎ast/src/lib.rs

+9-41
Original file line numberDiff line numberDiff line change
@@ -7,53 +7,21 @@ mod generic {
77
include!("gen/generic.rs");
88
}
99
mod impls;
10-
#[cfg(feature = "location")]
11-
pub mod located {
12-
include!("gen/located.rs");
13-
}
14-
#[cfg(feature = "location")]
15-
mod locator;
16-
#[cfg(feature = "location")]
17-
pub use crate::locator::locate;
18-
#[cfg(feature = "location")]
19-
pub use rustpython_compiler_core::SourceLocator;
20-
10+
#[cfg(feature = "source-code")]
11+
mod source_locator;
2112
#[cfg(feature = "unparse")]
2213
mod unparse;
2314

2415
pub use attributed::Attributed;
25-
pub use constant::{Constant, ConversionFlag};
16+
pub use constant::Constant;
2617
pub use generic::*;
18+
pub use rustpython_parser_core::{text_size, ConversionFlag};
2719

2820
pub type Suite<U = ()> = Vec<Stmt<U>>;
2921

30-
pub mod location {
31-
pub use rustpython_compiler_core::source_code::{OneIndexed, SourceLocation};
32-
33-
#[derive(Debug)]
34-
pub struct SourceRange {
35-
pub start: SourceLocation,
36-
pub end: Option<SourceLocation>,
37-
}
38-
39-
impl SourceRange {
40-
pub fn new(start: SourceLocation, end: SourceLocation) -> Self {
41-
Self {
42-
start,
43-
end: Some(end),
44-
}
45-
}
46-
pub fn unwrap_end(&self) -> SourceLocation {
47-
self.end.unwrap()
48-
}
49-
}
50-
51-
impl From<std::ops::Range<SourceLocation>> for SourceRange {
52-
fn from(value: std::ops::Range<SourceLocation>) -> Self {
53-
Self {
54-
start: value.start,
55-
end: Some(value.end),
56-
}
57-
}
58-
}
22+
#[cfg(feature = "source-code")]
23+
pub mod located {
24+
include!("gen/located.rs");
5925
}
26+
27+
pub use rustpython_parser_core::source_code;

‎ast/src/locator.rs renamed to ‎ast/src/source_locator.rs

+1-7
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,5 @@
11
use crate::attributed::Attributed;
2-
use crate::fold_helpers::Foldable;
3-
use crate::location::SourceRange;
4-
use rustpython_compiler_core::SourceLocator;
5-
6-
pub fn locate<X: Foldable<(), SourceRange>>(locator: &mut SourceLocator, ast: X) -> X::Mapped {
7-
ast.fold(locator).unwrap()
8-
}
2+
use rustpython_parser_core::source_code::{SourceLocator, SourceRange};
93

104
impl crate::fold::Fold<()> for SourceLocator<'_> {
115
type TargetU = SourceRange;

‎ast/src/unparse.rs

+4-6
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,5 @@
1-
use crate::{
2-
Arg, Arguments, Boolop, Cmpop, Comprehension, Constant, ConversionFlag, Expr, ExprKind,
3-
Operator,
4-
};
1+
use crate::ConversionFlag;
2+
use crate::{Arg, Arguments, Boolop, Cmpop, Comprehension, Constant, Expr, ExprKind, Operator};
53
use std::fmt;
64

75
mod precedence {
@@ -452,7 +450,7 @@ impl<'a> Unparser<'a> {
452450
fn unparse_formatted<U>(
453451
&mut self,
454452
val: &Expr<U>,
455-
conversion: usize,
453+
conversion: u32,
456454
spec: Option<&Expr<U>>,
457455
) -> fmt::Result {
458456
let buffered = to_string_fmt(|f| Unparser::new(f).unparse_expr(val, precedence::TEST + 1));
@@ -466,7 +464,7 @@ impl<'a> Unparser<'a> {
466464
self.p(&buffered)?;
467465
drop(buffered);
468466

469-
if conversion != ConversionFlag::None as usize {
467+
if conversion != ConversionFlag::None as u32 {
470468
self.p("!")?;
471469
let buf = &[conversion as u8];
472470
let c = std::str::from_utf8(buf).unwrap();

‎core/Cargo.toml

+5-3
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,13 @@
11
[package]
2-
name = "rustpython-compiler-core"
3-
description = "RustPython specific bytecode."
2+
name = "rustpython-parser-core"
3+
description = "RustPython parser data types."
44
version = "0.2.0"
55
authors = ["RustPython Team"]
66
edition = "2021"
77
repository = "https://github.com/RustPython/RustPython"
88
license = "MIT"
99

1010
[dependencies]
11-
bitflags = { workspace = true }
1211
itertools = { workspace = true }
1312
num-bigint = { workspace = true }
1413
num-complex = { workspace = true }
@@ -18,3 +17,6 @@ ruff_python_ast = { workspace = true }
1817

1918
lz4_flex = "0.9.2"
2019

20+
[features]
21+
default = ["source-code"]
22+
source-code = []

0 commit comments

Comments
 (0)