Skip to content

Commit 6d2b549

Browse files
overlookmoteloverlookmotel
committed
feat(napi/oxlint): pass AST in buffer to JS (#12350)
First working prototype of JS linter plugins. Rust now passes the AST to JS via buffer, and lint rule runner uses the JS rule's provided visitor to traverse the AST using lazy deserialization. Only a single rule is supported at present. Before anyone gets too excited, this is very much unfinished! It's incomplete, the AST is not compliant with ESTree, and it's not well tested. Really this is a proof of concept more than a usable feature at present. But we'll get there...
1 parent c5dff1e commit 6d2b549

File tree

18 files changed

+298
-64
lines changed

18 files changed

+298
-64
lines changed

‎.github/generated/ast_changes_watch_list.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,8 @@ src:
6565
- 'crates/oxc_syntax/src/serialize.rs'
6666
- 'crates/oxc_syntax/src/symbol.rs'
6767
- 'crates/oxc_traverse/src/generated/scopes_collector.rs'
68+
- 'napi/oxlint2/src/generated/constants.cjs'
69+
- 'napi/oxlint2/src/generated/raw_transfer_constants.rs'
6870
- 'napi/parser/generated/constants.js'
6971
- 'napi/parser/generated/deserialize/js.js'
7072
- 'napi/parser/generated/deserialize/ts.js'

‎Cargo.lock‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎crates/oxc_allocator/src/lib.rs‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,9 +88,58 @@ use pool_fixed_size as pool;
8888
target_endian = "little"
8989
))]
9090
use pool_fixed_size::FixedSizeAllocatorMetadata;
91+
// Export so can be used in `napi/oxlint2`
92+
#[cfg(all(
93+
feature = "fixed_size",
94+
not(feature = "disable_fixed_size"),
95+
target_pointer_width = "64",
96+
target_endian = "little"
97+
))]
98+
pub use pool_fixed_size::free_fixed_size_allocator;
9199

92100
pub use pool::{AllocatorGuard, AllocatorPool};
93101

102+
// Dummy implementations of interfaces from `pool_fixed_size`, just to stop clippy complaining.
103+
// Seems to be necessary due to feature unification.
104+
#[cfg(not(all(
105+
feature = "fixed_size",
106+
not(feature = "disable_fixed_size"),
107+
target_pointer_width = "64",
108+
target_endian = "little"
109+
)))]
110+
#[allow(missing_docs, clippy::missing_safety_doc, clippy::unused_self, clippy::allow_attributes)]
111+
mod dummies {
112+
use std::{ptr::NonNull, sync::atomic::AtomicBool};
113+
114+
use super::Allocator;
115+
116+
#[doc(hidden)]
117+
pub struct FixedSizeAllocatorMetadata {
118+
pub id: u32,
119+
pub alloc_ptr: NonNull<u8>,
120+
pub is_double_owned: AtomicBool,
121+
}
122+
123+
#[doc(hidden)]
124+
pub unsafe fn free_fixed_size_allocator(_metadata_ptr: NonNull<FixedSizeAllocatorMetadata>) {
125+
unreachable!();
126+
}
127+
128+
#[doc(hidden)]
129+
impl Allocator {
130+
pub unsafe fn fixed_size_metadata_ptr(&self) -> NonNull<FixedSizeAllocatorMetadata> {
131+
unreachable!();
132+
}
133+
}
134+
}
135+
#[cfg(not(all(
136+
feature = "fixed_size",
137+
not(feature = "disable_fixed_size"),
138+
target_pointer_width = "64",
139+
target_endian = "little"
140+
)))]
141+
pub use dummies::*;
142+
94143
#[cfg(all(
95144
feature = "fixed_size",
96145
not(feature = "disable_fixed_size"),

‎crates/oxc_allocator/src/pool_fixed_size.rs‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -142,7 +142,8 @@ pub struct FixedSizeAllocatorMetadata {
142142
const ALLOC_SIZE: usize = BLOCK_SIZE + TWO_GIB;
143143
const ALLOC_ALIGN: usize = TWO_GIB;
144144

145-
const ALLOC_LAYOUT: Layout = match Layout::from_size_align(ALLOC_SIZE, ALLOC_ALIGN) {
145+
/// Layout of backing allocations for fixed-size allocators.
146+
pub const ALLOC_LAYOUT: Layout = match Layout::from_size_align(ALLOC_SIZE, ALLOC_ALIGN) {
146147
Ok(layout) => layout,
147148
Err(_) => unreachable!(),
148149
};
@@ -287,7 +288,7 @@ impl Drop for FixedSizeAllocator {
287288
}
288289
}
289290

290-
/// Deallocate memory backing a [`FixedSizeAllocator`] if it's not double-owned
291+
/// Deallocate memory backing a `FixedSizeAllocator` if it's not double-owned
291292
/// (both owned by a `FixedSizeAllocator` on Rust side *and* held as a buffer on JS side).
292293
///
293294
/// If it is double-owned, don't deallocate the memory but set the flag that it's no longer double-owned
@@ -302,7 +303,7 @@ impl Drop for FixedSizeAllocator {
302303
/// Calling this function in any other circumstances would result in a double-free.
303304
///
304305
/// `metadata_ptr` must point to a valid `FixedSizeAllocatorMetadata`.
305-
unsafe fn free_fixed_size_allocator(metadata_ptr: NonNull<FixedSizeAllocatorMetadata>) {
306+
pub unsafe fn free_fixed_size_allocator(metadata_ptr: NonNull<FixedSizeAllocatorMetadata>) {
306307
// Get pointer to start of original allocation from `FixedSizeAllocatorMetadata`
307308
let alloc_ptr = {
308309
// SAFETY: This `Allocator` was created by the `FixedSizeAllocator`.
@@ -341,13 +342,13 @@ unsafe fn free_fixed_size_allocator(metadata_ptr: NonNull<FixedSizeAllocatorMeta
341342
unsafe impl Send for FixedSizeAllocator {}
342343

343344
impl Allocator {
344-
/// Get pointer to the [`FixedSizeAllocatorMetadata`] for this [`Allocator`].
345+
/// Get pointer to the `FixedSizeAllocatorMetadata` for this [`Allocator`].
345346
///
346347
/// # SAFETY
347348
/// * This `Allocator` must have been created by a `FixedSizeAllocator`.
348349
/// * This pointer must not be used to create a mutable reference to the `FixedSizeAllocatorMetadata`,
349350
/// only immutable references.
350-
unsafe fn fixed_size_metadata_ptr(&self) -> NonNull<FixedSizeAllocatorMetadata> {
351+
pub unsafe fn fixed_size_metadata_ptr(&self) -> NonNull<FixedSizeAllocatorMetadata> {
351352
// SAFETY: Caller guarantees this `Allocator` was created by a `FixedSizeAllocator`.
352353
//
353354
// `FixedSizeAllocator::new` writes `FixedSizeAllocatorMetadata` after the end of

‎crates/oxc_linter/src/external_linter.rs‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@ use std::{fmt::Debug, pin::Pin, sync::Arc};
22

33
use serde::{Deserialize, Serialize};
44

5+
use oxc_allocator::Allocator;
6+
57
pub type ExternalLinterLoadPluginCb = Arc<
68
dyn Fn(
79
String,
@@ -17,7 +19,11 @@ pub type ExternalLinterLoadPluginCb = Arc<
1719
>;
1820

1921
pub type ExternalLinterCb = Arc<
20-
dyn Fn(String, Vec<u32>) -> Result<Vec<LintResult>, Box<dyn std::error::Error + Send + Sync>>
22+
dyn Fn(
23+
String,
24+
Vec<u32>,
25+
&Allocator,
26+
) -> Result<Vec<LintResult>, Box<dyn std::error::Error + Send + Sync>>
2127
+ Sync
2228
+ Send,
2329
>;

‎crates/oxc_linter/src/lib.rs‎

Lines changed: 21 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -213,7 +213,7 @@ impl Linter {
213213
}
214214

215215
#[cfg(all(feature = "oxlint2", not(feature = "disable_oxlint2")))]
216-
self.run_external_rules(&external_rules, path, &ctx_host, allocator);
216+
self.run_external_rules(&external_rules, path, semantic, &ctx_host, allocator);
217217

218218
// Stop clippy complaining about unused vars
219219
#[cfg(not(all(feature = "oxlint2", not(feature = "disable_oxlint2"))))]
@@ -233,9 +233,12 @@ impl Linter {
233233
&self,
234234
external_rules: &[(ExternalRuleId, AllowWarnDeny)],
235235
path: &Path,
236+
semantic: &Semantic<'_>,
236237
ctx_host: &ContextHost,
237-
_allocator: &Allocator,
238+
allocator: &Allocator,
238239
) {
240+
use std::ptr;
241+
239242
use oxc_diagnostics::OxcDiagnostic;
240243
use oxc_span::Span;
241244

@@ -248,9 +251,23 @@ impl Linter {
248251
// `external_linter` always exists when `oxlint2` feature is enabled
249252
let external_linter = self.external_linter.as_ref().unwrap();
250253

254+
// Write offset of `Program` and source text length in metadata at end of buffer
255+
let program = semantic.nodes().program().unwrap();
256+
let program_offset = ptr::from_ref(program) as u32;
257+
#[expect(clippy::cast_possible_truncation)]
258+
let source_len = program.source_text.len() as u32;
259+
260+
let metadata = RawTransferMetadata::new(program_offset, source_len);
261+
let metadata_ptr = allocator.end_ptr().cast::<RawTransferMetadata>();
262+
// SAFETY: `Allocator` was created by `FixedSizeAllocator` which reserved space after `end_ptr`
263+
// for a `RawTransferMetadata`. `end_ptr` is aligned for `FixedSizeAllocator`.
264+
unsafe { metadata_ptr.write(metadata) };
265+
266+
// Pass AST and rule IDs to JS
251267
let result = (external_linter.run)(
252268
path.to_str().unwrap().to_string(),
253269
external_rules.iter().map(|(rule_id, _)| rule_id.raw()).collect(),
270+
allocator,
254271
);
255272
match result {
256273
Ok(diagnostics) => {
@@ -304,10 +321,9 @@ struct RawTransferMetadata2 {
304321
use RawTransferMetadata2 as RawTransferMetadata;
305322

306323
#[cfg(all(feature = "oxlint2", not(feature = "disable_oxlint2")))]
307-
#[expect(dead_code)]
308324
impl RawTransferMetadata {
309-
pub fn new(data_offset: u32, is_ts: bool) -> Self {
310-
Self { data_offset, is_ts, source_len: 0, _padding: 0 }
325+
pub fn new(data_offset: u32, source_len: u32) -> Self {
326+
Self { data_offset, is_ts: false, source_len, _padding: 0 }
311327
}
312328
}
313329

‎napi/oxlint2/Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ test = false
2222
doctest = false
2323

2424
[dependencies]
25+
oxc_allocator = { workspace = true, features = ["fixed_size"] }
2526
oxlint = { workspace = true, features = ["oxlint2", "allocator"] }
2627

2728
napi = { workspace = true, features = ["async"] }

‎napi/oxlint2/src/bindings.d.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,6 @@ export type JsLoadPluginCb =
44
((arg: string) => Promise<string>)
55

66
export type JsRunCb =
7-
((arg0: string, arg1: Array<number>) => string)
7+
((arg0: string, arg1: number, arg2: Uint8Array | undefined | null, arg3: Array<number>) => string)
88

99
export declare function lint(loadPlugin: JsLoadPluginCb, run: JsRunCb): Promise<boolean>

‎napi/oxlint2/src/generated/constants.cjs‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
// Auto-generated code, DO NOT EDIT DIRECTLY!
2+
// To edit this generated file you have to edit `tasks/ast_tools/src/generators/raw_transfer.rs`.
3+
4+
const BUFFER_SIZE = 2147483616,
5+
BUFFER_ALIGN = 4294967296,
6+
DATA_POINTER_POS_32 = 536870900,
7+
IS_TS_FLAG_POS = 2147483612,
8+
SOURCE_LEN_POS_32 = 536870901,
9+
PROGRAM_OFFSET = 0;
10+
11+
module.exports = {
12+
BUFFER_SIZE,
13+
BUFFER_ALIGN,
14+
DATA_POINTER_POS_32,
15+
IS_TS_FLAG_POS,
16+
SOURCE_LEN_POS_32,
17+
PROGRAM_OFFSET,
18+
};

‎napi/oxlint2/src/generated/raw_transfer_constants.rs‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
// Auto-generated code, DO NOT EDIT DIRECTLY!
2+
// To edit this generated file you have to edit `tasks/ast_tools/src/generators/raw_transfer.rs`.
3+
4+
#![expect(clippy::unreadable_literal)]
5+
#![allow(dead_code)]
6+
7+
pub const BLOCK_SIZE: usize = 2147483632;
8+
pub const BLOCK_ALIGN: usize = 4294967296;
9+
pub const BUFFER_SIZE: usize = 2147483616;
10+
pub const RAW_METADATA_SIZE: usize = 16;

0 commit comments

Comments
 (0)