Skip to content

Commit 66398fb

Browse files
jaudigerjaudiger
authored
Factorize HTTP commands code (#8088)
# Description In order to work on #2741, I'm preparing the code. # User-Facing Changes Both commands do support the timeout option. **But the timeout argument `-t, --timeout` has been migrated to `-m, --max-time`**. I had to make a choice since there is another option using the short command `t` which is "content-type". # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - `cargo test --workspace` to check that all tests pass # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date.
1 parent daeb3e5 commit 66398fb

3 files changed

Lines changed: 401 additions & 527 deletions

File tree

‎crates/nu-command/src/network/http/client.rs‎

Lines changed: 351 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,25 @@
1+
use crate::formats::value_to_json_value;
2+
use base64::engine::general_purpose::PAD;
3+
use base64::engine::GeneralPurpose;
4+
use base64::{alphabet, Engine};
5+
use nu_protocol::ast::Call;
6+
use nu_protocol::engine::{EngineState, Stack};
7+
use nu_protocol::{BufferedReader, PipelineData, RawStream, ShellError, Span, Value};
8+
use reqwest::blocking::{RequestBuilder, Response};
9+
use reqwest::{blocking, Error, StatusCode};
10+
use std::collections::HashMap;
11+
use std::io::BufReader;
12+
use std::path::PathBuf;
13+
use std::str::FromStr;
14+
use std::time::Duration;
15+
16+
#[derive(PartialEq, Eq)]
17+
pub enum BodyType {
18+
Json,
19+
Form,
20+
Unknown,
21+
}
22+
123
// Only panics if the user agent is invalid but we define it statically so either
224
// it always or never fails
325
pub fn http_client(allow_insecure: bool) -> reqwest::blocking::Client {
@@ -7,3 +29,332 @@ pub fn http_client(allow_insecure: bool) -> reqwest::blocking::Client {
729
.build()
830
.expect("Failed to build reqwest client")
931
}
32+
33+
pub fn response_to_buffer(
34+
response: blocking::Response,
35+
engine_state: &EngineState,
36+
span: Span,
37+
) -> PipelineData {
38+
// Try to get the size of the file to be downloaded.
39+
// This is helpful to show the progress of the stream.
40+
let buffer_size = match &response.headers().get("content-length") {
41+
Some(content_length) => {
42+
let content_length = &(*content_length).clone(); // binding
43+
44+
let content_length = content_length
45+
.to_str()
46+
.unwrap_or("")
47+
.parse::<u64>()
48+
.unwrap_or(0);
49+
50+
if content_length == 0 {
51+
None
52+
} else {
53+
Some(content_length)
54+
}
55+
}
56+
_ => None,
57+
};
58+
59+
let buffered_input = BufReader::new(response);
60+
61+
PipelineData::ExternalStream {
62+
stdout: Some(RawStream::new(
63+
Box::new(BufferedReader {
64+
input: buffered_input,
65+
}),
66+
engine_state.ctrlc.clone(),
67+
span,
68+
buffer_size,
69+
)),
70+
stderr: None,
71+
exit_code: None,
72+
span,
73+
metadata: None,
74+
trim_end_newline: false,
75+
}
76+
}
77+
78+
pub fn request_add_authorization_header(
79+
user: Option<String>,
80+
password: Option<String>,
81+
mut request: RequestBuilder,
82+
) -> RequestBuilder {
83+
let base64_engine = GeneralPurpose::new(&alphabet::STANDARD, PAD);
84+
85+
let login = match (user, password) {
86+
(Some(user), Some(password)) => {
87+
let mut enc_str = String::new();
88+
base64_engine.encode_string(&format!("{user}:{password}"), &mut enc_str);
89+
Some(enc_str)
90+
}
91+
(Some(user), _) => {
92+
let mut enc_str = String::new();
93+
base64_engine.encode_string(&format!("{user}:"), &mut enc_str);
94+
Some(enc_str)
95+
}
96+
(_, Some(password)) => {
97+
let mut enc_str = String::new();
98+
base64_engine.encode_string(&format!(":{password}"), &mut enc_str);
99+
Some(enc_str)
100+
}
101+
_ => None,
102+
};
103+
104+
if let Some(login) = login {
105+
request = request.header("Authorization", format!("Basic {login}"));
106+
}
107+
108+
request
109+
}
110+
111+
pub fn request_set_body(
112+
content_type: Option<String>,
113+
content_length: Option<String>,
114+
body: Value,
115+
mut request: RequestBuilder,
116+
) -> Result<RequestBuilder, ShellError> {
117+
// set the content-type header before using e.g., request.json
118+
// because that will avoid duplicating the header value
119+
if let Some(val) = &content_type {
120+
request = request.header("Content-Type", val);
121+
}
122+
123+
let body_type = match content_type {
124+
Some(it) if it == "application/json" => BodyType::Json,
125+
Some(it) if it == "application/x-www-form-urlencoded" => BodyType::Form,
126+
_ => BodyType::Unknown,
127+
};
128+
129+
match body {
130+
Value::Binary { val, .. } => {
131+
request = request.body(val);
132+
}
133+
Value::String { val, .. } => {
134+
request = request.body(val);
135+
}
136+
Value::Record { .. } if body_type == BodyType::Json => {
137+
let data = value_to_json_value(&body)?;
138+
request = request.json(&data);
139+
}
140+
Value::Record { .. } if body_type == BodyType::Form => {
141+
let data = value_to_json_value(&body)?;
142+
request = request.form(&data);
143+
}
144+
Value::List { vals, .. } if body_type == BodyType::Form => {
145+
if vals.len() % 2 != 0 {
146+
return Err(ShellError::IOError("unsupported body input".into()));
147+
}
148+
let data = vals
149+
.chunks(2)
150+
.map(|it| Ok((it[0].as_string()?, it[1].as_string()?)))
151+
.collect::<Result<Vec<(String, String)>, ShellError>>()?;
152+
request = request.form(&data)
153+
}
154+
_ => {
155+
return Err(ShellError::IOError("unsupported body input".into()));
156+
}
157+
};
158+
159+
if let Some(val) = content_length {
160+
request = request.header("Content-Length", val);
161+
}
162+
163+
Ok(request)
164+
}
165+
166+
pub fn request_set_timeout(
167+
timeout: Option<Value>,
168+
mut request: RequestBuilder,
169+
) -> Result<RequestBuilder, ShellError> {
170+
if let Some(timeout) = timeout {
171+
let val = timeout.as_i64()?;
172+
if val.is_negative() || val < 1 {
173+
return Err(ShellError::TypeMismatch(
174+
"Timeout value must be an integer and larger than 0".to_string(),
175+
// timeout is already guaranteed to not be an error
176+
timeout.expect_span(),
177+
));
178+
}
179+
180+
request = request.timeout(Duration::from_secs(val as u64));
181+
}
182+
183+
Ok(request)
184+
}
185+
186+
pub fn request_add_custom_headers(
187+
headers: Option<Value>,
188+
mut request: RequestBuilder,
189+
) -> Result<RequestBuilder, ShellError> {
190+
if let Some(headers) = headers {
191+
let mut custom_headers: HashMap<String, Value> = HashMap::new();
192+
193+
match &headers {
194+
Value::List { vals: table, .. } => {
195+
if table.len() == 1 {
196+
// single row([key1 key2]; [val1 val2])
197+
match &table[0] {
198+
Value::Record { cols, vals, .. } => {
199+
for (k, v) in cols.iter().zip(vals.iter()) {
200+
custom_headers.insert(k.to_string(), v.clone());
201+
}
202+
}
203+
204+
x => {
205+
return Err(ShellError::CantConvert(
206+
"string list or single row".into(),
207+
x.get_type().to_string(),
208+
headers.span().unwrap_or_else(|_| Span::new(0, 0)),
209+
None,
210+
));
211+
}
212+
}
213+
} else {
214+
// primitive values ([key1 val1 key2 val2])
215+
for row in table.chunks(2) {
216+
if row.len() == 2 {
217+
custom_headers.insert(row[0].as_string()?, row[1].clone());
218+
}
219+
}
220+
}
221+
}
222+
223+
x => {
224+
return Err(ShellError::CantConvert(
225+
"string list or single row".into(),
226+
x.get_type().to_string(),
227+
headers.span().unwrap_or_else(|_| Span::new(0, 0)),
228+
None,
229+
));
230+
}
231+
};
232+
233+
for (k, v) in &custom_headers {
234+
if let Ok(s) = v.as_string() {
235+
request = request.header(k, s);
236+
}
237+
}
238+
}
239+
240+
Ok(request)
241+
}
242+
243+
pub fn request_handle_response(
244+
engine_state: &EngineState,
245+
stack: &mut Stack,
246+
span: Span,
247+
requested_url: &String,
248+
raw: bool,
249+
response: Result<Response, Error>,
250+
) -> Result<PipelineData, ShellError> {
251+
// Explicitly turn 4xx and 5xx statuses into errors.
252+
match response {
253+
Ok(resp) => match resp.headers().get("content-type") {
254+
Some(content_type) => {
255+
let content_type = content_type.to_str().map_err(|e| {
256+
ShellError::GenericError(
257+
e.to_string(),
258+
"".to_string(),
259+
None,
260+
Some("MIME type were invalid".to_string()),
261+
Vec::new(),
262+
)
263+
})?;
264+
let content_type = mime::Mime::from_str(content_type).map_err(|_| {
265+
ShellError::GenericError(
266+
format!("MIME type unknown: {content_type}"),
267+
"".to_string(),
268+
None,
269+
Some("given unknown MIME type".to_string()),
270+
Vec::new(),
271+
)
272+
})?;
273+
let ext = match (content_type.type_(), content_type.subtype()) {
274+
(mime::TEXT, mime::PLAIN) => {
275+
let path_extension = url::Url::parse(requested_url)
276+
.map_err(|_| {
277+
ShellError::GenericError(
278+
format!("Cannot parse URL: {requested_url}"),
279+
"".to_string(),
280+
None,
281+
Some("cannot parse".to_string()),
282+
Vec::new(),
283+
)
284+
})?
285+
.path_segments()
286+
.and_then(|segments| segments.last())
287+
.and_then(|name| if name.is_empty() { None } else { Some(name) })
288+
.and_then(|name| {
289+
PathBuf::from(name)
290+
.extension()
291+
.map(|name| name.to_string_lossy().to_string())
292+
});
293+
path_extension
294+
}
295+
_ => Some(content_type.subtype().to_string()),
296+
};
297+
298+
let output = response_to_buffer(resp, engine_state, span);
299+
300+
if raw {
301+
return Ok(output);
302+
}
303+
304+
if let Some(ext) = ext {
305+
match engine_state.find_decl(format!("from {ext}").as_bytes(), &[]) {
306+
Some(converter_id) => engine_state.get_decl(converter_id).run(
307+
engine_state,
308+
stack,
309+
&Call::new(span),
310+
output,
311+
),
312+
None => Ok(output),
313+
}
314+
} else {
315+
Ok(output)
316+
}
317+
}
318+
None => Ok(response_to_buffer(resp, engine_state, span)),
319+
},
320+
Err(e) if e.is_timeout() => Err(ShellError::NetworkFailure(
321+
format!("Request to {requested_url} has timed out"),
322+
span,
323+
)),
324+
Err(e) if e.is_status() => match e.status() {
325+
Some(err_code) if err_code == StatusCode::NOT_FOUND => Err(ShellError::NetworkFailure(
326+
format!("Requested file not found (404): {requested_url:?}"),
327+
span,
328+
)),
329+
Some(err_code) if err_code == StatusCode::MOVED_PERMANENTLY => {
330+
Err(ShellError::NetworkFailure(
331+
format!("Resource moved permanently (301): {requested_url:?}"),
332+
span,
333+
))
334+
}
335+
Some(err_code) if err_code == StatusCode::BAD_REQUEST => Err(
336+
ShellError::NetworkFailure(format!("Bad request (400) to {requested_url:?}"), span),
337+
),
338+
Some(err_code) if err_code == StatusCode::FORBIDDEN => Err(ShellError::NetworkFailure(
339+
format!("Access forbidden (403) to {requested_url:?}"),
340+
span,
341+
)),
342+
_ => Err(ShellError::NetworkFailure(
343+
format!(
344+
"Cannot make request to {:?}. Error is {:?}",
345+
requested_url,
346+
e.to_string()
347+
),
348+
span,
349+
)),
350+
},
351+
Err(e) => Err(ShellError::NetworkFailure(
352+
format!(
353+
"Cannot make request to {:?}. Error is {:?}",
354+
requested_url,
355+
e.to_string()
356+
),
357+
span,
358+
)),
359+
}
360+
}

0 commit comments

Comments
 (0)