Networking

Recipe Crates Categories
Parse a URL from a string to a Url type url-badge cat-net-badge
Create a base URL by removing path segments url-badge cat-net-badge
Create new URLs from a base URL url-badge cat-net-badge
Extract the URL origin (scheme / host / port) url-badge cat-net-badge
Remove fragment identifiers and query pairs from a URL url-badge cat-net-badge
Make a HTTP GET request reqwest-badge cat-net-badge
Download a file to a temporary directory reqwest-badge tempdir-badge cat-net-badge cat-filesystem-badge
Query the GitHub API reqwest-badge serde-badge cat-net-badge cat-encoding-badge
Consume a paginated RESTful API reqwest-badge serde-badge cat-net-badge cat-encoding-badge
Check if an API resource exists reqwest-badge cat-net-badge
Set custom headers and URL parameters for a REST request reqwest-badge hyper-badge url-badge cat-net-badge
Create and delete Gist with GitHub API reqwest-badge serde-badge cat-net-badge cat-encoding-badge
POST a file to paste-rs reqwest-badge cat-net-badge
Listen on unused port TCP/IP std-badge cat-net-badge
Extract all links from a webpage reqwest-badge select-badge cat-net-badge
Extract all unique links from a MediaWiki markup reqwest-badge regex-badge cat-net-badge

Parse a URL from a string to a Url type

url-badge cat-net-badge

The parse method from the url crate validates and parses a &str into a Url struct. The input string may be malformed so this method returns Result<Url, ParseError>.

Once the URL has been parsed, it can be used with all of the methods on the Url type.

The URL in this code parses successfully, but swapping it out for a malformed URL will print a message containing an explanation of what went wrong.

# #[macro_use]
# extern crate error_chain;
extern crate url;

use url::Url;
#
# error_chain! {
#     foreign_links {
#         UrlParse(url::ParseError);
#     }
# }

fn run() -> Result<()> {
    let s = "https://github.com/rust-lang/rust/issues?labels=E-easy&state=open";

    let parsed = Url::parse(s)?;
    println!("The path part of the URL is: {}", parsed.path());

    Ok(())
}
#
# quick_main!(run);

Create a base URL by removing path segments

url-badge cat-net-badge

# #[macro_use]
# extern crate error_chain;
extern crate url;

use url::Url;
#
# error_chain! {
#     foreign_links {
#         UrlParse(url::ParseError);
#     }
#     errors {
#         CannotBeABase
#     }
# }

fn run() -> Result<()> {
    let full = "https://github.com/rust-lang/cargo?asdf";

    let url = Url::parse(full)?;
    let base = base_url(url)?;

    assert_eq!(base.as_str(), "https://github.com/");
    println!("The base of the URL is: {}", base);

    Ok(())
}

/// Returns the base of the given URL - the part not including any path segments
/// and query parameters.
fn base_url(mut url: Url) -> Result<Url> {
    // Clear path segments.
    match url.path_segments_mut() {
        Ok(mut path) => {
            path.clear();
        }
        Err(_) => {
            // Certain URLs cannot be turned into a base URL.
            return Err(Error::from_kind(ErrorKind::CannotBeABase));
        }
    }

    // Clear query parameters.
    url.set_query(None);

    Ok(url)
}
#
# quick_main!(run);

Create new URLs from a base URL

url-badge cat-net-badge

The join method creates a new URL from a base and relative path.

# #[macro_use]
# extern crate error_chain;
extern crate url;

use url::Url;
#
# error_chain! {
#     foreign_links {
#         UrlParse(url::ParseError);
#     }
# }

fn run() -> Result<()> {
    let path = "/rust-lang/cargo";

    let gh = build_github_url(path)?;

    assert_eq!(gh.as_str(), "https://github.com/rust-lang/cargo");
    println!("The joined URL is: {}", gh);

    Ok(())
}

fn build_github_url(path: &str) -> Result<Url> {
    // Hardcoded in our program. Caller's path will be joined to this.
    const GITHUB: &'static str = "https://github.com";

    let base = Url::parse(GITHUB).expect("hardcoded URL is known to be valid");
    let joined = base.join(path)?;

    Ok(joined)
}
#
# quick_main!(run);

Extract the URL origin (scheme / host / port)

url-badge cat-net-badge

The Url struct exposes various methods to extract information about the URL it represents.

# #[macro_use]
# extern crate error_chain;
extern crate url;

use url::{Url, Host};

# error_chain! {
#     foreign_links {
#         UrlParse(url::ParseError);
#     }
# }
#
fn run() -> Result<()> {
    let s = "ftp://rust-lang.org/examples";

    let url = Url::parse(s)?;

    assert_eq!(url.scheme(), "ftp");
    assert_eq!(url.host(), Some(Host::Domain("rust-lang.org")));
    assert_eq!(url.port_or_known_default(), Some(21));
    println!("The origin is as expected!");

    Ok(())
}
#
# quick_main!(run);

The same result can be obtained using the origin method as well.

# #[macro_use]
# extern crate error_chain;
extern crate url;

use url::{Url, Origin, Host};

# error_chain! {
#     foreign_links {
#         UrlParse(url::ParseError);
#     }
# }
#
fn run() -> Result<()> {
    let s = "ftp://rust-lang.org/examples";

    let url = Url::parse(s)?;

    let expected_scheme = "ftp".to_owned();
    let expected_host = Host::Domain("rust-lang.org".to_owned());
    let expected_port = 21;
    let expected = Origin::Tuple(expected_scheme, expected_host, expected_port);

    let origin = url.origin();
    assert_eq!(origin, expected);
    println!("The origin is as expected!");

    Ok(())
}
#
# quick_main!(run);

Remove fragment identifiers and query pairs from a URL

url-badge cat-net-badge

Parses Url and slices it with url::Position to strip unneeded URL parts.

# #[macro_use]
# extern crate error_chain;
extern crate url;

use url::{Url, Position};
#
# error_chain! {
#     foreign_links {
#         UrlParse(url::ParseError);
#     }
# }

fn run() -> Result<()> {
    let parsed = Url::parse("https://github.com/rust-lang/rust/issues?labels=E-easy&state=open")?;
    let cleaned: &str = &parsed[..Position::AfterPath];
    println!("cleaned: {}", cleaned);
    Ok(())
}
#
# quick_main!(run);

Make a HTTP GET request

reqwest-badge cat-net-badge

Parses the supplied URL and makes a synchronous HTTP GET request with reqwest::get. Prints obtained reqwest::Response status and headers subsequently reading HTTP response body into an allocated String via read_to_string.

# #[macro_use]
# extern crate error_chain;
extern crate reqwest;

use std::io::Read;
#
# error_chain! {
#     foreign_links {
#         Io(std::io::Error);
#         HttpRequest(reqwest::Error);
#     }
# }

fn run() -> Result<()> {
    let mut res = reqwest::get("http://httpbin.org/get")?;
    let mut body = String::new();
    res.read_to_string(&mut body)?;

    println!("Status: {}", res.status());
    println!("Headers:\n{}", res.headers());
    println!("Body:\n{}", body);

    Ok(())
}
#
# quick_main!(run);

Download a file to a temporary directory

reqwest-badge tempdir-badge cat-net-badge cat-filesystem-badge

Creates a temporary directory with TempDir::new and synchronously downloads a file over HTTP using reqwest::get. Creates a target File with name obtained from Response::url within TempDir::path and copies downloaded data into it with io::copy. The temporary directory is automatically removed on run function return.

# #[macro_use]
# extern crate error_chain;
extern crate reqwest;
extern crate tempdir;

use std::io::copy;
use std::fs::File;
use tempdir::TempDir;
#
# error_chain! {
#     foreign_links {
#         Io(std::io::Error);
#         HttpRequest(reqwest::Error);
#     }
# }

fn run() -> Result<()> {
    // create a temp dir with prefix "example"
    let tmp_dir = TempDir::new("example")?;
    // make HTTP request for remote content
    let target = "https://www.rust-lang.org/logos/rust-logo-512x512.png";
    let mut response = reqwest::get(target)?;

    let mut dest = {
        // extract target filename from URL
        let fname = response
            .url()
            .path_segments()
            .and_then(|segments| segments.last())
            .and_then(|name| if name.is_empty() { None } else { Some(name) })
            .unwrap_or("tmp.bin");

        println!("file to download: '{}'", fname);
        let fname = tmp_dir.path().join(fname);
        println!("will be located under: '{:?}'", fname);
        // create file with given name inside the temp dir
        File::create(fname)?
    };
    // data is copied into the target file
    copy(&mut response, &mut dest)?;
    // tmp_dir is implicitly removed
    Ok(())
}
#
# quick_main!(run);

Query the GitHub API

reqwest-badge serde-badge cat-net-badge cat-encoding-badge

Queries GitHub stargazers API v3 with reqwest::get to get list of all users who have marked a GitHub project with a star. reqwest::Response is deserialized with Response::json into User objects implementing serde::Deserialize.

# #[macro_use]
# extern crate error_chain;
#[macro_use]
extern crate serde_derive;
extern crate reqwest;

#[derive(Deserialize, Debug)]
struct User {
    login: String,
    id: u32,
    // remaining fields not deserialized for brevity
}
#
# error_chain! {
#     foreign_links {
#         Reqwest(reqwest::Error);
#     }
# }

fn run() -> Result<()> {
    let request_url = format!("https://api.github.com/repos/{owner}/{repo}/stargazers",
                              owner = "rust-lang-nursery",
                              repo = "rust-cookbook");
    println!("{}", request_url);
    let mut response = reqwest::get(&request_url)?;

    let users: Vec<User> = response.json()?;
    println!("{:?}", users);
    Ok(())
}
#
# quick_main!(run);

Check if an API resource exists

reqwest-badge cat-net-badge

Query the GitHub Users Endpoint using a HEAD request (Client::head) and then inspect the response code to determine success. This is a quick way to query a rest resource without needing to receive a body. You can also configure the reqwest::Client with ClientBuilder::timeout which ensures that a request will not last longer than what is passed to the timeout function.

# #[macro_use]
# extern crate error_chain;
extern crate reqwest;

use std::time::Duration;
use reqwest::ClientBuilder;
#
# error_chain! {
#     foreign_links {
#         Reqwest(reqwest::Error);
#     }
# }

fn run() -> Result<()> {
    let user = "ferris-the-crab";
    let request_url = format!("https://api.github.com/users/{}", user);
    println!("{}", request_url);

    // The timeout for the request is set to 5 seconds.
    let timeout = Duration::new(5, 0);

    let client = ClientBuilder::new()?.timeout(timeout).build()?;
    let response = client.head(&request_url)?.send()?;

    if response.status().is_success() {
        println!("{} is a user!", user);
    } else {
        println!("{} is not a user!", user);
    }

    Ok(())
}
#
# quick_main!(run);

Set custom headers and URL parameters for a REST request

reqwest-badge hyper-badge url-badge cat-net-badge

Sets both standard and custom HTTP headers as well as URL parameters for HTTP GET request. Firstly creates a custom header of type XPoweredBy with hyper::header! macro. Secondly calls Url::parse_with_params in order to build a complex URL with specified key value pairs. Lastly sets standard headers header::UserAgent and header::Authorization as well as custom one XPoweredBy with RequestBuilder::header prior to making the request with RequestBuilder::send.

The code is run against http://httpbin.org/headers service which responds with a JSON dict containing all request headers for easy verification.

# #[macro_use]
# extern crate error_chain;
extern crate url;
extern crate reqwest;
#[macro_use]
extern crate hyper;
#[macro_use]
extern crate serde_derive;

use std::collections::HashMap;
use url::Url;
use reqwest::header::{UserAgent, Authorization, Bearer};

// Custom header type
header! { (XPoweredBy, "X-Powered-By") => [String] }

// Helper for verification
#[derive(Deserialize, Debug)]
pub struct HeadersEcho {
    pub headers: HashMap<String, String>,
}
#
# error_chain! {
#     foreign_links {
#         Reqwest(reqwest::Error);
#         UrlParse(url::ParseError);
#     }
# }

fn run() -> Result<()> {
    let client = reqwest::Client::new()?;

    // Make request to webservice that will respond with JSON dict containing
    // the headders set on HTTP GET request.
    let url = Url::parse_with_params("http://httpbin.org/headers",
                                     &[("lang", "rust"), ("browser", "servo")])?;

    let mut response = client
        .get(url)?
        .header(UserAgent::new("Rust-test"))
        .header(Authorization(Bearer { token: "DEadBEEfc001cAFeEDEcafBAd".to_owned() }))
        .header(XPoweredBy("Guybrush Threepwood".to_owned()))
        .send()?;

    // JSON response should match the headers set on request
    let out: HeadersEcho = response.json()?;
    assert_eq!(out.headers["Authorization"],
               "Bearer DEadBEEfc001cAFeEDEcafBAd");
    assert_eq!(out.headers["User-Agent"], "Rust-test");
    assert_eq!(out.headers["X-Powered-By"], "Guybrush Threepwood");
    // Response contains full URL used to make the request
    assert_eq!(response.url().as_str(),
               "http://httpbin.org/headers?lang=rust&browser=servo");

    println!("{:?}", out);
    Ok(())
}
#
# quick_main!(run);

Create and delete Gist with GitHub API

reqwest-badge serde-badge cat-net-badge cat-encoding-badge

Creates a gist with POST request to GitHub gists API v3 using Client::post and subsequently removes it with DELETE request using Client::delete.

The reqwest::Client is responsible for details of both requests including URL, body and authentication. POST body comes from serde_json::json! macro which provides a way to pass an arbitrary JSON body. Call to RequestBuilder::json sets the request body while RequestBuilder::basic_auth handles authentication. Finally the call to RequestBuilder::send synchronously executes the requests.

# #[macro_use]
# extern crate error_chain;
extern crate reqwest;
#[macro_use]
extern crate serde_derive;
#[macro_use]
extern crate serde_json;

use std::env;
#
# error_chain! {
#     foreign_links {
#         EnvVar(env::VarError);
#         HttpRequest(reqwest::Error);
#     }
# }

#[derive(Deserialize, Debug)]
struct Gist {
    id: String,
    html_url: String,
    // remaining fields not deserialized for brevity
}

fn run() -> Result<()> {
    let gh_user = env::var("GH_USER")?;
    let gh_pass = env::var("GH_PASS")?;

    // The type `gist_body` is `serde_json::Value`
    let gist_body = json!({
        "description": "the description for this gist",
        "public": true,
        "files": {
             "main.rs": {
             "content": r#"fn main() { println!("hello world!");}"#
            }
        }});

    // create the gist
    let request_url = "https://api.github.com/gists";
    let client = reqwest::Client::new()?;
    let mut response = client
        .post(request_url)?
        .basic_auth(gh_user.clone(), Some(gh_pass.clone()))
        .json(&gist_body)?
        .send()?;

    let gist: Gist = response.json()?;
    println!("Created {:?}", gist);

    // delete the gist
    let request_url = format!("{}/{}",request_url, gist.id);
    let client = reqwest::Client::new()?;
    let response = client
        .delete(&request_url)?
        .basic_auth(gh_user, Some(gh_pass))
        .send()?;

    println!("Gist {} deleted! Status code: {}",gist.id, response.status());
    Ok(())
}
#
# quick_main!(run);

For the sake of simplicity the example uses HTTP Basic Auth in order to authorize access to GitHub API. A more typical use case would be to employ one of the much more complex OAuth authorization flows.

Consume a paginated RESTful API

reqwest-badge serde-badge cat-net-badge cat-encoding-badge

Wraps a paginated web API in a convenient Rust iterator. The iterator lazily fetches the next page of results from the remote server as it arrives at the end of each page.

# #[macro_use]
# extern crate error_chain;
#[macro_use]
extern crate serde_derive;
extern crate serde;
extern crate reqwest;
#
# error_chain! {
#     foreign_links {
#         Reqwest(reqwest::Error);
#     }
# }

#[derive(Deserialize)]
struct ApiResponse {
    dependencies: Vec<Dependency>,
    meta: Meta,
}

// Could capture more fields here if needed
#[derive(Deserialize)]
struct Dependency {
    crate_id: String,
}

#[derive(Deserialize)]
struct Meta {
    total: u32,
}

struct ReverseDependencies {
    crate_id: String,
    dependencies: <Vec<Dependency> as IntoIterator>::IntoIter,
    client: reqwest::Client,
    page: u32,
    per_page: u32,
    total: u32,
}

impl ReverseDependencies {
    fn of(crate_id: &str) -> Result<Self> {
        Ok(ReverseDependencies {
               crate_id: crate_id.to_owned(),
               dependencies: vec![].into_iter(),
               client: reqwest::Client::new()?,
               page: 0,
               per_page: 100,
               total: 0,
           })
    }

    fn try_next(&mut self) -> Result<Option<Dependency>> {
        // If the previous page has a dependency that hasn't been looked at.
        if let Some(dep) = self.dependencies.next() {
            return Ok(Some(dep));
        }

        // If there are no more reverse dependencies.
        if self.page > 0 && self.page * self.per_page >= self.total {
            return Ok(None);
        }

        // Fetch the next page.
        self.page += 1;
        let url = format!("https://crates.io/api/v1/crates/{}/reverse_dependencies?page={}&per_page={}",
                          self.crate_id,
                          self.page,
                          self.per_page);

        let response = self.client.get(&url)?.send()?.json::<ApiResponse>()?;
        self.dependencies = response.dependencies.into_iter();
        self.total = response.meta.total;
        Ok(self.dependencies.next())
    }
}

impl Iterator for ReverseDependencies {
    type Item = Result<Dependency>;

    fn next(&mut self) -> Option<Self::Item> {
        // Some juggling required here because `try_next` returns a result
        // containing an option, while `next` is supposed to return an option
        // containing a result.
        match self.try_next() {
            Ok(Some(dep)) => Some(Ok(dep)),
            Ok(None) => None,
            Err(err) => Some(Err(err)),
        }
    }
}

fn run() -> Result<()> {
    for dep in ReverseDependencies::of("serde")? {
        println!("reverse dependency: {}", dep?.crate_id);
    }
    Ok(())
}
#
# quick_main!(run);

POST a file to paste-rs.

reqwest-badge cat-net-badge

A connection is established to https://paste.rs using reqwest::Client, following the reqwest::RequestBuilder pattern. Calling Client::post with a URL establishes the destination, RequestBuilder::body sets the content to send by reading the file, and RequestBuilder::send blocks until the file uploads and the response is received. The response is read with read_to_string, and finally displayed in the console.

extern crate reqwest;

# #[macro_use]
# extern crate error_chain;
#
use std::fs::File;
use std::io::Read;
use reqwest::Client;
#
# error_chain! {
#     foreign_links {
#         HttpRequest(reqwest::Error);
#         IoError(::std::io::Error);
#     }
# }

fn run() -> Result<()> {
    let paste_api = "https://paste.rs";
    let file = File::open("message")?;
    let client = Client::new()?;

    // blocks until paste.rs returns a response
    let mut response = client.post(paste_api)?.body(file).send()?;
    let mut response_body = String::new();
    response.read_to_string(&mut response_body)?;
    println!("Your paste is located at: {}", response_body);
    Ok(())
}
#
# quick_main!(run);

Listen on unused port TCP/IP

std-badge cat-net-badge

In this example, the port is displayed on the console, and the program will listen until a request is made.

# #[macro_use]
# extern crate error_chain;
#
use std::net::{SocketAddrV4, Ipv4Addr, TcpListener};
use std::io::Read;
#
# error_chain! {
#     foreign_links {
#         Io(::std::io::Error);
#     }
# }

fn run() -> Result<()> {
    let loopback = Ipv4Addr::new(127, 0, 0, 1);
    // Assigning port 0 requests the OS to assign a free port
    let socket = SocketAddrV4::new(loopback, 0);
    let listener = TcpListener::bind(socket)?;
    let port = listener.local_addr()?;
    println!("Listening on {}, access this port to end the program", port);
    let (mut tcp_stream, addr) = listener.accept()?; //block  until requested
    println!("Connection received! {:?} is sending data.", addr);
    let mut input = String::new();
    // read from the socket until connection closed by client, discard byte count.
    let _ = tcp_stream.read_to_string(&mut input)?;
    println!("{:?} says {}", addr, input);
    Ok(())
}
#
# quick_main!(run);

The std library is leveraged to make a well formed IP/port with the SocketAddrV4 and Ipv4Addr structs. An unused random port is requested by passing 0 to TcpListener::bind. The assigned address is available via TcpListener::local_addr.

TcpListener::accept synchronously waits for an incoming connection and returns a (TcpStream, SocketAddrV4) representing the request. Reading on the socket with read_to_string will wait until the connection is closed which can be tested with telnet ip port. For example, if the program shows Listening on 127.0.0.1:11500, run

telnet 127.0.0.1 11500

After sending data in telnet press ctrl-] and type quit.

Extract all links from a webpage

reqwest-badge select-badge cat-net-badge

Use reqwest::get to perform a HTTP GET request and then use Document::from_read to parse the response into a HTML document. We can then retrieve all the links from the document by using find with the criteria of the Name being "a". This returns a Selection that we filter_map on to retrieve the urls from links that have the "href" attr.

# #[macro_use]
# extern crate error_chain;
extern crate reqwest;
extern crate select;

use select::document::Document;
use select::predicate::Name;
#
# error_chain! {
#    foreign_links {
#        ReqError(reqwest::Error);
#        IoError(std::io::Error);
#    }
# }

fn run() -> Result<()> {
    let res = reqwest::get("https://www.rust-lang.org/en-US/")?;

    let document = Document::from_read(res)?;

    let links = document.find(Name("a"))
        .filter_map(|n| n.attr("href"));

    for link in links {
        println!("{}", link);
    }

    Ok(())
}
#
# quick_main!(run);

Extract all unique links from a MediaWiki markup

reqwest-badge regex-badge cat-net-badge

Pull the source of a MediaWiki page using reqwest::get and then look for all entries of internal and external links with Regex::captures_iter. Using Cow avoids excessive String allocations.

MediaWiki link syntax is described here.

# #[macro_use]
# extern crate error_chain;
#[macro_use]
extern crate lazy_static;
extern crate reqwest;
extern crate regex;

use std::io::Read;
use std::collections::HashSet;
use std::borrow::Cow;
use regex::Regex;

# error_chain! {
#     foreign_links {
#         Io(std::io::Error);
#         Reqwest(reqwest::Error);
#         Regex(regex::Error);
#     }
# }
# 
fn extract_links(content: &str) -> Result<HashSet<Cow<str>>> {
    lazy_static! {
        static ref WIKI_REGEX: Regex =
            Regex::new(r"(?x)
                \[\[(?P<internal>[^\[\]|]*)[^\[\]]*\]\]    # internal links
                |
                (url=|URL\||\[)(?P<external>http.*?)[ \|}] # external links
            ").unwrap();
    }

    let links: HashSet<_> = WIKI_REGEX
        .captures_iter(content)
        .map(|c| match (c.name("internal"), c.name("external")) {
            (Some(val), None) => Cow::from(val.as_str().to_lowercase()),
            (None, Some(val)) => Cow::from(val.as_str()),
            _ => unreachable!(),
        })
        .collect();

    Ok(links)
}

fn run() -> Result<()> {
    let mut content = String::new();
    reqwest::get(
        "https://en.wikipedia.org/w/index.php?title=Rust_(programming_language)&action=raw",
    )?
        .read_to_string(&mut content)?;

    println!("{:#?}", extract_links(&content)?);

    Ok(())
}
#
# quick_main!(run);