🚧 Drafting Marktplaats client

Cargo.toml

@@ -13,8 +13,14 @@ version = "0.4.0"
 
 [dependencies]
 anyhow = "1.0.79"
+chrono = { version = "0.4.31", features = ["serde"] }
 clap = { version = "4.4.16", features = ["cargo", "derive", "env"] }
+dotenvy = "0.15.7"
+reqwest = { version = "0.11.23", default-features = false, features = ["gzip", "json", "rustls"] }
 sentry = { version = "0.32.1", default-features = false, features = ["anyhow", "backtrace", "contexts", "panic", "reqwest", "rustls", "tracing"] }
+serde = "1.0.195"
+serde_json = "1.0.111"
+tokio = { version = "1.35.1", features = ["macros"] }
 tracing = "0.1.40"
 tracing-appender = "0.2.3"
 tracing-subscriber = { version = "0.3.18", features = ["env-filter"] }

src/cli.rs

@@ -3,9 +3,18 @@ use clap::{Parser, Subcommand};
 #[derive(Parser)]
 #[command(author, version, about, long_about, propagate_version = true)]
 pub struct Cli {
+    #[command(subcommand)]
+    pub command: Command,
+
     #[clap(long, env = "SENTRY_DSN")]
     pub sentry_dsn: Option<String>,
 
     #[clap(long, env = "SENTRY_TRACES_SAMPLE_RATE", default_value = "1.0")]
     pub traces_sample_rate: f32,
 }
+
+#[derive(Subcommand)]
+pub enum Command {
+    /// Run the crawler.
+    Crawler,
+}

src/crawler.rs

@@ -0,0 +1,13 @@
+use crate::prelude::*;
+
+pub struct Crawler {}
+
+impl Crawler {
+    pub fn new() -> Result<Self> {
+        todo!()
+    }
+
+    pub async fn run(&self) -> Result {
+        todo!()
+    }
+}

src/main.rs

@@ -2,14 +2,26 @@
 
 use clap::Parser;
 
-use crate::{cli::Cli, prelude::*};
+use crate::{
+    cli::{Cli, Command},
+    crawler::Crawler,
+    prelude::*,
+};
 
 mod cli;
+mod crawler;
+mod marktplaats;
 mod prelude;
 mod tracing;
 
-fn main() -> Result<()> {
+#[tokio::main]
+async fn main() -> Result {
+    if dotenvy::dotenv().is_err() {
+        warn!("failed to load `.env`");
+    }
     let cli = Cli::parse();
     let _tracing_guards = tracing::init(cli.sentry_dsn, cli.traces_sample_rate)?;
-    Ok(())
+    match cli.command {
+        Command::Crawler => Crawler::new()?.run().await,
+    }
 }

src/marktplaats.rs

@@ -0,0 +1,47 @@
+use reqwest::Url;
+use tracing::instrument;
+
+use crate::{
+    marktplaats::models::{Listing, SearchResponse},
+    prelude::*,
+};
+
+mod models;
+
+pub struct Marktplaats(reqwest::Client);
+
+impl Marktplaats {
+    pub fn new() -> Result<Self> {
+        Ok(Self(
+            reqwest::ClientBuilder::new()
+                .user_agent("Googlebot/2.1 (+http://www.google.com/bot.html)")
+                .build()
+                .context("failed to build Marktplaats client")?,
+        ))
+    }
+
+    /// Search for a single listing.
+    ///
+    /// If multiple listings match the query, only the first one is returned.
+    #[instrument(skip_all, level = "debug", fields(query))]
+    pub async fn find_one(&self, query: &str) -> Result<Option<Listing>> {
+        let url = Url::parse_with_params(
+            "https://www.marktplaats.nl/lrp/api/search?limit=1",
+            &[("query", query)],
+        )?;
+        let response = self
+            .0
+            .get(url)
+            .send()
+            .await
+            .with_context(|| format!("failed to search `{query}`"))?
+            .error_for_status()
+            .with_context(|| format!("failed to search `{query}` (bad status code)"))?
+            .json::<SearchResponse>()
+            .await
+            .with_context(|| format!("failed to deserialize search response for `{query}`"))?
+            .listings
+            .pop();
+        Ok(response)
+    }
+}