From d26931f4c6babe0e95e50626844c7e8a52c961f0 Mon Sep 17 00:00:00 2001 From: Marc Plano-Lesay Date: Thu, 1 May 2025 19:32:42 +1000 Subject: [PATCH] Add a Bitmagnet import action --- Cargo.lock | 99 +++++++++++++- Cargo.toml | 4 + .../down.sql | 1 + .../up.sql | 9 ++ src/actions/bitmagnet/action.rs | 70 ++++++++++ src/actions/bitmagnet/client.rs | 129 ++++++++++++++++++ src/actions/bitmagnet/config.rs | 8 ++ src/actions/bitmagnet/mod.rs | 7 + src/actions/mod.rs | 1 + src/actions/transmission/action.rs | 1 - src/config.rs | 4 + src/db.rs | 29 +++- src/main.rs | 20 +++ src/models.rs | 19 ++- src/schema.rs | 11 +- 15 files changed, 402 insertions(+), 10 deletions(-) create mode 100644 migrations/2025-05-01-084343_create_bitmagnet_processed/down.sql create mode 100644 migrations/2025-05-01-084343_create_bitmagnet_processed/up.sql create mode 100644 src/actions/bitmagnet/action.rs create mode 100644 src/actions/bitmagnet/client.rs create mode 100644 src/actions/bitmagnet/config.rs create mode 100644 src/actions/bitmagnet/mod.rs diff --git a/Cargo.lock b/Cargo.lock index f2e672c..5a530c7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -111,6 +111,12 @@ dependencies = [ "bytemuck", ] +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + [[package]] name = "autocfg" version = "1.4.0" @@ -688,6 +694,25 @@ dependencies = [ "tracing", ] +[[package]] +name = "h2" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75249d144030531f8dee69fe9cea04d3edf809a017ae445e2abdff6629e86633" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http 1.3.1", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "hashbrown" version = "0.15.2" @@ -790,7 +815,7 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "h2", + "h2 0.3.26", "http 0.2.12", "http-body 0.4.6", "httparse", @@ -813,6 +838,7 @@ dependencies = [ "bytes", "futures-channel", "futures-util", + "h2 0.4.9", "http 1.3.1", "http-body 1.0.1", "httparse", @@ -854,6 +880,22 @@ dependencies = [ "tokio-native-tls", ] +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes", + "http-body-util", + "hyper 1.6.0", + "hyper-util", + "native-tls", + "tokio", + "tokio-native-tls", + "tower-service", +] + [[package]] name = "hyper-util" version = "0.1.11" @@ -1154,6 +1196,16 @@ version = "0.4.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" +[[package]] +name = "magnet-url" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b4c4004e88aca00cc0c60782e5642c8fc628deca19e530ce58aa76e737d74" +dependencies = [ + "lazy_static", + "regex", +] + [[package]] name = "maybe-async" version = "0.2.10" @@ -1538,15 +1590,19 @@ dependencies = [ "figment", "figment_file_provider_adapter", "log", + "magnet-url", "multimap", "pretty_env_logger", "regex", + "reqwest 0.12.15", "roux", "serde", + "serde_json", "tempfile", "tokio", "transmission-rpc", "url", + "urlencoding", ] [[package]] @@ -1600,11 +1656,11 @@ dependencies = [ "encoding_rs", "futures-core", "futures-util", - "h2", + "h2 0.3.26", "http 0.2.12", "http-body 0.4.6", "hyper 0.14.32", - "hyper-tls", + "hyper-tls 0.5.0", "ipnet", "js-sys", "log", @@ -1618,7 +1674,7 @@ dependencies = [ "serde_json", "serde_urlencoded", "sync_wrapper 0.1.2", - "system-configuration", + "system-configuration 0.5.1", "tokio", "tokio-native-tls", "tower-service", @@ -1637,18 +1693,22 @@ checksum = "d19c46a6fdd48bc4dab94b6103fccc55d34c67cc0ad04653aad4ea2a07cd7bbb" dependencies = [ "base64 0.22.1", "bytes", + "encoding_rs", "futures-core", "futures-util", + "h2 0.4.9", "http 1.3.1", "http-body 1.0.1", "http-body-util", "hyper 1.6.0", "hyper-rustls", + "hyper-tls 0.6.0", "hyper-util", "ipnet", "js-sys", "log", "mime", + "native-tls", "once_cell", "percent-encoding", "pin-project-lite", @@ -1660,7 +1720,9 @@ dependencies = [ "serde_json", "serde_urlencoded", "sync_wrapper 1.0.2", + "system-configuration 0.6.1", "tokio", + "tokio-native-tls", "tokio-rustls", "tower", "tower-service", @@ -1986,7 +2048,18 @@ checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" dependencies = [ "bitflags 1.3.2", "core-foundation", - "system-configuration-sys", + "system-configuration-sys 0.5.0", +] + +[[package]] +name = "system-configuration" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" +dependencies = [ + "bitflags 2.9.0", + "core-foundation", + "system-configuration-sys 0.6.0", ] [[package]] @@ -1999,6 +2072,16 @@ dependencies = [ "libc", ] +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "tempfile" version = "3.19.1" @@ -2336,6 +2419,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + [[package]] name = "utf16_iter" version = "1.0.5" diff --git a/Cargo.toml b/Cargo.toml index 1f7c02e..3642381 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,6 +12,7 @@ clap = { version = "4.5.32", features = ["derive"] } roux = "2.2.14" figment = { version = "0.10", features = ["toml", "json", "env"] } serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" tokio = { version = "1.44.2", features = ["rt", "rt-multi-thread", "macros"] } regex = "1.10.3" figment_file_provider_adapter = "0.1.1" @@ -29,3 +30,6 @@ clap-verbosity-flag = "3.0.2" pretty_env_logger = "0.5.0" async-trait = "0.1.77" console = "0.15.8" +reqwest = "0.12.15" +magnet-url = "2.0.0" +urlencoding = "2.1.3" diff --git a/migrations/2025-05-01-084343_create_bitmagnet_processed/down.sql b/migrations/2025-05-01-084343_create_bitmagnet_processed/down.sql new file mode 100644 index 0000000..d9a93fe --- /dev/null +++ b/migrations/2025-05-01-084343_create_bitmagnet_processed/down.sql @@ -0,0 +1 @@ +-- This file should undo anything in `up.sql` diff --git a/migrations/2025-05-01-084343_create_bitmagnet_processed/up.sql b/migrations/2025-05-01-084343_create_bitmagnet_processed/up.sql new file mode 100644 index 0000000..eec796f --- /dev/null +++ b/migrations/2025-05-01-084343_create_bitmagnet_processed/up.sql @@ -0,0 +1,9 @@ +CREATE TABLE bitmagnet_processed +( + id INTEGER PRIMARY KEY, + magnet_id INTEGER NOT NULL, + processed_at DATETIME NOT NULL, + FOREIGN KEY (magnet_id) REFERENCES magnets(id) +); + +CREATE INDEX bitmagnet_processed_magnet_id ON bitmagnet_processed(magnet_id); \ No newline at end of file diff --git a/src/actions/bitmagnet/action.rs b/src/actions/bitmagnet/action.rs new file mode 100644 index 0000000..90a976c --- /dev/null +++ b/src/actions/bitmagnet/action.rs @@ -0,0 +1,70 @@ +use crate::actions::action::{Action, ProcessedMagnets}; +use crate::actions::bitmagnet::client::BitmagnetClient; +use crate::actions::bitmagnet::config::BitmagnetConfig; +use crate::db::{BitmagnetProcessedTable, Database}; +use color_eyre::eyre::Result; +use log::{debug, warn}; + +/// Action for submitting magnet links to Bitmagnet +pub struct BitmagnetAction { + client: BitmagnetClient, +} + +impl BitmagnetAction { + pub async fn new(config: &BitmagnetConfig) -> Result { + let client = BitmagnetClient::new(config)?; + + Ok(BitmagnetAction { client }) + } +} + +#[async_trait::async_trait] +impl Action for BitmagnetAction { + /// Return the name of the action + fn name(&self) -> &str { + "Bitmagnet" + } + + /// Process all unprocessed magnet links and return the list of processed magnets + async fn process_unprocessed_magnets(&mut self, db: &mut Database) -> Result { + let unprocessed_magnets = + db.get_unprocessed_magnets_for_table::()?; + let mut processed_magnets = Vec::new(); + let mut failed_magnets = Vec::new(); + + for magnet in unprocessed_magnets { + if let Some(id) = magnet.id { + match self + .client + .submit_magnet( + &magnet.link, + &magnet.published_at.and_utc(), + &magnet.imdb_id, + ) + .await + { + Ok(_) => { + debug!( + "Successfully submitted magnet link to Bitmagnet: {}", + magnet.title + ); + debug!("Magnet link: {}", magnet.link); + db.mark_magnet_processed_for_table::(id)?; + processed_magnets.push(magnet); + } + Err(e) => { + warn!("Failed to submit magnet link to Bitmagnet: {}", e); + failed_magnets.push(magnet); + } + } + } else { + warn!("Skipping magnet with null ID: {}", magnet.link); + } + } + + Ok(ProcessedMagnets { + success: processed_magnets, + failed: failed_magnets, + }) + } +} diff --git a/src/actions/bitmagnet/client.rs b/src/actions/bitmagnet/client.rs new file mode 100644 index 0000000..59aadf8 --- /dev/null +++ b/src/actions/bitmagnet/client.rs @@ -0,0 +1,129 @@ +use crate::actions::bitmagnet::config::BitmagnetConfig; +use chrono::{DateTime, Utc}; +use color_eyre::eyre::{eyre, Result, WrapErr}; +use log::{info, warn}; +use magnet_url::Magnet; +use reqwest::Client; +use serde_json::json; +use url::Url; +use urlencoding::decode; + +/// High-level Bitmagnet client +pub struct BitmagnetClient { + client: Client, + base_url: Url, +} + +impl BitmagnetClient { + /// Create a new Bitmagnet client from configuration + pub fn new(config: &BitmagnetConfig) -> Result { + if !config.enable { + return Err(eyre!("Bitmagnet action is disabled")); + } + + let url_str = &config.host; + let base_url = Url::parse(url_str).wrap_err_with(|| format!("Invalid URL: {}", url_str))?; + + let client = Client::new(); + + Ok(BitmagnetClient { client, base_url }) + } + + /// Submit a magnet link to Bitmagnet + pub async fn submit_magnet( + &self, + magnet: &str, + published_at: &DateTime, + imdb_id: &Option, + ) -> Result<()> { + let url = self + .base_url + .join("import") + .wrap_err("Failed to construct API URL")?; + + let structured_magnet = + Magnet::new(magnet).map_err(|e| eyre!("Invalid magnet link: {:?}", e))?; + + // Create a JSON object with the required fields + let mut json_body = json!({ + "publishedAt": published_at.to_rfc3339(), + "source": "reddit-magnet", + }); + + match structured_magnet.xt { + Some(info_hash) => { + json_body["infoHash"] = json!(info_hash); + } + None => { + return Err(eyre!("Info hash not found in magnet link")); + } + } + + if let Some(name) = structured_magnet.dn { + json_body["name"] = json!(decode(&*name)?); + } + + if let Some(id) = imdb_id { + json_body["contentSource"] = json!("imdb"); + json_body["contentId"] = json!(id); + } + + if let Some(size) = structured_magnet.xl { + json_body["size"] = json!(size); + } + + let response = self + .client + .post(url) + .json(&json_body) + .send() + .await + .wrap_err("Failed to send request to Bitmagnet API")?; + + if !response.status().is_success() { + let status = response.status(); + let error_text = response + .text() + .await + .unwrap_or_else(|_| "Unknown error".to_string()); + return Err(eyre!( + "Bitmagnet API returned error: {} - {}", + status, + error_text + )); + } + + Ok(()) + } + + /// Extract the info hash from a magnet link + fn extract_info_hash(magnet: &str) -> Result { + // Magnet links typically have the format: magnet:?xt=urn:btih:&dn=&... + let parts: Vec<&str> = magnet.split('&').collect(); + + for part in parts { + if part.starts_with("magnet:?xt=urn:btih:") { + return Ok(part[20..].to_string()); + } + if part.starts_with("xt=urn:btih:") { + return Ok(part[12..].to_string()); + } + } + + Err(eyre!("Info hash not found in magnet link")) + } + + /// Extract the name from a magnet link + fn extract_name(magnet: &str) -> Option { + // Magnet links typically have the format: magnet:?xt=urn:btih:&dn=&... + let parts: Vec<&str> = magnet.split('&').collect(); + + for part in parts { + if part.starts_with("dn=") { + return Some(part[3..].to_string()); + } + } + + None + } +} diff --git a/src/actions/bitmagnet/config.rs b/src/actions/bitmagnet/config.rs new file mode 100644 index 0000000..99634f1 --- /dev/null +++ b/src/actions/bitmagnet/config.rs @@ -0,0 +1,8 @@ +use serde::{Deserialize, Serialize}; + +/// Configuration for the Bitmagnet action +#[derive(Debug, Serialize, Deserialize)] +pub struct BitmagnetConfig { + pub enable: bool, + pub host: String, +} diff --git a/src/actions/bitmagnet/mod.rs b/src/actions/bitmagnet/mod.rs new file mode 100644 index 0000000..4a8ea7a --- /dev/null +++ b/src/actions/bitmagnet/mod.rs @@ -0,0 +1,7 @@ +mod action; +mod client; +mod config; + +pub use action::BitmagnetAction; +pub use client::BitmagnetClient; +pub use config::BitmagnetConfig; diff --git a/src/actions/mod.rs b/src/actions/mod.rs index 542d786..3674fa3 100644 --- a/src/actions/mod.rs +++ b/src/actions/mod.rs @@ -1,2 +1,3 @@ pub mod action; +pub mod bitmagnet; pub mod transmission; diff --git a/src/actions/transmission/action.rs b/src/actions/transmission/action.rs index 2871c48..fd3df00 100644 --- a/src/actions/transmission/action.rs +++ b/src/actions/transmission/action.rs @@ -51,7 +51,6 @@ impl Action for TransmissionAction { } } else { warn!("Skipping magnet with null ID: {}", magnet.link); - // Consider adding to failed_magnets if we want to report these as well } } diff --git a/src/config.rs b/src/config.rs index 1c33a37..0741960 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1,3 +1,4 @@ +use crate::actions::bitmagnet::BitmagnetConfig; use crate::actions::transmission::TransmissionConfig; use crate::args::Args; use color_eyre::eyre::{eyre, Result, WrapErr}; @@ -24,6 +25,9 @@ pub struct SourceConfig { /// Main application configuration #[derive(Debug, Serialize, Deserialize)] pub struct Config { + #[serde(default)] + pub bitmagnet: Option, + #[serde(default)] pub transmission: Option, diff --git a/src/db.rs b/src/db.rs index 462d174..6c0a3f0 100644 --- a/src/db.rs +++ b/src/db.rs @@ -1,5 +1,5 @@ -use crate::models::{Magnet, NewMagnet, NewTransmissionProcessed}; -use crate::schema::{magnets, transmission_processed}; +use crate::models::{Magnet, NewBitmagnetProcessed, NewMagnet, NewTransmissionProcessed}; +use crate::schema::{bitmagnet_processed, magnets, transmission_processed}; use crate::PostInfo; use color_eyre::eyre::{eyre, Result, WrapErr}; use diesel::prelude::*; @@ -16,8 +16,33 @@ pub trait ProcessedTable { fn mark_processed(conn: &mut SqliteConnection, magnet_id: i32) -> Result<()>; } +pub struct BitmagnetProcessedTable; pub struct TransmissionProcessedTable; +impl ProcessedTable for BitmagnetProcessedTable { + fn get_processed_ids(conn: &mut SqliteConnection) -> Result> { + bitmagnet_processed::table + .select(bitmagnet_processed::magnet_id) + .load(conn) + .wrap_err("Failed to load processed magnet IDs for Bitmagnet") + } + + fn mark_processed(conn: &mut SqliteConnection, magnet_id: i32) -> Result<()> { + let now = chrono::Utc::now().naive_utc(); + let new_processed = NewBitmagnetProcessed { + magnet_id, + processed_at: &now, + }; + + diesel::insert_into(bitmagnet_processed::table) + .values(&new_processed) + .execute(conn) + .wrap_err("Failed to mark magnet as processed by Bitmagnet")?; + + Ok(()) + } +} + impl ProcessedTable for TransmissionProcessedTable { fn get_processed_ids(conn: &mut SqliteConnection) -> Result> { transmission_processed::table diff --git a/src/main.rs b/src/main.rs index 818c7d6..cdb306b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,4 +1,5 @@ use crate::actions::action::Action; +use crate::actions::bitmagnet::BitmagnetAction; use crate::actions::transmission::TransmissionAction; use crate::args::Args; use crate::config::{get_db_path, load_config}; @@ -133,6 +134,25 @@ async fn main() -> Result<()> { // Initialize actions let mut actions: Vec> = Vec::new(); + // Add Bitmagnet action if enabled + if let Some(bitmagnet_config) = conf.bitmagnet { + if bitmagnet_config.enable { + info!("Initializing Bitmagnet action"); + match BitmagnetAction::new(&bitmagnet_config).await { + Ok(bitmagnet_action) => { + actions.push(Box::new(bitmagnet_action)); + } + Err(e) => { + warn!("Failed to initialize Bitmagnet action: {}", e); + } + } + } else { + debug!("Bitmagnet action is disabled"); + } + } else { + debug!("No Bitmagnet configuration found"); + } + // Add Transmission action if enabled if let Some(transmission_config) = conf.transmission { if transmission_config.enable { diff --git a/src/models.rs b/src/models.rs index 284bbcc..c7aca1c 100644 --- a/src/models.rs +++ b/src/models.rs @@ -1,4 +1,4 @@ -use crate::schema::{magnets, transmission_processed}; +use crate::schema::{bitmagnet_processed, magnets, transmission_processed}; use chrono::NaiveDateTime; use diesel::prelude::*; @@ -27,6 +27,23 @@ pub struct NewMagnet<'a> { pub imdb_id: Option<&'a str>, } +#[derive(Queryable, Selectable)] +#[diesel(table_name = bitmagnet_processed)] +#[diesel(check_for_backend(diesel::sqlite::Sqlite))] +pub struct BitmagnetProcessed { + pub id: Option, + pub magnet_id: i32, + pub processed_at: NaiveDateTime, +} + +#[derive(Insertable)] +#[diesel(table_name = bitmagnet_processed)] +#[diesel(check_for_backend(diesel::sqlite::Sqlite))] +pub struct NewBitmagnetProcessed<'a> { + pub magnet_id: i32, + pub processed_at: &'a NaiveDateTime, +} + #[derive(Queryable, Selectable)] #[diesel(table_name = transmission_processed)] #[diesel(check_for_backend(diesel::sqlite::Sqlite))] diff --git a/src/schema.rs b/src/schema.rs index a129276..710cd35 100644 --- a/src/schema.rs +++ b/src/schema.rs @@ -1,5 +1,13 @@ // @generated automatically by Diesel CLI. +diesel::table! { + bitmagnet_processed (id) { + id -> Nullable, + magnet_id -> Integer, + processed_at -> Timestamp, + } +} + diesel::table! { magnets (id) { id -> Nullable, @@ -20,6 +28,7 @@ diesel::table! { } } +diesel::joinable!(bitmagnet_processed -> magnets (magnet_id)); diesel::joinable!(transmission_processed -> magnets (magnet_id)); -diesel::allow_tables_to_appear_in_same_query!(magnets, transmission_processed,); +diesel::allow_tables_to_appear_in_same_query!(bitmagnet_processed, magnets, transmission_processed,);