Use a structured magnet URL type
This will be required to parse them and submit them to bitmagnet.
This commit is contained in:
parent
90c3fc5ee3
commit
70684eb2a2
5 changed files with 70 additions and 32 deletions
11
Cargo.lock
generated
11
Cargo.lock
generated
|
|
@ -1154,6 +1154,16 @@ version = "0.4.27"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94"
|
checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "magnet-url"
|
||||||
|
version = "2.0.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "ac3b4c4004e88aca00cc0c60782e5642c8fc628deca19e530ce58aa76e737d74"
|
||||||
|
dependencies = [
|
||||||
|
"lazy_static",
|
||||||
|
"regex",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "maybe-async"
|
name = "maybe-async"
|
||||||
version = "0.2.10"
|
version = "0.2.10"
|
||||||
|
|
@ -1538,6 +1548,7 @@ dependencies = [
|
||||||
"figment",
|
"figment",
|
||||||
"figment_file_provider_adapter",
|
"figment_file_provider_adapter",
|
||||||
"log",
|
"log",
|
||||||
|
"magnet-url",
|
||||||
"multimap",
|
"multimap",
|
||||||
"pretty_env_logger",
|
"pretty_env_logger",
|
||||||
"regex",
|
"regex",
|
||||||
|
|
|
||||||
|
|
@ -29,3 +29,4 @@ clap-verbosity-flag = "3.0.2"
|
||||||
pretty_env_logger = "0.5.0"
|
pretty_env_logger = "0.5.0"
|
||||||
async-trait = "0.1.77"
|
async-trait = "0.1.77"
|
||||||
console = "0.15.8"
|
console = "0.15.8"
|
||||||
|
magnet-url = "2.0.0"
|
||||||
|
|
|
||||||
25
src/db.rs
25
src/db.rs
|
|
@ -5,6 +5,7 @@ use color_eyre::eyre::{eyre, Result, WrapErr};
|
||||||
use diesel::prelude::*;
|
use diesel::prelude::*;
|
||||||
use diesel::sqlite::SqliteConnection;
|
use diesel::sqlite::SqliteConnection;
|
||||||
use diesel_migrations::{embed_migrations, EmbeddedMigrations, MigrationHarness};
|
use diesel_migrations::{embed_migrations, EmbeddedMigrations, MigrationHarness};
|
||||||
|
use magnet_url::Magnet as MagnetUrl;
|
||||||
use std::fs::create_dir_all;
|
use std::fs::create_dir_all;
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
|
|
||||||
|
|
@ -85,19 +86,21 @@ impl Database {
|
||||||
// Filter out magnet links that already exist in the database
|
// Filter out magnet links that already exist in the database
|
||||||
let existing_links: Vec<String> = magnets::table
|
let existing_links: Vec<String> = magnets::table
|
||||||
.select(magnets::link)
|
.select(magnets::link)
|
||||||
.filter(magnets::link.eq_any(&post.magnet_links))
|
.filter(
|
||||||
|
magnets::link.eq_any(&post.magnet_links.iter().map(|m| m.to_string()).collect::<Vec<_>>()),
|
||||||
|
)
|
||||||
.load(&mut self.conn)
|
.load(&mut self.conn)
|
||||||
.wrap_err("Failed to query existing magnets")?;
|
.wrap_err("Failed to query existing magnets")?;
|
||||||
|
|
||||||
let links = post
|
let links = post
|
||||||
.magnet_links
|
.magnet_links
|
||||||
.iter()
|
.iter()
|
||||||
.filter(|link| !existing_links.contains(link))
|
.filter(|link| !existing_links.contains(&link.to_string()))
|
||||||
.map(|m| NewMagnet {
|
.map(|m| NewMagnet {
|
||||||
title: post.title.as_str(),
|
title: post.title.as_str(),
|
||||||
submitter: post.submitter.as_str(),
|
submitter: post.submitter.as_str(),
|
||||||
subreddit: post.subreddit.as_str(),
|
subreddit: post.subreddit.as_str(),
|
||||||
link: m,
|
link: m.to_string().as_str(),
|
||||||
published_at: &published_at,
|
published_at: &published_at,
|
||||||
})
|
})
|
||||||
.collect::<Vec<NewMagnet>>();
|
.collect::<Vec<NewMagnet>>();
|
||||||
|
|
@ -163,8 +166,8 @@ mod tests {
|
||||||
submitter: "test_user".to_string(),
|
submitter: "test_user".to_string(),
|
||||||
subreddit: "test_subreddit".to_string(),
|
subreddit: "test_subreddit".to_string(),
|
||||||
magnet_links: vec![
|
magnet_links: vec![
|
||||||
"magnet:?xt=urn:btih:test1".to_string(),
|
MagnetUrl::new_no_validation("magnet:?xt=urn:btih:test1"),
|
||||||
"magnet:?xt=urn:btih:test2".to_string(),
|
MagnetUrl::new_no_validation("magnet:?xt=urn:btih:test2"),
|
||||||
],
|
],
|
||||||
timestamp: Utc::now(),
|
timestamp: Utc::now(),
|
||||||
};
|
};
|
||||||
|
|
@ -201,8 +204,8 @@ mod tests {
|
||||||
submitter: "test_user".to_string(),
|
submitter: "test_user".to_string(),
|
||||||
subreddit: "test_subreddit".to_string(),
|
subreddit: "test_subreddit".to_string(),
|
||||||
magnet_links: vec![
|
magnet_links: vec![
|
||||||
"magnet:?xt=urn:btih:test1".to_string(),
|
MagnetUrl::new_no_validation("magnet:?xt=urn:btih:test1"),
|
||||||
"magnet:?xt=urn:btih:test2".to_string(),
|
MagnetUrl::new_no_validation("magnet:?xt=urn:btih:test2"),
|
||||||
],
|
],
|
||||||
timestamp: Utc::now(),
|
timestamp: Utc::now(),
|
||||||
};
|
};
|
||||||
|
|
@ -218,8 +221,8 @@ mod tests {
|
||||||
submitter: "test_user2".to_string(),
|
submitter: "test_user2".to_string(),
|
||||||
subreddit: "test_subreddit2".to_string(),
|
subreddit: "test_subreddit2".to_string(),
|
||||||
magnet_links: vec![
|
magnet_links: vec![
|
||||||
"magnet:?xt=urn:btih:test1".to_string(), // Duplicate
|
MagnetUrl::new_no_validation("magnet:?xt=urn:btih:test1"), // Duplicate
|
||||||
"magnet:?xt=urn:btih:test3".to_string(), // New
|
MagnetUrl::new_no_validation("magnet:?xt=urn:btih:test3"), // New
|
||||||
],
|
],
|
||||||
timestamp: Utc::now(),
|
timestamp: Utc::now(),
|
||||||
};
|
};
|
||||||
|
|
@ -239,8 +242,8 @@ mod tests {
|
||||||
submitter: "test_user3".to_string(),
|
submitter: "test_user3".to_string(),
|
||||||
subreddit: "test_subreddit3".to_string(),
|
subreddit: "test_subreddit3".to_string(),
|
||||||
magnet_links: vec![
|
magnet_links: vec![
|
||||||
"magnet:?xt=urn:btih:test1".to_string(), // Duplicate
|
MagnetUrl::new_no_validation("magnet:?xt=urn:btih:test1"), // Duplicate
|
||||||
"magnet:?xt=urn:btih:test2".to_string(), // Duplicate
|
MagnetUrl::new_no_validation("magnet:?xt=urn:btih:test2"), // Duplicate
|
||||||
],
|
],
|
||||||
timestamp: Utc::now(),
|
timestamp: Utc::now(),
|
||||||
};
|
};
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,8 @@
|
||||||
pub type Magnet = String;
|
use color_eyre::eyre::{eyre, Result};
|
||||||
|
use magnet_url::Magnet;
|
||||||
|
|
||||||
/// Extract magnet links from text
|
/// Extract magnet links from text
|
||||||
pub fn extract_magnet_links(text: &str) -> Vec<Magnet> {
|
pub fn extract_magnet_links(text: &str) -> Result<Vec<Magnet>> {
|
||||||
let mut links = Vec::new();
|
let mut links = Vec::new();
|
||||||
|
|
||||||
let mut start_idx = 0;
|
let mut start_idx = 0;
|
||||||
|
|
@ -13,11 +14,14 @@ pub fn extract_magnet_links(text: &str) -> Vec<Magnet> {
|
||||||
.map(|e| start + e)
|
.map(|e| start + e)
|
||||||
.unwrap_or(text.len());
|
.unwrap_or(text.len());
|
||||||
|
|
||||||
links.push(text[start..end].to_string());
|
let link = &text[start..end];
|
||||||
|
let magnet = Magnet::new(link)
|
||||||
|
.map_err(|e| eyre!("Failed to parse magnet link '{}': {:?}", link, e))?;
|
||||||
|
links.push(magnet);
|
||||||
start_idx = end;
|
start_idx = end;
|
||||||
}
|
}
|
||||||
|
|
||||||
links
|
Ok(links)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
|
|
@ -30,7 +34,7 @@ mod tests {
|
||||||
|
|
||||||
let links = extract_magnet_links(text);
|
let links = extract_magnet_links(text);
|
||||||
|
|
||||||
assert!(links.is_empty());
|
assert!(links.unwrap().is_empty());
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|
@ -39,67 +43,85 @@ mod tests {
|
||||||
|
|
||||||
let links = extract_magnet_links(text);
|
let links = extract_magnet_links(text);
|
||||||
|
|
||||||
assert!(links.is_empty());
|
assert!(links.unwrap().is_empty());
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_single_magnet_link() {
|
fn test_single_magnet_link() {
|
||||||
let text = "Here is a magnet link: magnet:?xt=urn:btih:example";
|
let text = "Here is a magnet link: magnet:?xt=urn:btih:example";
|
||||||
|
|
||||||
let links = extract_magnet_links(text);
|
let links = extract_magnet_links(text).unwrap();
|
||||||
|
|
||||||
assert_eq!(links.len(), 1);
|
assert_eq!(links.len(), 1);
|
||||||
assert_eq!(links[0], "magnet:?xt=urn:btih:example");
|
assert_eq!(
|
||||||
|
links[0],
|
||||||
|
Magnet::new_no_validation("magnet:?xt=urn:btih:example")
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_multiple_magnet_links() {
|
fn test_multiple_magnet_links() {
|
||||||
let text = "First link: magnet:?xt=urn:btih:example1 and second link: magnet:?xt=urn:btih:example2";
|
let text = "First link: magnet:?xt=urn:btih:example1 and second link: magnet:?xt=urn:btih:example2";
|
||||||
|
|
||||||
let links = extract_magnet_links(text);
|
let links = extract_magnet_links(text).unwrap();
|
||||||
|
|
||||||
assert_eq!(links.len(), 2);
|
assert_eq!(links.len(), 2);
|
||||||
assert_eq!(links[0], "magnet:?xt=urn:btih:example1");
|
assert_eq!(
|
||||||
assert_eq!(links[1], "magnet:?xt=urn:btih:example2");
|
links[0],
|
||||||
|
Magnet::new_no_validation("magnet:?xt=urn:btih:example1")
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
links[1],
|
||||||
|
Magnet::new_no_validation("magnet:?xt=urn:btih:example2")
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_magnet_link_at_beginning() {
|
fn test_magnet_link_at_beginning() {
|
||||||
let text = "magnet:?xt=urn:btih:example at the beginning";
|
let text = "magnet:?xt=urn:btih:example at the beginning";
|
||||||
|
|
||||||
let links = extract_magnet_links(text);
|
let links = extract_magnet_links(text).unwrap();
|
||||||
|
|
||||||
assert_eq!(links.len(), 1);
|
assert_eq!(links.len(), 1);
|
||||||
assert_eq!(links[0], "magnet:?xt=urn:btih:example");
|
assert_eq!(
|
||||||
|
links[0],
|
||||||
|
Magnet::new_no_validation("magnet:?xt=urn:btih:example")
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_magnet_link_at_end() {
|
fn test_magnet_link_at_end() {
|
||||||
let text = "Link at the end: magnet:?xt=urn:btih:example";
|
let text = "Link at the end: magnet:?xt=urn:btih:example";
|
||||||
|
|
||||||
let links = extract_magnet_links(text);
|
let links = extract_magnet_links(text).unwrap();
|
||||||
|
|
||||||
assert_eq!(links.len(), 1);
|
assert_eq!(links.len(), 1);
|
||||||
assert_eq!(links[0], "magnet:?xt=urn:btih:example");
|
assert_eq!(
|
||||||
|
links[0],
|
||||||
|
Magnet::new_no_validation("magnet:?xt=urn:btih:example")
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_magnet_link_without_whitespace() {
|
fn test_magnet_link_without_whitespace() {
|
||||||
let text = "Text containing a link:magnet:?xt=urn:btih:example";
|
let text = "Text containing a link:magnet:?xt=urn:btih:example";
|
||||||
|
|
||||||
let links = extract_magnet_links(text);
|
let links = extract_magnet_links(text).unwrap();
|
||||||
|
|
||||||
assert_eq!(links.len(), 1);
|
assert_eq!(links.len(), 1);
|
||||||
assert_eq!(links[0], "magnet:?xt=urn:btih:example");
|
assert_eq!(
|
||||||
|
links[0],
|
||||||
|
Magnet::new_no_validation("magnet:?xt=urn:btih:example")
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_complex_magnet_link() {
|
fn test_complex_magnet_link() {
|
||||||
let text = "Complex link: magnet:?xt=urn:btih:a1b2c3d4e5f6g7h8i9j0&dn=example+file&tr=udp%3A%2F%2Ftracker.example.com%3A80";
|
let text = "Complex link: magnet:?xt=urn:btih:a1b2c3d4e5f6g7h8i9j0&dn=example+file&tr=udp%3A%2F%2Ftracker.example.com%3A80";
|
||||||
|
|
||||||
let links = extract_magnet_links(text);
|
let links = extract_magnet_links(text).unwrap();
|
||||||
|
|
||||||
assert_eq!(links.len(), 1);
|
assert_eq!(links.len(), 1);
|
||||||
assert_eq!(links[0], "magnet:?xt=urn:btih:a1b2c3d4e5f6g7h8i9j0&dn=example+file&tr=udp%3A%2F%2Ftracker.example.com%3A80");
|
assert_eq!(links[0], Magnet::new_no_validation("magnet:?xt=urn:btih:a1b2c3d4e5f6g7h8i9j0&dn=example+file&tr=udp%3A%2F%2Ftracker.example.com%3A80"));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -3,11 +3,12 @@ use crate::actions::transmission::TransmissionAction;
|
||||||
use crate::args::Args;
|
use crate::args::Args;
|
||||||
use crate::config::{get_db_path, load_config};
|
use crate::config::{get_db_path, load_config};
|
||||||
use crate::db::Database;
|
use crate::db::Database;
|
||||||
use crate::magnet::{extract_magnet_links, Magnet};
|
use crate::magnet::extract_magnet_links;
|
||||||
use chrono::{DateTime, Utc};
|
use chrono::{DateTime, Utc};
|
||||||
use clap::Parser;
|
use clap::Parser;
|
||||||
use color_eyre::eyre::{eyre, Result, WrapErr};
|
use color_eyre::eyre::{eyre, Result, WrapErr};
|
||||||
use log::{debug, info, warn};
|
use log::{debug, info, warn};
|
||||||
|
use magnet_url::Magnet;
|
||||||
use multimap::MultiMap;
|
use multimap::MultiMap;
|
||||||
use reddit_client::RedditClient;
|
use reddit_client::RedditClient;
|
||||||
use regex::Regex;
|
use regex::Regex;
|
||||||
|
|
@ -104,7 +105,7 @@ async fn main() -> Result<()> {
|
||||||
let body = &post.body;
|
let body = &post.body;
|
||||||
let subreddit = &post.subreddit;
|
let subreddit = &post.subreddit;
|
||||||
|
|
||||||
let magnet_links = extract_magnet_links(body);
|
let magnet_links = extract_magnet_links(body)?;
|
||||||
if !magnet_links.is_empty() {
|
if !magnet_links.is_empty() {
|
||||||
let post_info = PostInfo {
|
let post_info = PostInfo {
|
||||||
title: title.to_string(),
|
title: title.to_string(),
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue