Skip to content
Snippets Groups Projects
Commit c9039680 authored by insert's avatar insert
Browse files

Add user agent.

parent 5c53ad2c
Branches
No related merge requests found
......@@ -58,7 +58,7 @@ dependencies = [
"serde",
"sha-1",
"smallvec",
"time",
"time 0.2.26",
"tokio",
]
......@@ -168,7 +168,7 @@ dependencies = [
"actix-web-codegen",
"ahash",
"bytes",
"cookie",
"cookie 0.15.0",
"derive_more",
"either",
"encoding_rs",
......@@ -186,7 +186,7 @@ dependencies = [
"serde_urlencoded",
"smallvec",
"socket2",
"time",
"time 0.2.26",
"url",
]
......@@ -227,6 +227,136 @@ dependencies = [
"memchr",
]
[[package]]
name = "async-attributes"
version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a3203e79f4dd9bdda415ed03cf14dae5a2bf775c683a00f94e9cd1faf0f596e5"
dependencies = [
"quote",
"syn",
]
[[package]]
name = "async-channel"
version = "1.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2114d64672151c0c5eaa5e131ec84a74f06e1e559830dabba01ca30605d66319"
dependencies = [
"concurrent-queue",
"event-listener",
"futures-core",
]
[[package]]
name = "async-executor"
version = "1.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "871f9bb5e0a22eeb7e8cf16641feb87c9dc67032ccf8ff49e772eb9941d3a965"
dependencies = [
"async-task",
"concurrent-queue",
"fastrand",
"futures-lite",
"once_cell",
"slab",
]
[[package]]
name = "async-global-executor"
version = "2.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9586ec52317f36de58453159d48351bc244bc24ced3effc1fce22f3d48664af6"
dependencies = [
"async-channel",
"async-executor",
"async-io",
"async-mutex",
"blocking",
"futures-lite",
"num_cpus",
"once_cell",
"tokio",
]
[[package]]
name = "async-io"
version = "1.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4bbfd5cf2794b1e908ea8457e6c45f8f8f1f6ec5f74617bf4662623f47503c3b"
dependencies = [
"concurrent-queue",
"fastrand",
"futures-lite",
"libc",
"log",
"once_cell",
"parking",
"polling",
"slab",
"socket2",
"waker-fn",
"winapi",
]
[[package]]
name = "async-lock"
version = "2.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e6a8ea61bf9947a1007c5cada31e647dbc77b103c679858150003ba697ea798b"
dependencies = [
"event-listener",
]
[[package]]
name = "async-mutex"
version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "479db852db25d9dbf6204e6cb6253698f175c15726470f78af0d918e99d6156e"
dependencies = [
"event-listener",
]
[[package]]
name = "async-std"
version = "1.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d9f06685bad74e0570f5213741bea82158279a4103d988e57bfada11ad230341"
dependencies = [
"async-attributes",
"async-channel",
"async-global-executor",
"async-io",
"async-lock",
"crossbeam-utils",
"futures-channel",
"futures-core",
"futures-io",
"futures-lite",
"gloo-timers",
"kv-log-macro",
"log",
"memchr",
"num_cpus",
"once_cell",
"pin-project-lite",
"pin-utils",
"slab",
"wasm-bindgen-futures",
]
[[package]]
name = "async-task"
version = "4.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e91831deabf0d6d7ec49552e489aed63b7456a7a3c46cff62adad428110b0af0"
[[package]]
name = "atomic-waker"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "065374052e7df7ee4047b1160cca5e1467a12351a40b3da123c870ba0b8eda2a"
[[package]]
name = "autocfg"
version = "1.0.1"
......@@ -260,6 +390,20 @@ dependencies = [
"generic-array",
]
[[package]]
name = "blocking"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c5e170dbede1f740736619b776d7251cb1b9095c435c34d8ca9f57fcd2f335e9"
dependencies = [
"async-channel",
"async-task",
"atomic-waker",
"fastrand",
"futures-lite",
"once_cell",
]
[[package]]
name = "brotli-sys"
version = "0.3.2"
......@@ -307,6 +451,12 @@ dependencies = [
"bytes",
]
[[package]]
name = "cache-padded"
version = "1.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "631ae5198c9be5e753e5cc215e1bd73c2b466a3565173db433f52bb9d3e66dba"
[[package]]
name = "cc"
version = "1.0.67"
......@@ -319,6 +469,28 @@ version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "chrono"
version = "0.4.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73"
dependencies = [
"libc",
"num-integer",
"num-traits",
"time 0.1.44",
"winapi",
]
[[package]]
name = "concurrent-queue"
version = "1.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "30ed07550be01594c6026cff2a1d7fe9c8f683caa798e12b68694ac9e88286a3"
dependencies = [
"cache-padded",
]
[[package]]
name = "const_fn"
version = "0.4.7"
......@@ -331,6 +503,17 @@ version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e"
[[package]]
name = "cookie"
version = "0.14.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "03a5d7b21829bc7b4bf4754a978a241ae54ea55a40f92bb20216e54096f4b951"
dependencies = [
"percent-encoding",
"time 0.2.26",
"version_check",
]
[[package]]
name = "cookie"
version = "0.15.0"
......@@ -338,10 +521,26 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ffdf8865bac3d9a3bde5bde9088ca431b11f5d37c7a578b8086af77248b76627"
dependencies = [
"percent-encoding",
"time",
"time 0.2.26",
"version_check",
]
[[package]]
name = "cookie_store"
version = "0.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3818dfca4b0cb5211a659bbcbb94225b7127407b2b135e650d717bfb78ab10d3"
dependencies = [
"cookie 0.14.4",
"idna",
"log",
"publicsuffix",
"serde",
"serde_json",
"time 0.2.26",
"url",
]
[[package]]
name = "core-foundation"
version = "0.9.1"
......@@ -373,6 +572,17 @@ dependencies = [
"cfg-if",
]
[[package]]
name = "crossbeam-utils"
version = "0.8.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4feb231f0d4d6af81aed15928e58ecf5816aa62a2393e2c82f46973e92a9a278"
dependencies = [
"autocfg",
"cfg-if",
"lazy_static",
]
[[package]]
name = "cssparser"
version = "0.27.2"
......@@ -400,6 +610,16 @@ dependencies = [
"syn",
]
[[package]]
name = "ctor"
version = "0.1.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5e98e2ad1a782e33928b96fc3948e7c355e5af34ba4de7670fe8bac2a3b2006d"
dependencies = [
"quote",
"syn",
]
[[package]]
name = "derive_more"
version = "0.99.13"
......@@ -463,6 +683,21 @@ dependencies = [
"cfg-if",
]
[[package]]
name = "event-listener"
version = "2.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f7531096570974c3a9dcf9e4b8e1cede1ec26cf5046219fb3b9d897503b9be59"
[[package]]
name = "fastrand"
version = "1.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "77b705829d1e87f762c2df6da140b26af5839e1033aa84aa5f56bb688e4e1bdb"
dependencies = [
"instant",
]
[[package]]
name = "flate2"
version = "1.0.20"
......@@ -531,6 +766,27 @@ version = "0.3.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "098cd1c6dda6ca01650f1a37a794245eb73181d0d4d4e955e2f3c37db7af1815"
[[package]]
name = "futures-io"
version = "0.3.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "365a1a1fb30ea1c03a830fdb2158f5236833ac81fa0ad12fe35b29cddc35cb04"
[[package]]
name = "futures-lite"
version = "1.11.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b4481d0cd0de1d204a4fa55e7d45f07b1d958abcb06714b3446438e2eff695fb"
dependencies = [
"fastrand",
"futures-core",
"futures-io",
"memchr",
"parking",
"pin-project-lite",
"waker-fn",
]
[[package]]
name = "futures-sink"
version = "0.3.14"
......@@ -603,7 +859,20 @@ checksum = "c9495705279e7140bf035dde1f6e750c162df8b625267cd52cc44e0b156732c8"
dependencies = [
"cfg-if",
"libc",
"wasi 0.10.2+wasi-snapshot-preview1",
"wasi 0.10.0+wasi-snapshot-preview1",
]
[[package]]
name = "gloo-timers"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "47204a46aaff920a1ea58b11d03dec6f704287d27561724a4631e450654a891f"
dependencies = [
"futures-channel",
"futures-core",
"js-sys",
"wasm-bindgen",
"web-sys",
]
[[package]]
......@@ -779,11 +1048,13 @@ version = "0.1.0"
dependencies = [
"actix-web",
"imagesize",
"lazy_static",
"mime",
"reqwest",
"scraper",
"serde",
"serde_json",
"ytextract",
]
[[package]]
......@@ -795,6 +1066,15 @@ dependencies = [
"wasm-bindgen",
]
[[package]]
name = "kv-log-macro"
version = "1.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0de8b303297635ad57c9f5059fd9cee7a47f8e8daa09df0fcd07dd39fb22977f"
dependencies = [
"log",
]
[[package]]
name = "language-tags"
version = "0.2.2"
......@@ -847,6 +1127,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710"
dependencies = [
"cfg-if",
"value-bag",
]
[[package]]
......@@ -958,6 +1239,25 @@ dependencies = [
"winapi",
]
[[package]]
name = "num-integer"
version = "0.1.44"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db"
dependencies = [
"autocfg",
"num-traits",
]
[[package]]
name = "num-traits"
version = "0.2.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290"
dependencies = [
"autocfg",
]
[[package]]
name = "num_cpus"
version = "1.13.0"
......@@ -1013,6 +1313,12 @@ dependencies = [
"vcpkg",
]
[[package]]
name = "parking"
version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "427c3892f9e783d91cc128285287e70a59e206ca452770ece88a76f7a3eddd72"
[[package]]
name = "parking_lot"
version = "0.11.1"
......@@ -1142,6 +1448,19 @@ version = "0.3.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c"
[[package]]
name = "polling"
version = "2.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4fc12d774e799ee9ebae13f4076ca003b40d18a11ac0f3641e6f899618580b7b"
dependencies = [
"cfg-if",
"libc",
"log",
"wepoll-sys",
"winapi",
]
[[package]]
name = "ppv-lite86"
version = "0.2.10"
......@@ -1169,6 +1488,16 @@ dependencies = [
"unicode-xid",
]
[[package]]
name = "publicsuffix"
version = "1.5.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "95b4ce31ff0a27d93c8de1849cf58162283752f065a90d508f1105fa6c9a213f"
dependencies = [
"idna",
"url",
]
[[package]]
name = "quote"
version = "1.0.9"
......@@ -1312,6 +1641,8 @@ checksum = "2296f2fac53979e8ccbc4a1136b25dcefd37be9ed7e4a1f6b05a6029c84ff124"
dependencies = [
"base64",
"bytes",
"cookie 0.14.4",
"cookie_store",
"encoding_rs",
"futures-core",
"futures-util",
......@@ -1329,6 +1660,7 @@ dependencies = [
"pin-project-lite",
"serde",
"serde_urlencoded",
"time 0.2.26",
"tokio",
"tokio-native-tls",
"url",
......@@ -1683,6 +2015,37 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8eaa81235c7058867fa8c0e7314f33dcce9c215f535d1913822a2b3f5e289f3c"
[[package]]
name = "thiserror"
version = "1.0.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e0f4a65597094d4483ddaed134f409b2cb7c1beccf25201a9f73c719254fa98e"
dependencies = [
"thiserror-impl",
]
[[package]]
name = "thiserror-impl"
version = "1.0.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7765189610d8241a44529806d6fd1f2e0a08734313a35d5b3a556f92b381f3c0"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "time"
version = "0.1.44"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255"
dependencies = [
"libc",
"wasi 0.10.0+wasi-snapshot-preview1",
"winapi",
]
[[package]]
name = "time"
version = "0.2.26"
......@@ -1747,6 +2110,7 @@ dependencies = [
"libc",
"memchr",
"mio",
"num_cpus",
"once_cell",
"parking_lot",
"pin-project-lite",
......@@ -1864,6 +2228,15 @@ version = "0.7.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9"
[[package]]
name = "value-bag"
version = "1.0.0-alpha.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6b676010e055c99033117c2343b33a40a30b91fecd6c49055ac9cd2d6c305ab1"
dependencies = [
"ctor",
]
[[package]]
name = "vcpkg"
version = "0.2.12"
......@@ -1876,6 +2249,12 @@ version = "0.9.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5fecdca9a5291cc2b8dcf7dc02453fee791a280f3743cb0905f8822ae463b3fe"
[[package]]
name = "waker-fn"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9d5b2c62b4012a3e1eca5a7e077d13b3bf498c4073e33ccd58626607748ceeca"
[[package]]
name = "want"
version = "0.3.0"
......@@ -1894,9 +2273,9 @@ checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519"
[[package]]
name = "wasi"
version = "0.10.2+wasi-snapshot-preview1"
version = "0.10.0+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6"
checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f"
[[package]]
name = "wasm-bindgen"
......@@ -1976,6 +2355,15 @@ dependencies = [
"wasm-bindgen",
]
[[package]]
name = "wepoll-sys"
version = "3.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0fcb14dea929042224824779fbc82d9fab8d2e6d3cbc0ac404de8edf489e77ff"
dependencies = [
"cc",
]
[[package]]
name = "winapi"
version = "0.3.9"
......@@ -2006,3 +2394,19 @@ checksum = "0120db82e8a1e0b9fb3345a539c478767c0048d842860994d96113d5b667bd69"
dependencies = [
"winapi",
]
[[package]]
name = "ytextract"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c30f2482efb40ca7de76dbff24fe554b5d4ac5e526a213962517312f3b69bcb"
dependencies = [
"async-std",
"chrono",
"once_cell",
"regex",
"reqwest",
"serde",
"serde_json",
"thiserror",
]
......@@ -7,7 +7,9 @@ edition = "2018"
[dependencies]
actix-web = "4.0.0-beta.6"
lazy_static = "1.4.0"
imagesize = "0.8.8"
ytextract = "0.1.0"
reqwest = "0.11.3"
scraper = "0.12.0"
serde_json = "1"
......
......@@ -13,6 +13,8 @@ Image proxy and metadata scraper.
- Use LRU cache for data. [See uluru](https://github.com/servo/uluru)
- Full support for OpenGraph. [See ogp.me](https://ogp.me)
- Full support for Twitter cards. See [a](https://developer.twitter.com/en/docs/twitter-for-websites/cards/overview/summary) and [b](https://developer.twitter.com/en/docs/twitter-for-websites/cards/overview/summary-card-with-large-image).
- Add max length for strings from meta tags.
- Support theme-color.
## .
......
#!/bin/bash
export version=0.1.0-patch.0
export version=0.1.1
# echo "pub const VERSION: &str = \"${version}\";" > src/version.rs
#[macro_use]
extern crate lazy_static;
use actix_web::{web, App, HttpServer};
pub mod routes;
......
......@@ -27,7 +27,7 @@ pub async fn get(info: Query<Parameters>) -> Result<impl Responder, Error> {
metadata.resolve_external().await;
if metadata.is_none() {
return Ok(web::Json(Embed::None))
return Ok(web::Json(Embed::None));
}
Ok(web::Json(Embed::Website(metadata)))
......
use actix_web::{HttpResponse, Responder, web::Query};
use actix_web::{web::Query, HttpResponse, Responder};
use serde::Deserialize;
use crate::util::request::fetch;
......@@ -14,11 +14,11 @@ pub async fn get(info: Query<Parameters>) -> Result<impl Responder, Error> {
let (resp, mime) = fetch(&url).await?;
if let mime::IMAGE = mime.type_() {
let body = resp.bytes().await.map_err(|_| Error::FailedToConsumeBytes)?;
Ok(
HttpResponse::Ok()
.body(body)
)
let body = resp
.bytes()
.await
.map_err(|_| Error::FailedToConsumeBytes)?;
Ok(HttpResponse::Ok().body(body))
} else {
Err(Error::NotAllowedToProxy)
}
......
......@@ -10,8 +10,11 @@ use super::media::{Media, MediaSize};
#[derive(Debug, Serialize)]
pub struct Metadata {
#[serde(skip_serializing_if = "Option::is_none")]
title: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
description: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
image: Option<Media>,
}
......
use std::collections::HashMap;
use mime::Mime;
use reqwest::{header::CONTENT_TYPE, Response};
use reqwest::{header::CONTENT_TYPE, Client, ClientBuilder, Response};
use scraper::{Html, Selector};
use super::result::Error;
lazy_static! {
static ref CLIENT: Client = reqwest::Client::builder()
.user_agent(
"Mozilla/5.0 (compatible; January/1.0; +https://gitlab.insrt.uk/revolt/january)"
)
.build()
.unwrap();
}
pub async fn fetch(url: &str) -> Result<(Response, Mime), Error> {
let resp = reqwest::get(url).await.map_err(|_| Error::ReqwestFailed)?;
let resp = CLIENT
.get(url)
.send()
.await
.map_err(|_| Error::ReqwestFailed)?;
if !resp.status().is_success() {
dbg!(resp.text().await.unwrap());
return Err(Error::RequestFailed);
}
......@@ -19,8 +32,10 @@ pub async fn fetch(url: &str) -> Result<(Response, Mime), Error> {
.ok_or_else(|| Error::MissingContentType)?
.to_str()
.map_err(|_| Error::ConversionFailed)?;
let mime: mime::Mime = content_type.parse().map_err(|_| Error::FailedToParseContentType)?;
let mime: mime::Mime = content_type
.parse()
.map_err(|_| Error::FailedToParseContentType)?;
Ok((resp, mime))
}
......@@ -42,7 +57,10 @@ pub async fn consume_metatags(resp: Response) -> Result<HashMap<String, String>,
}
pub async fn consume_size(resp: Response) -> Result<(isize, isize), Error> {
let bytes = resp.bytes().await.map_err(|_| Error::FailedToConsumeBytes)?;
let bytes = resp
.bytes()
.await
.map_err(|_| Error::FailedToConsumeBytes)?;
if let Ok(size) = imagesize::blob_size(&bytes) {
Ok((size.width as isize, size.height as isize))
} else {
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment