Skip to content

Commit

Permalink
feature/from_archives: MVP
Browse files Browse the repository at this point in the history
  • Loading branch information
kingwingfly committed Jun 18, 2024
1 parent ad326fa commit d1927a4
Show file tree
Hide file tree
Showing 10 changed files with 96 additions and 30 deletions.
6 changes: 3 additions & 3 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

6 changes: 3 additions & 3 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ resolver = "2"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

[workspace.package]
version = "0.2.28"
version = "0.2.29"
authors = ["Louis <836250617@qq.com>"]
description = "Back up your favorite online resources with CLI."
license = "MIT"
Expand All @@ -16,8 +16,8 @@ documentation = ""
[workspace.dependencies]
fav_core = { path = "fav_core", version = "0.1.3" }
fav_derive = { path = "fav_derive", version = "0.0.2" }
fav_utils = { path = "fav_utils", version = "0.0.11" }
fav_cli = { path = "fav_cli", version = "0.2.28" }
fav_utils = { path = "fav_utils", version = "0.0.12" }
fav_cli = { path = "fav_cli", version = "0.2.29" }

[profile.release]
lto = "fat"
Expand Down
11 changes: 9 additions & 2 deletions fav_cli/src/bili/action.rs
Original file line number Diff line number Diff line change
Expand Up @@ -62,9 +62,15 @@ pub(super) fn status_all(
show_sets: bool,
show_res: bool,
only_track: bool,
show_all: bool,
) -> FavCoreResult<()> {
if show_sets {
let sub = sets.subset(|s| s.check_status(StatusFlags::TRACK) | !only_track);
let sub = sets.subset(|s| {
(s.check_status(StatusFlags::TRACK) | !only_track)
& ((s.upper.mid == 0 // self-created set's mid is 0
&& !s.check_status(StatusFlags::EXPIRED))
| show_all)
});
sub.table();
}
if show_res {
Expand All @@ -79,7 +85,8 @@ pub(super) fn status_all(
pub(super) async fn fetch(sets: &mut BiliSets) -> FavCoreResult<()> {
let bili = Bili::read()?;
bili.fetch_sets(sets).await?;
let mut sub = sets.subset(|s| s.check_status(StatusFlags::TRACK));
let mut sub =
sets.subset(|s| s.check_status(StatusFlags::TRACK) & !s.check_status(StatusFlags::EXPIRED));
bili.batch_fetch_set(&mut sub, 8).await?;
for set in sub.iter_mut() {
let mut sub = set.subset(|r| {
Expand Down
16 changes: 11 additions & 5 deletions fav_cli/src/bili/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,9 @@ enum Commands {
/// Show tracked only
#[arg(long, short)]
track: bool,
/// Show sets including AchivesSets(合集)
#[arg(long, short)]
all: bool,
},
/// Track a remote source
Track {
Expand Down Expand Up @@ -114,7 +117,7 @@ impl Cli {
clap_complete::generate(shell, &mut cmd, "fav", &mut std::io::stdout());
}
subcmd => {
let mut sets = BiliSets::read()?;
let mut sets = BiliSets::read().unwrap_or_default();
let res = match subcmd {
Commands::Auth { subcmd: authcmd } => {
match authcmd {
Expand All @@ -132,21 +135,24 @@ impl Cli {
sets: show_sets,
res: show_res,
track: only_track,
all: show_all,
} => match id {
Some(id) => {
if show_sets | show_res | only_track {
if show_sets | show_res | only_track | show_all {
Cli::command()
.error(
ErrorKind::ArgumentConflict,
"The id to 'fav status' does not take -s, -r, -t, options.",
"The id to 'fav status' does not take -s, -r, -t, -a, options.",
)
.exit();
}
status(&mut sets, id)
}
None => match (show_sets, show_res) {
(false, false) => status_all(&mut sets, true, false, only_track),
_ => status_all(&mut sets, show_sets, show_res, only_track),
(false, false) => {
status_all(&mut sets, true, false, only_track, show_all)
}
_ => status_all(&mut sets, show_sets, show_res, only_track, show_all),
},
},
Commands::Fetch => fetch(&mut sets).await,
Expand Down
2 changes: 1 addition & 1 deletion fav_utils/Cargo.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[package]
name = "fav_utils"
version = "0.0.11"
version = "0.0.12"
authors.workspace = true
description = "Fav's utils crate; A collection of utilities and data structures for the fav project"
license.workspace = true
Expand Down
2 changes: 1 addition & 1 deletion fav_utils/build.rs
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
let gen = std::fs::read_to_string(&path)?;
let processed = gen.replace("#!", "//").replace("//!", "//");
std::fs::write(path, processed)?;
println!("cargo:return-if-changed=proto");
println!("cargo:return-if-changed=proto/bili.proto");
println!("cargo:return-if-changed=build.rs");
Ok(())
}
Expand Down
2 changes: 2 additions & 0 deletions fav_utils/proto/bili.proto
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,8 @@ message BiliSet {
int32 media_count = 4;
Upper upper = 14;
repeated BiliRes medias = 15;
repeated BiliRes archives = 16;
bool is_archives_list = 17;
}

message BiliSets {
Expand Down
12 changes: 12 additions & 0 deletions fav_utils/src/bili/api.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,9 @@ impl ApiProvider for Bili {
ApiKind::QrPoll => &QrPollApi,
ApiKind::Logout => &LogoutApi,
ApiKind::FetchSets => &SetsApi,
ApiKind::FetchAchivesSets => &AchivesSetsApi,
ApiKind::FetchSet => &SetApi,
ApiKind::FetchAchivesSet => &AchivesSetApi,
ApiKind::FetchRes => &ResApi,
ApiKind::Wbi => &WbiApi,
ApiKind::Pull => &PullApi,
Expand All @@ -25,7 +27,9 @@ pub enum ApiKind {
QrPoll,
Logout,
FetchSets,
FetchAchivesSets,
FetchSet,
FetchAchivesSet,
FetchRes,
Wbi,
Pull,
Expand All @@ -49,10 +53,18 @@ struct LogoutApi;
#[api(endpoint("https://api.bilibili.com/x/v3/fav/folder/created/list-all"), params(&["up_mid"]), cookies(&["SESSDATA"]))]
struct SetsApi;

#[derive(Api)]
#[api(endpoint("https://api.bilibili.com/x/v3/fav/folder/collected/list"), params(&["up_mid", "pn", "ps", "platform"]), cookies(&["SESSDATA"]))]
struct AchivesSetsApi;

#[derive(Api)]
#[api(endpoint("https://api.bilibili.com/x/v3/fav/resource/list"), params(&["media_id", "pn", "ps"]), cookies(&["SESSDATA"]))]
struct SetApi;

#[derive(Api)]
#[api(endpoint("https://api.bilibili.com/x/polymer/web-space/seasons_archives_list"), params(&["mid", "season_id", "page_num", "page_size"]), cookies(&["SESSDATA"]))]
struct AchivesSetApi;

#[derive(Api)]
#[api(endpoint("https://api.bilibili.com/x/web-interface/view"), params(&["bvid"]), cookies(&["SESSDATA"]))]
struct ResApi;
Expand Down
35 changes: 31 additions & 4 deletions fav_utils/src/bili/ops.rs
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,25 @@ impl SetsOps for Bili {
.request_proto(ApiKind::FetchSets, params, "/data")
.await?;
info!("Fetch sets successfully.");
let mut pn = 1;
loop {
let params = vec![
self.cookies().get("DedeUserID").expect(HINT).to_owned(),
pn.to_string(),
"20".to_string(),
"web".to_string(),
];
let resp = self.request(ApiKind::FetchAchivesSets, params).await?;
let json: serde_json::Value = fav_core::ops::resp2json(resp, "/data").await?;
let mut new: BiliSets = fav_core::ops::json2proto(&json)?;
new.iter_mut().for_each(|set| set.is_archives_list = true);
*sets |= new;
if !json["has_more"].as_bool().unwrap() {
break;
}
pn += 1;
}
info!("Fetch archives sets successfully.");
Ok(())
}
}
Expand All @@ -71,20 +90,28 @@ impl SetOps for Bili {
Any: Send,
{
let id = set.id.to_string();
let is_archives_list = set.is_archives_list;
let mid = set.upper.mid.to_string();
info!("Fetching set<{}>", id);
let page_count = set.media_count.saturating_sub(1) / 20 + 1;
let mut stream = tokio_stream::iter(1..=page_count)
.map(|pn| {
let pn = pn.to_string();
let params = vec![id.clone(), pn, "20".to_string()];
self.request_proto::<BiliSet>(ApiKind::FetchSet, params, "/data")
let mut params = vec![id.clone(), pn, "20".to_string()];
match is_archives_list {
true => {
params.insert(0, mid.clone());
self.request_proto::<BiliSet>(ApiKind::FetchAchivesSet, params, "/data")
}
false => self.request_proto::<BiliSet>(ApiKind::FetchSet, params, "/data"),
}
})
.buffer_unordered(8);
tokio::select! {
res = async {
while let Some(res) = stream.next().await {
match res {
Ok(res) => *set |= res.with_res_status_on(StatusFlags::FAV),
Ok(s) => *set |= s.with_res_status_on(StatusFlags::FAV),
Err(e) => return Err(e),
}
}
Expand All @@ -93,7 +120,7 @@ impl SetOps for Bili {
} => {
res
}
_ = cancelled => Err(FavCoreError::Cancel)
_ = cancelled => Err(FavCoreError::Cancel)
}
}
}
Expand Down
34 changes: 23 additions & 11 deletions fav_utils/src/bili/res.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,17 @@ impl BitOrAssign for BiliSets {
.into_iter()
.for_each(|s| match self.iter_mut().find(|s1| s1.id == s.id) {
Some(s1) => {
s1.media_count = s.media_count;
*s1 |= s
if s.media_count == 0
&& !s1.check_status(StatusFlags::EXPIRED)
&& s.title == "该合集已失效"
{
s1.title += "(已失效)";
s1.on_status(StatusFlags::EXPIRED);
} else {
s1.title = s.title;
s1.media_count = s.media_count;
s1.off_status(StatusFlags::EXPIRED);
}
}
None => cache.push(s),
});
Expand All @@ -21,14 +30,17 @@ impl BitOrAssign for BiliSets {
impl BitOrAssign for BiliSet {
/// Merge two sets. If the left set is track, the resources merged into will be track
fn bitor_assign(&mut self, rhs: Self) {
rhs.medias.into_iter().for_each(|mut r| {
if self.iter().all(|r1| r1.bvid != r.bvid) {
if self.check_status(StatusFlags::TRACK) {
r.on_status(StatusFlags::TRACK);
rhs.medias
.into_iter()
.chain(rhs.archives)
.for_each(|mut r| {
if self.iter().all(|r1| r1.bvid != r.bvid) {
if self.check_status(StatusFlags::TRACK) {
r.on_status(StatusFlags::TRACK);
}
self.medias.push(r);
}
self.medias.push(r);
}
});
});
}
}

Expand Down Expand Up @@ -57,11 +69,11 @@ impl Set for BiliSet {
type Res = BiliRes;

fn iter(&self) -> impl Iterator<Item = &BiliRes> {
self.medias.iter()
self.medias.iter().chain(self.archives.iter())
}

fn iter_mut(&mut self) -> impl Iterator<Item = &mut BiliRes> {
self.medias.iter_mut()
self.medias.iter_mut().chain(self.archives.iter_mut())
}
}

Expand Down

0 comments on commit d1927a4

Please sign in to comment.