dart: support v2 (#75)

src/dart.rs

@@ -1,25 +1,29 @@
-use crate::common::{Mission, SnapshotConfig, SnapshotPath};
+use crate::common::{Mission, SnapshotConfig, SnapshotPath, TransferURL};
 use crate::error::{Error, Result};
-use crate::traits::SnapshotStorage;
+use crate::metadata::SnapshotMeta;
+use crate::traits::{SnapshotStorage, SourceStorage};
 
 use async_trait::async_trait;
 use futures_util::{stream, StreamExt, TryStreamExt};
 use serde_json::Value;
 use slog::{info, warn};
+use structopt::StructOpt;
 
-#[derive(Debug)]
+#[derive(Debug, Clone, StructOpt)]
 pub struct Dart {
+    #[structopt(long, default_value = "https://mirrors.tuna.tsinghua.edu.cn/dart-pub")]
     pub base: String,
+    #[structopt(long)]
     pub debug: bool,
 }
 
 #[async_trait]
-impl SnapshotStorage<SnapshotPath> for Dart {
+impl SnapshotStorage<SnapshotMeta> for Dart {
     async fn snapshot(
         &mut self,
         mission: Mission,
         config: &SnapshotConfig,
-    ) -> Result<Vec<SnapshotPath>> {
+    ) -> Result<Vec<SnapshotMeta>> {
         let logger = mission.logger;
         let progress = mission.progress;
         let client = mission.client;
@@ -66,7 +70,7 @@ impl SnapshotStorage<SnapshotPath> for Dart {
 
         progress.inc_length(package_name.len() as u64);
 
-        let snapshots: Result<Vec<Vec<String>>> =
+        let snapshots: Result<Vec<Vec<SnapshotMeta>>> =
             stream::iter(package_name.into_iter().map(|name| {
                 let client = client.clone();
                 let base = format!("{}/", self.base);
@@ -84,15 +88,24 @@ impl SnapshotStorage<SnapshotPath> for Dart {
 
                     let data: Value = serde_json::from_str(&package).unwrap();
                     let versions = data.get("versions").unwrap().as_array().unwrap();
-                    let archives: Vec<String> = versions
+                    let archives: Vec<SnapshotMeta> = versions
                         .iter()
                         .filter_map(|version| version.get("archive_url"))
                         .filter_map(|archive_url| archive_url.as_str())
-                        .map(|archive_url| archive_url.replace(&base, ""))
+                        .map(|archive_url| {
+                            if archive_url.starts_with(&base) {
+                                SnapshotMeta {
+                                    key: archive_url[base.len()..].to_string(),
+                                    ..Default::default()
+                                }
+                            } else {
+                                panic!("Unmatched base URL {}", archive_url);
+                            }
+                        })
                         .collect();
 
                     progress.inc(1);
-                    Ok::<Vec<String>, Error>(archives)
+                    Ok::<Vec<SnapshotMeta>, Error>(archives)
                 };
                 async move {
                     match func.await {
@@ -112,10 +125,17 @@ impl SnapshotStorage<SnapshotPath> for Dart {
 
         progress.finish_with_message("done");
 
-        Ok(crate::utils::snapshot_string_to_path(snapshot))
+        Ok(snapshot)
     }
 
     fn info(&self) -> String {
         format!("dart, {:?}", self)
     }
 }
+
+#[async_trait]
+impl SourceStorage<SnapshotMeta, TransferURL> for Dart {
+    async fn get_object(&self, snapshot: &SnapshotMeta, _mission: &Mission) -> Result<TransferURL> {
+        Ok(TransferURL(format!("{}/{}", self.base, snapshot.key)))
+    }
+}

src/github_release.rs

@@ -81,7 +81,11 @@ impl SnapshotStorage<SnapshotMeta> for GitHubRelease {
             .take(self.version_to_retain)
             .flatten()
             .map(|asset| SnapshotMeta {
-                key: asset.browser_download_url.replace(&replace_string, ""),
+                key: if asset.browser_download_url.starts_with(&replace_string) {
+                    asset.browser_download_url[replace_string.len()..].to_string()
+                } else {
+                    panic!("Unmatched base URL: {:?}", asset)
+                },
                 size: Some(asset.size),
                 last_modified: Some(asset.updated_at.timestamp() as u64),
                 ..Default::default()

src/main.rs

@@ -162,6 +162,14 @@ fn main() {
                     index_bytes_pipe!(buffer_path, prefix, true)
                 );
             }
+            Source::DartPub(source) => {
+                transfer!(
+                    opts,
+                    source,
+                    transfer_config,
+                    index_bytes_pipe!(buffer_path, prefix, false)
+                );
+            }
         }
     });
 }

src/opts.rs

@@ -1,5 +1,6 @@
 use crate::conda::CondaConfig;
 use crate::crates_io::CratesIo as CratesIoConfig;
+use crate::dart::Dart;
 use crate::file_backend::FileBackend;
 use crate::github_release::GitHubRelease;
 use crate::homebrew::Homebrew as HomebrewConfig;
@@ -27,6 +28,8 @@ pub enum Source {
     Rsync(RsyncConfig),
     #[structopt(about = "GitHub Releases")]
     GithubRelease(GitHubRelease),
+    #[structopt(about = "dart pub.dev")]
+    DartPub(Dart),
 }
 
 #[derive(Debug)]