Skip to content

Commit

Permalink
Validate spec on creation
Browse files Browse the repository at this point in the history
  • Loading branch information
@dominikWin
dominikWin committed Sep 5, 2024
1 parent 93ff8e1 commit f06698a
Show file tree
Hide file tree
Showing 6 changed files with 66 additions and 23 deletions.
6 changes: 3 additions & 3 deletions vidformer-cli/src/bench.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ use glob::glob;
use serde::Deserialize;
use std::collections::BTreeMap;
use std::sync::Arc;
use vidformer::run_spec;
use vidformer::run;

#[derive(Deserialize)]
struct DveBench {
Expand Down Expand Up @@ -69,11 +69,11 @@ pub(crate) fn cmd_benchmark(opt: &BenchmarkCmd) {

println!("Running spec {} benches", entry.display());
for _i in 0..opt.warmup_runs {
run_spec(&spec, "/tmp/output.mp4", &context, &config, &None).unwrap();
run(&spec, "/tmp/output.mp4", &context, &config, &None).unwrap();
}

for i in 0..opt.runs {
let stat = run_spec(&spec, "/tmp/output.mp4", &context, &config, &None).unwrap();
let stat = run(&spec, "/tmp/output.mp4", &context, &config, &None).unwrap();

let benchmark_stat = BenchmarkStat {
bench: entry.display().to_string(),
Expand Down
4 changes: 2 additions & 2 deletions vidformer-cli/src/main.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ use clap::{Parser, Subcommand};
use num_rational::Rational64;
use std::{collections::BTreeMap, fs::File, io::Write};

use vidformer::{filter, run_spec, sir, source, spec};
use vidformer::{filter, run, sir, source, spec};

mod bench;
mod yrden;
Expand Down Expand Up @@ -177,7 +177,7 @@ fn cmd_x() {
let dve_config = std::sync::Arc::new(dve_config);

println!("Running spec...");
let stats = run_spec(&spec, output_path, &context, &dve_config, &None).unwrap();
let stats = run(&spec, output_path, &context, &dve_config, &None).unwrap();

dbg!(&stats);

Expand Down
29 changes: 23 additions & 6 deletions vidformer-cli/src/yrden.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -382,6 +382,7 @@ async fn yrden_http_req(
serde_json::from_reader(spec_content).expect("Unable to parse JSON");

let spec: Box<dyn spec::Spec> = Box::new(spec);
let spec = std::sync::Arc::new(spec);
let mut filters = crate::default_filters();

for (name, filter) in request.filters {
Expand Down Expand Up @@ -423,18 +424,34 @@ async fn yrden_http_req(
};
let dve_config = std::sync::Arc::new(dve_config);

// Validate the spec
let validation_status = vidformer::validate(&spec, &context, &dve_config);
if let Err(err) = validation_status {
return Ok(hyper::Response::builder()
.status(hyper::StatusCode::BAD_REQUEST)
.header("Access-Control-Allow-Origin", "*")
.body(http_body_util::Full::new(hyper::body::Bytes::from(
format!("Error validating spec: {}", err),
)))
.unwrap());
}

let host_prefix = {
let global: std::sync::MutexGuard<'_, YrdenGlobal> = global.lock().unwrap();
global.host_prefix.clone()
};

let (namespace_id, playlist, stream, ranges) =
vidformer::create_spec_hls(spec.as_ref(), &host_prefix, &context, &dve_config);
let (namespace_id, playlist, stream, ranges) = vidformer::create_spec_hls(
spec.as_ref().as_ref(),
&host_prefix,
&context,
&dve_config,
);

let namespace = YrdenNamespace {
context,
dve_config,
spec: std::sync::Arc::new(spec),
spec,
playlist,
stream,
ranges,
Expand Down Expand Up @@ -625,7 +642,7 @@ async fn yrden_http_req(
let spec = std::sync::Arc::new(spec);

let spec_result = tokio::task::spawn_blocking(move || {
vidformer::run_spec(&spec, &output_path2, &context, &dve_config, &None)
vidformer::run(&spec, &output_path2, &context, &dve_config, &None)
})
.await
.unwrap();
Expand Down Expand Up @@ -748,7 +765,7 @@ async fn yrden_http_req(

// todo, don't unwrap
let spec_result = tokio::task::spawn_blocking(move || {
vidformer::run_spec(
vidformer::run(
&spec,
&tmp_path_2,
&context,
Expand Down Expand Up @@ -847,7 +864,7 @@ async fn yrden_http_req(

// todo, don't unwrap
let spec_result = tokio::task::spawn_blocking(move || {
vidformer::run_spec(
vidformer::run(
&spec,
&tmp_path_2,
&context,
Expand Down
29 changes: 27 additions & 2 deletions vidformer/src/dve.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1040,8 +1040,8 @@ impl ExecContext {
}
}

/// Execute a vidformer spec
pub fn run_spec(
/// Execute a spec, or a range of a spec.
pub fn run(
spec: &Arc<Box<dyn Spec>>,
output_path: &str,
context: &Arc<Context>,
Expand Down Expand Up @@ -1117,6 +1117,31 @@ pub fn run_spec(
exec_contex.run()
}

/// Validate that a spec can be run.
pub fn validate(
spec: &Arc<Box<dyn Spec>>,
context: &Arc<Context>,
config: &Arc<Config>,
) -> Result<(), Error> {
let expected_output_type = config.expected_output_type();

let process_span = Arc::new(crate::sir::ProcessSpan::create(
spec.as_ref().as_ref(),
context,
&None,
));

// Type check frames
for oframe in &process_span.frames {
let frame_type = type_frame(context, config, oframe)?;
if frame_type != expected_output_type {
return Err(Error::InvalidOutputFrameType);
}
}

Ok(())
}

fn encoder_thread(
config: Arc<Config>,
stat: Arc<StatRunner>,
Expand Down
3 changes: 2 additions & 1 deletion vidformer/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ mod pool;
mod util;

pub use dve::{
create_spec_hls, run_spec, Config, Context, EncoderConfig, Error, Range, RangeTsFormat, Stats,
create_spec_hls, run, validate, Config, Context, EncoderConfig, Error, Range, RangeTsFormat,
Stats,
};
pub use util::{codecs, init, CodecDescriptor};
18 changes: 9 additions & 9 deletions vidformer/tests/basic_tests.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,7 @@ fn test_placeholder() {
let spec = std::sync::Arc::new(spec);
let context = std::sync::Arc::new(context);
let dve_config = std::sync::Arc::new(dve_config);
let stats = run_spec(&spec, output_path, &context, &dve_config, &None).unwrap();
let stats = run(&spec, output_path, &context, &dve_config, &None).unwrap();
assert_eq!(stats.max_decoder_count, 0);
assert_eq!(stats.frames_written, 24 * 3);

Expand Down Expand Up @@ -169,7 +169,7 @@ fn test_bad_resolution() {
let context = std::sync::Arc::new(context);
let dve_config = std::sync::Arc::new(dve_config);
assert!(matches!(
run_spec(
run(
&spec,
test_output_path!(test_bad_resolution),
&context,
Expand Down Expand Up @@ -227,7 +227,7 @@ fn test_non_existant_source() {
let context = std::sync::Arc::new(context);
let dve_config = std::sync::Arc::new(dve_config);

let ret = run_spec(
let ret = run(
&spec,
test_output_path!(test_non_existant_source),
&context,
Expand Down Expand Up @@ -299,7 +299,7 @@ fn test_no_source_file() {
let context = std::sync::Arc::new(context);
let dve_config = std::sync::Arc::new(dve_config);

let ret = run_spec(
let ret = run(
&spec,
test_output_path!(test_non_existant_source),
&context,
Expand Down Expand Up @@ -371,7 +371,7 @@ fn test_tos_transcode_1dec() {
num_frames: NUM_FRAMES,
}));
let output_path = test_output_path!(test_tos_transcode_1dec);
let stats = run_spec(&spec, output_path, &context, &dve_config, &None).unwrap();
let stats = run(&spec, output_path, &context, &dve_config, &None).unwrap();

assert_eq!(stats.max_decoder_count, 1);
assert_eq!(stats.frames_written, NUM_FRAMES as usize);
Expand Down Expand Up @@ -400,7 +400,7 @@ fn test_tos_transcode_2dec() {
num_frames: NUM_FRAMES,
}));
let output_path = test_output_path!(test_tos_transcode_2dec);
let stats = run_spec(&spec, output_path, &context, &dve_config, &None).unwrap();
let stats = run(&spec, output_path, &context, &dve_config, &None).unwrap();

assert!(stats.max_decoder_count >= 1 && stats.max_decoder_count <= 2);
assert_eq!(stats.frames_written, NUM_FRAMES as usize);
Expand Down Expand Up @@ -429,7 +429,7 @@ fn test_tos_transcode_4dec() {
num_frames: NUM_FRAMES,
}));
let output_path = test_output_path!(test_tos_transcode_4dec);
let stats = run_spec(&spec, output_path, &context, &dve_config, &None).unwrap();
let stats = run(&spec, output_path, &context, &dve_config, &None).unwrap();

assert!(stats.max_decoder_count >= 1 && stats.max_decoder_count <= 4);
assert_eq!(stats.frames_written, NUM_FRAMES as usize);
Expand Down Expand Up @@ -458,7 +458,7 @@ fn test_tos_transcode_manydec() {
num_frames: NUM_FRAMES,
}));
let output_path = test_output_path!(test_tos_transcode_manydec);
let stats = run_spec(&spec, output_path, &context, &dve_config, &None).unwrap();
let stats = run(&spec, output_path, &context, &dve_config, &None).unwrap();

assert!(stats.max_decoder_count >= 1 && stats.max_decoder_count <= 100);
assert_eq!(stats.frames_written, NUM_FRAMES as usize);
Expand Down Expand Up @@ -486,7 +486,7 @@ fn test_tos_transcode_1dec_1pool() {
let spec: std::sync::Arc<Box<dyn spec::Spec>> =
std::sync::Arc::new(Box::new(ClipSpec { num_frames: 2 * 24 })); // make sure we only need one source GOP
let output_path = test_output_path!(test_tos_transcode_1dec_1pool);
let stats = run_spec(&spec, output_path, &context, &dve_config, &None).unwrap();
let stats = run(&spec, output_path, &context, &dve_config, &None).unwrap();

assert_eq!(stats.max_decoder_count, 1);
assert_eq!(stats.decoders_created, 1); // this is just a basic streaming edit. if our algorithm works it should just decode the one needed GOP
Expand Down

0 comments on commit f06698a

Please sign in to comment.