-
-
Notifications
You must be signed in to change notification settings - Fork 29
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Incremental runs #174
Closed
Closed
Incremental runs #174
Changes from all commits
Commits
File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,44 @@ | ||
// Copyright 2023 Paxos | ||
|
||
//! Logic for incremantal runs | ||
use crate::{ | ||
mutate::{Mutant, MutantHash}, | ||
options::Options, | ||
output::{PositiveOutcome, OUTDIR_NAME, POSITIVE_OUTCOMES_FILE}, | ||
}; | ||
use anyhow::{anyhow, Result}; | ||
use camino::{Utf8Path, Utf8PathBuf}; | ||
use std::{collections::HashSet, fs}; | ||
|
||
pub fn filter_by_last_positive_outcomes( | ||
mutants: Vec<Mutant>, | ||
dir: &Utf8PathBuf, | ||
options: &Options, | ||
) -> (Option<Vec<PositiveOutcome>>, Vec<Mutant>) { | ||
let read_path: &Utf8Path = options.output_in_dir.as_ref().map_or(dir, |p| p.as_path()); | ||
// TODO: add logging here for error cases | ||
let last_positive_outcomes = match read_last_positive_outcomes(read_path) { | ||
Ok(outcomes) => Some(outcomes), | ||
Err(_) => None, | ||
}; | ||
// if last_positive_outcomes is none the hash set will be empty thereby allowing all mutants to be considered | ||
let existing_mutants: HashSet<MutantHash> = last_positive_outcomes | ||
.iter() | ||
.flatten() | ||
.map(|o| o.mutant_hash()) | ||
.collect(); | ||
let mutants = mutants | ||
.into_iter() | ||
.filter(|m| !existing_mutants.contains(&m.calculate_hash())) | ||
.collect(); | ||
(last_positive_outcomes, mutants) | ||
} | ||
|
||
fn read_last_positive_outcomes(read_path: &Utf8Path) -> Result<Vec<PositiveOutcome>> { | ||
let path = read_path.join(OUTDIR_NAME).join(POSITIVE_OUTCOMES_FILE); | ||
fs::read_to_string(path.clone()) | ||
.map(|contents| serde_json::from_str(&contents).map_err(|e| anyhow!("{}", e))) | ||
// If we can’t read the file, we assume that it doesn’t exist and we return an empty list. | ||
// Later, the file will get written and any error will be surfaced to the user. | ||
.unwrap_or(Ok(vec![])) | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -12,16 +12,19 @@ use anyhow::{Context, Result}; | |
use camino::Utf8Path; | ||
use fs2::FileExt; | ||
use path_slash::PathExt; | ||
use serde::Serialize; | ||
use serde::{Deserialize, Serialize}; | ||
use time::format_description::well_known::Rfc3339; | ||
use time::OffsetDateTime; | ||
use tracing::info; | ||
|
||
use crate::mutate::MutantHash; | ||
use crate::outcome::{LabOutcome, SummaryOutcome}; | ||
use crate::*; | ||
|
||
const OUTDIR_NAME: &str = "mutants.out"; | ||
const ROTATED_NAME: &str = "mutants.out.old"; | ||
pub const OUTDIR_NAME: &str = "mutants.out"; | ||
pub const ROTATED_NAME: &str = "mutants.out.old"; | ||
pub const OUTCOMES_FILE: &str = "outcomes.json"; | ||
pub const POSITIVE_OUTCOMES_FILE: &str = "positive_outcomes.json"; | ||
const LOCK_JSON: &str = "lock.json"; | ||
const LOCK_POLL: Duration = Duration::from_millis(100); | ||
|
||
|
@@ -98,6 +101,7 @@ pub struct OutputDir { | |
/// A file holding a list of mutants where testing timed out, as text, one per line. | ||
timeout_list: File, | ||
unviable_list: File, | ||
last_positive_outcomes: Option<Vec<PositiveOutcome>>, | ||
/// The accumulated overall lab outcome. | ||
pub lab_outcome: LabOutcome, | ||
} | ||
|
@@ -113,7 +117,10 @@ impl OutputDir { | |
/// | ||
/// If the directory already exists and `lock.json` exists and is locked, this waits for | ||
/// the lock to be released. The returned `OutputDir` holds a lock for its lifetime. | ||
pub fn new(in_dir: &Utf8Path) -> Result<OutputDir> { | ||
pub fn new( | ||
in_dir: &Utf8Path, | ||
last_positive_outcomes: Option<Vec<PositiveOutcome>>, | ||
) -> Result<OutputDir> { | ||
if !in_dir.exists() { | ||
fs::create_dir(in_dir).context("create output parent directory {in_dir:?}")?; | ||
} | ||
|
@@ -161,6 +168,7 @@ impl OutputDir { | |
caught_list, | ||
timeout_list, | ||
unviable_list, | ||
last_positive_outcomes, | ||
}) | ||
} | ||
|
||
|
@@ -178,21 +186,45 @@ impl OutputDir { | |
&self.path | ||
} | ||
|
||
/// Save positive outcomes for incremental runs | ||
/// | ||
/// Called multiple times as the lab runs. | ||
pub fn maybe_write_positive_outcomes(&self) -> Result<()> { | ||
if let Some(last_positive_outcomes) = &self.last_positive_outcomes { | ||
let positive_outcomes = self | ||
.lab_outcome | ||
.outcomes | ||
.iter() | ||
.filter_map(|o: &ScenarioOutcome| PositiveOutcome::try_from(o).ok()) | ||
.chain(last_positive_outcomes.iter().cloned()) | ||
.collect::<Vec<PositiveOutcome>>(); | ||
|
||
serde_json::to_writer_pretty( | ||
BufWriter::new(File::create(self.path.join(POSITIVE_OUTCOMES_FILE))?), | ||
&positive_outcomes, | ||
) | ||
.context(format!("write {}", POSITIVE_OUTCOMES_FILE)) | ||
} else { | ||
Ok(()) | ||
} | ||
} | ||
|
||
/// Update the state of the overall lab. | ||
/// | ||
/// Called multiple times as the lab runs. | ||
pub fn write_lab_outcome(&self) -> Result<()> { | ||
serde_json::to_writer_pretty( | ||
BufWriter::new(File::create(self.path.join("outcomes.json"))?), | ||
BufWriter::new(File::create(self.path.join(OUTCOMES_FILE))?), | ||
&self.lab_outcome, | ||
) | ||
.context("write outcomes.json") | ||
.context(format!("write {}", OUTCOMES_FILE)) | ||
} | ||
|
||
/// Add the result of testing one scenario. | ||
pub fn add_scenario_outcome(&mut self, scenario_outcome: &ScenarioOutcome) -> Result<()> { | ||
self.lab_outcome.add(scenario_outcome.to_owned()); | ||
self.write_lab_outcome()?; | ||
self.maybe_write_positive_outcomes()?; | ||
let scenario = &scenario_outcome.scenario; | ||
if let Scenario::Mutant(mutant) = scenario { | ||
let file = match scenario_outcome.summary() { | ||
|
@@ -281,7 +313,7 @@ mod test { | |
let tmp = minimal_source_tree(); | ||
let tmp_path: &Utf8Path = tmp.path().try_into().unwrap(); | ||
let workspace = Workspace::open(tmp_path).unwrap(); | ||
let output_dir = OutputDir::new(&workspace.dir).unwrap(); | ||
let output_dir = OutputDir::new(&workspace.dir, None).unwrap(); | ||
assert_eq!( | ||
list_recursive(tmp.path()), | ||
&[ | ||
|
@@ -309,7 +341,7 @@ mod test { | |
let temp_dir_path = Utf8Path::from_path(temp_dir.path()).unwrap(); | ||
|
||
// Create an initial output dir with one log. | ||
let output_dir = OutputDir::new(temp_dir_path).unwrap(); | ||
let output_dir = OutputDir::new(temp_dir_path, None).unwrap(); | ||
output_dir.create_log(&Scenario::Baseline).unwrap(); | ||
assert!(temp_dir | ||
.path() | ||
|
@@ -318,7 +350,7 @@ mod test { | |
drop(output_dir); // release the lock. | ||
|
||
// The second time we create it in the same directory, the old one is moved away. | ||
let output_dir = OutputDir::new(temp_dir_path).unwrap(); | ||
let output_dir = OutputDir::new(temp_dir_path, None).unwrap(); | ||
output_dir.create_log(&Scenario::Baseline).unwrap(); | ||
assert!(temp_dir | ||
.path() | ||
|
@@ -331,7 +363,7 @@ mod test { | |
drop(output_dir); | ||
|
||
// The third time (and later), the .old directory is removed. | ||
let output_dir = OutputDir::new(temp_dir_path).unwrap(); | ||
let output_dir = OutputDir::new(temp_dir_path, None).unwrap(); | ||
output_dir.create_log(&Scenario::Baseline).unwrap(); | ||
assert!(temp_dir | ||
.path() | ||
|
@@ -347,3 +379,47 @@ mod test { | |
.is_file()); | ||
} | ||
} | ||
|
||
/// Caught and unviable scenario outcome | ||
#[derive(Serialize, Deserialize, Debug, Clone, Eq, PartialEq)] | ||
pub struct PositiveOutcome { | ||
pub mutant_hash: MutantHash, | ||
/// This is always `Unviable` or `Caught`, because other outcomes aren’t psotive outcomes | ||
pub summary: SummaryOutcome, | ||
} | ||
|
||
impl PositiveOutcome { | ||
fn new(mutant: &Mutant, summary: SummaryOutcome) -> Result<Self> { | ||
match summary { | ||
SummaryOutcome::CaughtMutant | SummaryOutcome::Unviable => Ok(Self { | ||
mutant_hash: mutant.calculate_hash(), | ||
summary, | ||
}), | ||
_ => Err(anyhow::anyhow!( | ||
"outcome {:?} isn’t a positive outcome", | ||
summary | ||
)), | ||
} | ||
} | ||
|
||
/// Gets the hash of the associated mutant | ||
pub fn mutant_hash(&self) -> MutantHash { | ||
self.mutant_hash | ||
} | ||
} | ||
|
||
impl TryFrom<&ScenarioOutcome> for PositiveOutcome { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This bit is currently broken, we are working to fix it. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Fixed now |
||
type Error = anyhow::Error; | ||
|
||
fn try_from(scenario_outcome: &ScenarioOutcome) -> Result<Self> { | ||
let mutant = match &scenario_outcome.scenario { | ||
Scenario::Mutant(mutant) => mutant, | ||
Scenario::Baseline => { | ||
return Err(anyhow::anyhow!( | ||
"baseline can’t be converted to positive outcome" | ||
)) | ||
} | ||
}; | ||
Self::new(mutant, scenario_outcome.summary()) | ||
} | ||
} |
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This is deprecated and the warning says we should use DefaultHasher, but the doc says:
and in this case, we need the hashes to be stable since they are persisted on disk.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I wonder if we could just avoid the serialized format depending on a hash algorithm: could you store them in the originally generated order or even just sorted by location and name?