From d6e0724de12860ee5a380b14a0b6319d78accda1 Mon Sep 17 00:00:00 2001 From: epatelli Date: Sat, 15 Feb 2020 11:21:09 +0000 Subject: [PATCH 1/9] * Fix create RandomVariableSet from IID RandomVariable --- .../+random/@RandomVariable/RandomVariable.m | 8 +------ .../@RandomVariableSet/RandomVariableSet.m | 21 ++++++++++--------- 2 files changed, 12 insertions(+), 17 deletions(-) diff --git a/+opencossan/+common/+inputs/+random/@RandomVariable/RandomVariable.m b/+opencossan/+common/+inputs/+random/@RandomVariable/RandomVariable.m index 7de0dc34..9c8e1528 100644 --- a/+opencossan/+common/+inputs/+random/@RandomVariable/RandomVariable.m +++ b/+opencossan/+common/+inputs/+random/@RandomVariable/RandomVariable.m @@ -2,13 +2,7 @@ %RANDOMVARIABLE This class is abstract and defines presettings for % several specializations. % - % For more detailed information, see - % . - % - % Properties: - % Mean - % Std - % Shift + % See also: RandomVariableSet, Input %{ This file is part of OpenCossan . diff --git a/+opencossan/+common/+inputs/+random/@RandomVariableSet/RandomVariableSet.m b/+opencossan/+common/+inputs/+random/@RandomVariableSet/RandomVariableSet.m index bd1d0e3e..ef83f0c0 100644 --- a/+opencossan/+common/+inputs/+random/@RandomVariableSet/RandomVariableSet.m +++ b/+opencossan/+common/+inputs/+random/@RandomVariableSet/RandomVariableSet.m @@ -1,5 +1,7 @@ classdef RandomVariableSet < opencossan.common.CossanObject %RANDOMVARIABLESET Constructs object RandomVariableSet + % + % See also: RandomVariable, Input %{ This file is part of OpenCossan . @@ -58,6 +60,7 @@ ["Correlation", "Covariance"],{[],[]}, varargin{:}); end + % Now we define all the inputs not filtered out by the parsers obj@opencossan.common.CossanObject(super_args{:}); if nargin > 0 @@ -116,23 +119,21 @@ end methods (Static) - function obj = fromIidRandomVariables(rv,n,varargin) + function obj = fromIidRandomVariables(varargin) import opencossan.common.inputs.random.RandomVariableSet - p = inputParser; - p.FunctionName = 'opencossan.common.inputs.random.RandomVariableSet.fromIidRandomVariables'; - p.addRequired('rv') - p.addRequired('n'); - p.parse(rv,n); + [required, varargin] = opencossan.common.utilities.parseRequiredNameValuePairs(... + ["Number","RandomVariable"], varargin{:}); - [optional, varargin] = opencossan.common.utilities.parseOptionalNameValuePairs(... - "basename", {"RV"}, varargin{:}); + [optional, ~] = opencossan.common.utilities.parseOptionalNameValuePairs(... + "NamePrefix","RV_", varargin{:}); + - members(1:p.Results.n) = p.Results.rv; + members(1:p.Results.n) = required.randomvariable; names = strings(1,n); for i = 1:p.Results.n - names(i) = sprintf("%s_%d", optional.basename, i); + names(i) = sprintf("%s_%d", optional.nameprefix, i); end varargin = [varargin {'members', members, 'names', names}]; From 2d6f2d6e04cc54d79298648c0c4bb1013da67df4 Mon Sep 17 00:00:00 2001 From: Edoardo Patelli Date: Sat, 15 Feb 2020 19:10:33 +0000 Subject: [PATCH 2/9] First commit of the reworked Mio --- .../@RandomVariableSet/RandomVariableSet.m | 19 +- +opencossan/+common/@Model/Model.m | 4 +- .../JobManagerInterface.m | 1 - .../@JobManagerInterface/display.m | 18 -- +opencossan/+workers/@Evaluator/Evaluator.m | 290 +++++++----------- +opencossan/+workers/@Evaluator/add.m | 26 +- +opencossan/+workers/@Evaluator/apply.m | 26 +- .../+workers/@Evaluator/createWrapper.m | 14 +- .../@Evaluator/deterministicAnalysis.m | 2 +- +opencossan/+workers/@Evaluator/display.m | 39 --- .../+workers/@Evaluator/executeWorkersGrid.m | 20 +- .../@Evaluator/executeWorkersHorizontal.m | 14 +- .../+workers/@MatlabWorker/MatlabWorker.m | 124 ++++++++ .../{@Mio => @MatlabWorker}/checkPinput.m | 0 .../{@Mio => @MatlabWorker}/compile.m | 0 .../createSimulationData.m | 0 .../{@Mio => @MatlabWorker}/createWrapper.m | 0 .../deterministicAnalysis.m | 0 .../{@Mio => @MatlabWorker}/evaluate.m | 0 .../{@Mio => @MatlabWorker}/execute.m | 0 .../{@Mio => @MatlabWorker}/prepareInput.m | 0 .../{@Mio => @MatlabWorker}/retrieveResults.m | 0 .../+workers/{@Mio => @MatlabWorker}/run.m | 0 .../{@Mio => @MatlabWorker}/runFunction.m | 0 .../+workers/{@Mio => @MatlabWorker}/runJob.m | 0 .../{@Mio => @MatlabWorker}/runJobCompiled.m | 0 .../{@Mio => @MatlabWorker}/runJobMatlab.m | 0 .../{@Mio => @MatlabWorker}/runScript.m | 10 +- .../validateConstructor.m | 0 +opencossan/+workers/@Mio/Mio.m | 148 --------- +opencossan/+workers/@Worker/Worker.m | 27 +- .../@OpenCossan/validateCossanInputs.m | 2 +- .../CantileverBeam/TutorialCantileverBeam.m | 40 ++- .../TutorialCantileverBeamAnsys.m | 22 +- .../ExampleMatlabWorkerFunction.m} | 0 .../ExampleMatlabWorkerMatrix.m} | 0 .../ExampleMatlabWorkerStructure.m} | 0 .../ExampleMatlabWorkerTable.m} | 0 .../CossanObjects/TutorialJobManagerSlurm.m | 63 ++++ .../{TutorialMio.m => TutorialMatlabWorker.m} | 112 +++---- .../CossanObjects/TutorialSensitivity.m | 46 ++- .../+opencossan}/+workers/EvaluatorTest.m | 61 ++-- .../{MioTest.m => MatlabWorkerTest.m} | 100 +++--- 43 files changed, 589 insertions(+), 639 deletions(-) delete mode 100644 +opencossan/+highperformancecomputing/@JobManagerInterface/display.m delete mode 100644 +opencossan/+workers/@Evaluator/display.m create mode 100644 +opencossan/+workers/@MatlabWorker/MatlabWorker.m rename +opencossan/+workers/{@Mio => @MatlabWorker}/checkPinput.m (100%) rename +opencossan/+workers/{@Mio => @MatlabWorker}/compile.m (100%) rename +opencossan/+workers/{@Mio => @MatlabWorker}/createSimulationData.m (100%) rename +opencossan/+workers/{@Mio => @MatlabWorker}/createWrapper.m (100%) rename +opencossan/+workers/{@Mio => @MatlabWorker}/deterministicAnalysis.m (100%) rename +opencossan/+workers/{@Mio => @MatlabWorker}/evaluate.m (100%) rename +opencossan/+workers/{@Mio => @MatlabWorker}/execute.m (100%) mode change 100755 => 100644 rename +opencossan/+workers/{@Mio => @MatlabWorker}/prepareInput.m (100%) rename +opencossan/+workers/{@Mio => @MatlabWorker}/retrieveResults.m (100%) rename +opencossan/+workers/{@Mio => @MatlabWorker}/run.m (100%) rename +opencossan/+workers/{@Mio => @MatlabWorker}/runFunction.m (100%) rename +opencossan/+workers/{@Mio => @MatlabWorker}/runJob.m (100%) rename +opencossan/+workers/{@Mio => @MatlabWorker}/runJobCompiled.m (100%) rename +opencossan/+workers/{@Mio => @MatlabWorker}/runJobMatlab.m (100%) rename +opencossan/+workers/{@Mio => @MatlabWorker}/runScript.m (93%) rename +opencossan/+workers/{@Mio => @MatlabWorker}/validateConstructor.m (100%) delete mode 100644 +opencossan/+workers/@Mio/Mio.m rename docs/tutorials/CossanObjects/{Files4Mio/ExampleMioFunction.m => Files4MatlabWorker/ExampleMatlabWorkerFunction.m} (100%) rename docs/tutorials/CossanObjects/{Files4Mio/ExampleMioMatrix.m => Files4MatlabWorker/ExampleMatlabWorkerMatrix.m} (100%) rename docs/tutorials/CossanObjects/{Files4Mio/ExampleMioStructure.m => Files4MatlabWorker/ExampleMatlabWorkerStructure.m} (100%) rename docs/tutorials/CossanObjects/{Files4Mio/ExampleMioTable.m => Files4MatlabWorker/ExampleMatlabWorkerTable.m} (100%) mode change 100755 => 100644 create mode 100644 docs/tutorials/CossanObjects/TutorialJobManagerSlurm.m rename docs/tutorials/CossanObjects/{TutorialMio.m => TutorialMatlabWorker.m} (66%) rename test/{ => unit/+opencossan}/+workers/EvaluatorTest.m (78%) rename test/unit/+opencossan/+workers/{MioTest.m => MatlabWorkerTest.m} (73%) diff --git a/+opencossan/+common/+inputs/+random/@RandomVariableSet/RandomVariableSet.m b/+opencossan/+common/+inputs/+random/@RandomVariableSet/RandomVariableSet.m index ef83f0c0..794fb1a5 100644 --- a/+opencossan/+common/+inputs/+random/@RandomVariableSet/RandomVariableSet.m +++ b/+opencossan/+common/+inputs/+random/@RandomVariableSet/RandomVariableSet.m @@ -120,23 +120,30 @@ methods (Static) function obj = fromIidRandomVariables(varargin) + % Required input arguments: + % * Number: Number of Random Variable in the set + % * RandomVariable: IID Random variable + % + % Optional arguments + % * NamePrefix: Prefix used to construct IID random variables + % import opencossan.common.inputs.random.RandomVariableSet [required, varargin] = opencossan.common.utilities.parseRequiredNameValuePairs(... ["Number","RandomVariable"], varargin{:}); - [optional, ~] = opencossan.common.utilities.parseOptionalNameValuePairs(... + [optional, superArg] = opencossan.common.utilities.parseOptionalNameValuePairs(... "NamePrefix","RV_", varargin{:}); - members(1:p.Results.n) = required.randomvariable; + members(1:required.number) = required.randomvariable; - names = strings(1,n); - for i = 1:p.Results.n - names(i) = sprintf("%s_%d", optional.nameprefix, i); + names = strings(1,required.number); + for i = 1:required.number + names(i) = sprintf("%s%d", optional.nameprefix, i); end - varargin = [varargin {'members', members, 'names', names}]; + varargin = [superArg {'members', members, 'names', names}]; obj = RandomVariableSet(varargin{:}); end end diff --git a/+opencossan/+common/@Model/Model.m b/+opencossan/+common/@Model/Model.m index 1553bf0f..3d70e52d 100644 --- a/+opencossan/+common/@Model/Model.m +++ b/+opencossan/+common/@Model/Model.m @@ -50,10 +50,10 @@ % Check that all names required by the evaluator are % present in the input - assert(all(ismember(obj.Evaluator.Cinputnames,... + assert(all(ismember(obj.Evaluator.InputNames,... obj.InputNames)),'OpenCossan:Model:MissingInputs',... 'The Input object must contain all inputs required by the Evaluator: ''%s''', ... - strjoin(obj.Evaluator.Cinputnames,''', ''')); + strjoin(obj.Evaluator.InputNames,''', ''')); end end diff --git a/+opencossan/+highperformancecomputing/@JobManagerInterface/JobManagerInterface.m b/+opencossan/+highperformancecomputing/@JobManagerInterface/JobManagerInterface.m index 6df5f3cf..5bb34540 100644 --- a/+opencossan/+highperformancecomputing/@JobManagerInterface/JobManagerInterface.m +++ b/+opencossan/+highperformancecomputing/@JobManagerInterface/JobManagerInterface.m @@ -95,7 +95,6 @@ end - display(Xobj) % Methods to retrieve information from grid Cmembers = getQueues(Xobj); % Get Queues names diff --git a/+opencossan/+highperformancecomputing/@JobManagerInterface/display.m b/+opencossan/+highperformancecomputing/@JobManagerInterface/display.m deleted file mode 100644 index ca217dce..00000000 --- a/+opencossan/+highperformancecomputing/@JobManagerInterface/display.m +++ /dev/null @@ -1,18 +0,0 @@ -function display(Xobj) -%DISPLAY Displays the object JobManagerInterface -% -% ================================================================== -% COSSAN-X - The next generation of the computational stochastic analysis -% University of Innsbruck, Copyright 1993-2011 IfM -% ================================================================== - -%% Output to Screen -% Name and description -OpenCossan.cossanDisp('===================================================================',3); -OpenCossan.cossanDisp([' JobManagerInterface Object - ' Xobj.Sdescription ],1); -OpenCossan.cossanDisp('===================================================================',3); - -% main paramenters - - - diff --git a/+opencossan/+workers/@Evaluator/Evaluator.m b/+opencossan/+workers/@Evaluator/Evaluator.m index 243a68db..27087db9 100644 --- a/+opencossan/+workers/@Evaluator/Evaluator.m +++ b/+opencossan/+workers/@Evaluator/Evaluator.m @@ -1,227 +1,163 @@ -classdef Evaluator +classdef Evaluator < opencossan.common.CossanObject %EVALUATOR Constructor function for class EVALUATOR - % The evaluator is the object that controls the execution of the analysis. It - % communicates with the JobManager to distribute the jobs on the grid. + % The evaluator is the object that controls the execution of the analysis. + % If a JobManager is defined, it distributes jobs on the cluster/grid. % - % See also: https://cossan.co.uk/wiki/index.php/@Evaluator - % - % Author: Edoardo Patelli - % COSSAN Working Group - % email address: openengine@cossan.co.uk - % Website: http://www.cossan.co.uk + % See also: Worker - % ===================================================================== - % This file is part of openCOSSAN. The open general purpose matlab - % toolbox for numerical analysis, risk and uncertainty quantification. - % - % openCOSSAN is free software: you can redistribute it and/or modify - % it under the terms of the GNU General Public License as published by - % the Free Software Foundation, either version 3 of the License. - % - % openCOSSAN is distributed in the hope that it will be useful, - % but WITHOUT ANY WARRANTY; without even the implied warranty of - % MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - % GNU General Public License for more details. - % - % You should have received a copy of the GNU General Public License - % along with openCOSSAN. If not, see . - % ===================================================================== + %{ +This file is part of OpenCossan . +Copyright (C) 2006-2020 COSSAN WORKING GROUP + +OpenCossan is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License or, +(at your option) any later version. + +OpenCossan is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with OpenCossan. If not, see . + %} properties - Sdescription % Description of the object - XjobInterface % JobManagerInterface - CXsolvers % Cell array of solvers - CSnames % Names of the evaluators - CSqueues % Names of queues for solvers - CShostnames % Names of hostnames where evaluate solvers - CSparallelEnvironments % Name of the parallel environment of each solver - Vslots % Number of slots used in each job - VLcompiled % Number of slots used in each job - Vconcurrent % Number of concurrent execution of each solver - LremoteInjectExtract = false %TODO: make it true by default - LverticalSplit = false % if true split the analysis in vertical components (see wiki for more details) - NJobs % max number of jobs submitted for each analysis - SwrapperInputFile % Name of the input Matlab file loaded by the job - SwrapperOutputFile % Name of the output Matlab file create by the job + JobInterface opencossan.highperformancecomputing.JobManagerInterface % Define JobManager to submit job to grid/cluster computer + Solvers(1,:) %List of OpenCossan Workers opencossan.workers.Worker + SolversName(1,:) string % Names of the workers (optional) + Queues(1,:) cell % Where to submit solvers + Hostnames(1,:) cell % Names of hostnames where to evaluate workers + ParallelEnvironments(:,1) cell % Name of the parallel environment of each solver + Slots(1,:) double {mustBeInteger} % Number of slots used in each job + IsCompiled(1,:) logical % Number of slots used in each job + MaxCuncurrentJobs(1,:) double {mustBeInteger,mustBePositive} = 1 % Number of concurrent execution of each solver + RemoteInjectExtract = false %TODO: make it true by default + VerticalSplit = false % if true split the analysis in vertical components (see wiki for more details) + MaxNumberofJobs double {mustBeInteger,mustBePositive} = 1 % max number of jobs submitted for each analysis + WrapperMatlabInputName(1,1) string % Name of the input Matlab file loaded by the job + WrapperMatlabOutputName(1,1) string % Name of the output Matlab file create by the job end properties (Dependent=true) - Coutputnames % Output variables defined in the Evaluator - Cinputnames % Input variables required by the Evaluator + OutputNames % Output variables defined in the Evaluator + InputNames % Input variables required by the Evaluator end methods - function Xev=Evaluator(varargin) + function obj=Evaluator(varargin) % EVALUATOR This constructor defines an Evaluator that is collection % of Connectors. % % Please see the reference manual for the complete documentation % % Copyright 1983-2015 COSSAN Working Group - % Author: Edoardo Patelli + % Author: Edoardo Patelli + if nargin==0 % Crate empty object - return + superArg={}; + else + [requiredArgs, varargin] = opencossan.common.utilities.parseRequiredNameValuePairs(... + "Solvers", varargin{:}); + + % Define optional arguments and default values + + OptionalsArguments={... + "JobInterface", opencossan.highperformancecomputing.JobManagerInterface.empty(1,0);... + "Queues",[];... + "Hostnames",[];... + "ParallelEnvironments",[];... + "Slots",[];... + "IsCompiled",false;... + "MaxCuncurrentJobs",[];... + "RemoteInjectExtract", false;... + "VerticalSplit",[];... + "MaxNumberofJobs",[];... + "WrapperMatlabInputName","";... + "WrapperMatlabOutputName","";... + "SolversName",[]}; + + [optionalArg, superArg] = opencossan.common.utilities.parseOptionalNameValuePairs(... + [OptionalsArguments{:,1}],{OptionalsArguments{:,2}}, varargin{:}); + end - %% Process input argments - for k=1:2:length(varargin) + % Now we define all the inputs not filtered out by the parsers + obj@opencossan.common.CossanObject(superArg{:}); + + if nargin>0 + + obj.Solvers = requiredArgs.solvers; - switch lower(varargin{k}) - case 'xsolutionsequence' - assert(isa(varargin{k+1},'opencossan.workers.SolutionSequence'), ... - 'openCOSSAN:Evaluator', ... - 'The object of class %s is not valid after the PropertyName %s', ... - class(varargin{k+1}),varargin{k}) - - Xev.CXsolvers{end+1}=varargin{k+1}; - case 'xmio' - assert(isa(varargin{k+1},'opencossan.workers.Mio'), ... - 'openCOSSAN:Evaluator', ... - 'The object of class %s is not valid after the PropertyName %s', ... - class(varargin{k+1}),varargin{k}) - Xev.CXsolvers{end+1}=varargin{k+1}; - - case 'xconnector' - assert(isa(varargin{k+1},'opencossan.workers.Connector'), ... - 'openCOSSAN:Evaluator', ... - 'The object of class %s is not valid after the PropertyName %s', ... - class(varargin{k+1}),varargin{k}) - Xev.CXsolvers{end+1}=varargin{k+1}; - - case 'xmetamodel' - assert(isa(varargin{k+1},'opencossan.metamodels.MetaModel'), ... - 'openCOSSAN:Evaluator', ... - 'The object of class %s is not valid after the PropertyName %s', ... - class(varargin{k+1}),varargin{k}) - Xev.CXsolvers{end+1}=varargin{k+1}; - - case 'xjobmanagerinterface' - assert(isa(varargin{k+1},'opencossan.highperformancecomputing.JobManagerInterface'), ... - 'openCOSSAN:Evaluator', ... - 'The object of class %s is not valid after the PropertyName %s', ... - class(varargin{k+1}),varargin{k}) - Xev.XjobInterface=varargin{k+1}; - case 'cxjobmanagerinterface' - assert(isa(varargin{k+1}{1},'opencossan.highperformancecomputing.JobManagerInterface'), ... - 'openCOSSAN:Evaluator', ... - 'The object of class %s is not valid after the PropertyName %s', ... - class(varargin{k+1}{1}),varargin{k}) - Xev.XjobInterface=varargin{k+1}{1}; - case {'sdescription'} - Xev.Sdescription=varargin{k+1}; - case {'lremoteinjectextract'} - Xev.LremoteInjectExtract=varargin{k+1}; - case {'cshostnames'} - Xev.CShostnames=varargin{k+1}; - case {'csparallelenvironments','cspe'} - Xev.CSparallelEnvironments=varargin{k+1}; - case {'csqueues'} - Xev.CSqueues=varargin{k+1}; - case {'csnames','csmembers'} - Xev.CSnames=varargin{k+1}; - case {'vconcurrent','nconcurrent'} - Xev.Vconcurrent=varargin{k+1}; - case {'vslots','nslots'} - Xev.Vslots=varargin{k+1}; - case 'cxmembers' - % The object are retrieved from the cell array - for iobj=1:length(varargin{k+1}) - switch class(varargin{k+1}{iobj}) - case {'opencossan.workers.Connector',... - 'opencossan.workers.Mio',... - 'opencossan.workers.SolutionSequence'} - Xev.CXsolvers{end+1}=varargin{k+1}{iobj}; - case 'opencossan.highperformancecomputing.JobManagerInterface' - Xev.XjobInterface=varargin{k+1}{iobj}; - case {'opencossan.metamodels.ResponseSurface',... - 'opencossan.metamodels.KrigingModel'} - Xev.CXsolvers{end+1}=varargin{k+1}{iobj}; - case {'opencossan.reliability.PerformanceFunction'} - Xev.CXsolvers{end+1}=varargin{k+1}{iobj}; - otherwise - error('openCOSSAN:Evaluator',... - ['The object of class ' ... - class(varargin{k+1}{iobj}) ' not allowed']); - end - end - case 'ccxmembers' - % The object are retrieved from the cell array - for iobj=1:length(varargin{k+1}) - switch class(varargin{k+1}{iobj}{1}) - case {'opencossan.workers.Connector',... - 'opencossan.workers.Mio',... - 'opencossan.workers.SolutionSequence'} - Xev.CXsolvers{end+1}=varargin{k+1}{iobj}{1}; - case {'opencossan.metamodels.ResponseSurface'} - Xev.CXsolvers{end+1}=varargin{k+1}{iobj}{1}; - case 'opencossan.highperformancecomputing.JobManagerInterface' - Xev.XjobInterface=varargin{k+1}{iobj}{1}; - case {'opencossan.reliability.PerformanceFunction'} - Xev.CXsolvers{end+1}=varargin{k+1}{iobj}{1}; - otherwise - error('openCOSSAN:Evaluator',... - ['The object of class ' ... - class(varargin{k+1}{iobj}{1}) ' not allowed']); - end - end - case 'lverticalsplit' - Xev.LverticalSplit=varargin{k+1}; - otherwise - error('openCOSSAN:Evaluator',... - 'PropertyName %s is not a valid PropertyName',varargin{k}); + obj.SolversName = optionalArg.solversname; + + if ~isempty(obj.SolversName) + assert(numel(obj.Solvers) == numel(obj.SolversName),... + 'Evaluator:IllegalArguments',... + 'Number of solvers (%i) specified must be the same size of SolversName (%i)',... + numel(obj.Solvers),numel(obj.SolversName)); end + + obj.JobInterface = optionalArg.jobinterface; + obj.Queues = optionalArg.queues; + obj.Hostnames = optionalArg.hostnames; + obj.ParallelEnvironments = optionalArg.parallelenvironments; + obj.Slots = optionalArg.slots; + obj.IsCompiled = optionalArg.iscompiled; + obj.MaxCuncurrentJobs = optionalArg.maxcuncurrentjobs; + obj.VerticalSplit = optionalArg.verticalsplit; + obj.MaxNumberofJobs = optionalArg.maxnumberofjobs; + obj.WrapperMatlabInputName = optionalArg.wrappermatlabinputname; + obj.WrapperMatlabOutputName = optionalArg.wrappermatlaboutputname; end - % Validate Evaluator object - Xev=validateObject(Xev); end %end constructor - Xout=apply(Xobj,Pinput) % Run the analysis - - Xout=display(Xobj) % Show details of the evaluator + Xout=apply(Xobj,Pinput) % Run the analysis Xout=deterministicAnalysis(Xobj,Xinput) % Perform the deterministic analysis Xout=add(Xobj,varargin) % Add a worker to the evaluator object - function Coutputnames=get.Coutputnames(Xobj) + function OutputNames=get.OutputNames(Xobj) % Extract output names from the target object - if isempty(Xobj.CXsolvers) - Coutputnames={}; + if isempty(Xobj.Solvers) + OutputNames={}; else - Coutputnames={}; - for n=1:length(Xobj.CXsolvers) - if isrow(Xobj.CXsolvers{n}.OutputNames) - Caddoutput=Xobj.CXsolvers{n}.OutputNames; + OutputNames={}; + for n=1:length(Xobj.Solvers) + if isrow(Xobj.Solvers(n).OutputNames) + Caddoutput=Xobj.Solvers{n}.OutputNames; else - Caddoutput=transpose(Xobj.CXsolvers{n}.OutputNames); + Caddoutput=transpose(Xobj.Solvers(n).OutputNames); end - Coutputnames=[Coutputnames Caddoutput]; %#ok + OutputNames=[OutputNames Caddoutput]; %#ok end end end - function Cinputnames=get.Cinputnames(Xobj) + function InputNames=get.InputNames(Xobj) % Extract output names from the target object - if isempty(Xobj.CXsolvers) - Cinputnames={}; + if isempty(Xobj.Solvers) + InputNames={}; else - Cinputnames=Xobj.CXsolvers{1}.InputNames; + InputNames=Xobj.Solvers(1).InputNames; CoutEvaluator={}; - for n=2:length(Xobj.CXsolvers) - CaddInputs=Xobj.CXsolvers{n}.InputNames; % tmp variable + for n=2:length(Xobj.Solvers) + CaddInputs=Xobj.Solvers(n).InputNames; % tmp variable % Remove already present inputs Vindex=false(length(CaddInputs),1); for j=1:length(CaddInputs) - Vindex(j)=any(strcmp(Cinputnames,CaddInputs(j))); + Vindex(j)=any(strcmp(InputNames,CaddInputs(j))); end CaddInputs(Vindex)=[]; - if isrow(Xobj.CXsolvers{n-1}.Coutputnames) - Caddoutput=Xobj.CXsolvers{n-1}.Coutputnames; + if isrow(Xobj.CXsolvers(n-1).Coutputnames) + Caddoutput=Xobj.CXsolvers(n-1).Coutputnames; else - Caddoutput=transpose(Xobj.CXsolvers{n-1}.Coutputnames); + Caddoutput=transpose(Xobj.CXsolvers(n-1).Coutputnames); end CoutEvaluator=[CoutEvaluator Caddoutput]; %#ok @@ -229,7 +165,7 @@ % Remove Evaluator inputs provided by the previous Evaluator CaddInputs(strcmp(CoutEvaluator(j),CaddInputs))=[]; end - Cinputnames=[Cinputnames CaddInputs]; %#ok + InputNames=[InputNames CaddInputs]; %#ok end end end @@ -239,11 +175,11 @@ end methods (Access=protected) - [Xout,varargout]=executeWorkersLocal(Xobj,XsimInput); + [Xout,varargout]=executeWorkersLocal(Xobj,XsimInput); outputTable=executeWorkersHorizontal(Xobj,inputTable); outputTable=executeWorkersVertical(Xobj,inputTable); [Xout,varargout]=executeWorkersGrid(Xobj,XsimInput,Xjob); - Xout=validateObject(Xobj); + Xout=validateObject(Xobj); end methods (Static) diff --git a/+opencossan/+workers/@Evaluator/add.m b/+opencossan/+workers/@Evaluator/add.m index 2549d042..6e48cc95 100644 --- a/+opencossan/+workers/@Evaluator/add.m +++ b/+opencossan/+workers/@Evaluator/add.m @@ -29,34 +29,14 @@ %% Process input argments for k=1:2:length(varargin) switch lower(varargin{k}) - case 'xsolutionsequence' + case 'solver' assert(isa(varargin{k+1},'opencossan.workers.SolutionSequence'), ... 'openCOSSAN:Evaluator', ... 'The object of class %s is not valid after the PropertyName %s', ... class(varargin{k+1}),varargin{k}) - Xev.CXsolvers{end+1}=varargin{k+1}; - case 'xmio' - assert(isa(varargin{k+1},'opencossan.workers.Mio'), ... - 'openCOSSAN:Evaluator', ... - 'The object of class %s is not valid after the PropertyName %s', ... - class(varargin{k+1}),varargin{k}) - Xev.CXsolvers{end+1}=varargin{k+1}; - - case 'xconnector' - assert(isa(varargin{k+1},'opencossan.workers.Connector'), ... - 'openCOSSAN:Evaluator', ... - 'The object of class %s is not valid after the PropertyName %s', ... - class(varargin{k+1}),varargin{k}) - Xev.CXsolvers{end+1}=varargin{k+1}; - - case 'xmetamodel' - assert(strcmp(superclasses(varargin{k+1}),'opencossan.metamodels.MetaModel'), ... - 'openCOSSAN:Evaluator', ... - 'The object of class %s is not valid after the PropertyName %s', ... - class(varargin{k+1}),varargin{k}) - Xev.CXsolvers{end+1}=varargin{k+1}; - + Xev.Solvers(end+1)=varargin{k+1}; + case {'shostname','cshostnames'} Xev.CShostnames{end+1}=varargin{k+1}; case {'sparallelenvironment','spe','csparallelenviroments'} diff --git a/+opencossan/+workers/@Evaluator/apply.m b/+opencossan/+workers/@Evaluator/apply.m index 932d277a..079bb87c 100644 --- a/+opencossan/+workers/@Evaluator/apply.m +++ b/+opencossan/+workers/@Evaluator/apply.m @@ -61,22 +61,22 @@ % Add input variables to the SimulationData object XSimInp=SimulationData('Table',TableInput); -if isempty(Xobj.CXsolvers) +if isempty(Xobj.Solvers) XSimData=XSimInp; % Add input samples to the Simulation output object XSimData.Sdescription= 'created by the Evaluator with no workers'; return end -if Xobj.LverticalSplit +if Xobj.VerticalSplit % Setting the JobManager - if ~isempty(Xobj.CSqueues{1}) + if ~isempty(Xobj.Queues) % Setting the JobManager - Xjob=JobManager('XjobManagerInterface',Xobj.XjobInterface, ... - 'Squeue',Xobj.CSqueues{1},'Shostname',Xobj.CShostnames{1}, ... - 'SparallelEnvironment',Xobj.CSparallelEnvironments{1},... - 'Nslots',Xobj.Vslots(1),... - 'Nconcurrent',Xobj.Vconcurrent(1),... + Xjob=JobManager('XjobManagerInterface',Xobj.JobInterface, ... + 'Squeue',Xobj.Queues(1),'Shostname',Xobj.Hostnames(1), ... + 'SparallelEnvironment',Xobj.ParallelEnvironments{1},... + 'Nslots',Xobj.Slots(1),... + 'Nconcurrent',Xobj.MaxCuncurrentJobs(1),... 'Sdescription','JobManager created by Evaluator.executeWorkers'); % TODO: Not implemented @@ -87,13 +87,13 @@ else % Setting the JobManager - if ~isempty(Xobj.CSqueues{1}) + if ~isempty(Xobj.Queues) % Setting the JobManager Xjob=JobManager('XjobManagerInterface',Xobj.XjobInterface, ... - 'Squeue',Xobj.CSqueues{1},'Shostname',Xobj.CShostnames{1}, ... - 'SparallelEnvironment',Xobj.CSparallelEnvironments{1},... - 'Nslots',Xobj.Vslots(1),... - 'Nconcurrent',Xobj.Vconcurrent(1),... + 'Squeue',Xobj.Queues(1),'Shostname',Xobj.Hostnames{1}, ... + 'SparallelEnvironment',Xobj.ParallelEnvironments{1},... + 'Nslots',Xobj.Slots(1),... + 'Nconcurrent',Xobj.MaxCuncurrentJobs(1),... 'Sdescription','JobManager created by Evaluator.executeWorkers'); % TODO: Not implemented diff --git a/+opencossan/+workers/@Evaluator/createWrapper.m b/+opencossan/+workers/@Evaluator/createWrapper.m index 41ce6a75..0fdd8952 100644 --- a/+opencossan/+workers/@Evaluator/createWrapper.m +++ b/+opencossan/+workers/@Evaluator/createWrapper.m @@ -42,30 +42,30 @@ function createWrapper(Xobj,Nfid) fprinff(Nfid,'display([''Execution Time: '' datestr(now,0)]'); % save output fprintf(Nfid,'%% load Input '); -fprintf(Nfid, ' load %s; \n',Xobj.SwrapperInputName); +fprintf(Nfid, ' load %s; \n',Xobj.WrapperMatlabInputName); % Loop around samples fprintf(Nfid, ' for n=1:lenght()'); % Loop around the solvers -for isol=1:length(Xobj.CXsolvers) - fprintf(Nfid,'%% SOLVER %i (Type: %s)\n',isol,class(Xobj.CXsolvers{isol})); +for isol=1:length(Xobj.Solvers) + fprintf(Nfid,'%% SOLVER %i (Type: %s)\n',isol,class(Xobj.Solvers(isol))); % Define input and output filename Xobj.CXsolvers{n}.SwrapperInputName=sprintf('workerINPUT%i.mat',isol); Xobj.CXsolvers{n}.SwrapperOutputName=sprintf('workerOUTPUT%i.mat',isol); % Create wrapper for solvers - createWrapper(Xobj.CXsolvers{n},Nfid); + createWrapper(Xobj.Solvers(n),Nfid); end % Merge results fprintf(Nfid, 'if n==1, %s=%s; else %s=merge(%s,%s); end',... - Xobj.SwrapperOutputName,Xobj.CXsolvers{n}.SwrapperOutputName,... - Xobj.SwrapperOutputName,Xobj.SwrapperOutputName,Xobj.CXsolvers{n}.SwrapperOutputName); + Xobj.WrapperMatlabOutputName,Xobj.Solvers(n).WrapperMatlabOutputName,... + Xobj.WrapperMatlabOutputName,Xobj.WrapperMatlabOutputName,Xobj.CXsolvers(n).WrapperMatlabOutputName); % Close loop around samples fprintf(Nfid, ' end'); % save output fprintf(Nfid,'%% save output'); -fprintf(Nfid, ' save %s %s; \n',Xobj.SwrapperOutputName, Xobj.SwrapperOutputName); +fprintf(Nfid, ' save %s %s; \n',Xobj.WrapperMatlabOutputName, Xobj.WrapperMatlabOutputName); % Catch error fprintf(Nfid,'catch ME\n'); diff --git a/+opencossan/+workers/@Evaluator/deterministicAnalysis.m b/+opencossan/+workers/@Evaluator/deterministicAnalysis.m index 7c9ac923..33c698b6 100644 --- a/+opencossan/+workers/@Evaluator/deterministicAnalysis.m +++ b/+opencossan/+workers/@Evaluator/deterministicAnalysis.m @@ -30,7 +30,7 @@ %% Check inputs assert(isa(Xinput,'opencossan.common.inputs.Input'), ... - 'openCOSSAN:Evaluator:deterministicAnalysis', ... + 'Evaluator:deterministicAnalysis', ... 'An input object is required to perform the deterministic analysis'); %% Retrieve default (nominal values) diff --git a/+opencossan/+workers/@Evaluator/display.m b/+opencossan/+workers/@Evaluator/display.m deleted file mode 100644 index b124511f..00000000 --- a/+opencossan/+workers/@Evaluator/display.m +++ /dev/null @@ -1,39 +0,0 @@ -function display(Xobj) -%DISPLAY Display the details of the Evaluator object -% -% ================================================================== -% COSSAN-X - The next generation of the computational stochastic analysis -% University of Innsbruck, Copyright 1993-2011 IfM -% ================================================================== - -%% Name and description -OpenCossan.cossanDisp('===================================================================',2); -OpenCossan.cossanDisp([' Object ' class(Xobj) ' - Description: ' Xobj.Sdescription],1); -OpenCossan.cossanDisp('===================================================================',2); -%% Show the embedded object -if isempty(Xobj.CXsolvers) - OpenCossan.cossanDisp(' The evaluator contains no solvers',1); -else - OpenCossan.cossanDisp(' The evaluator contains: ',1); - for n=1:length(Xobj.CXsolvers) - OpenCossan.cossanDisp([sprintf('%2i Worker ',n) ' (' Xobj.CXsolvers{n}.Sdescription ') ' ],1); - OpenCossan.cossanDisp([' * type ' class(Xobj.CXsolvers{n}) ],1) - if isempty(Xobj.CSqueues{n}) - OpenCossan.cossanDisp([' * evaluate on the localhost'],1) - else - if isempty(Xobj.CShostnames{n}) - OpenCossan.cossanDisp([' * evaluate on the grid queue ' Xobj.CSqueues{n}],1) - else - OpenCossan.cossanDisp([' * evaluate on the grid queue ' ... - Xobj.CSqueues{n} ' and hostname ' Xobj.CShostnames{n}],1) - end - end - end -end - - - - - - - diff --git a/+opencossan/+workers/@Evaluator/executeWorkersGrid.m b/+opencossan/+workers/@Evaluator/executeWorkersGrid.m index ab0835f5..02198ba9 100644 --- a/+opencossan/+workers/@Evaluator/executeWorkersGrid.m +++ b/+opencossan/+workers/@Evaluator/executeWorkersGrid.m @@ -67,10 +67,10 @@ ' -r workers.remoteWorkerJob -nosplash -nodesktop']; % Copy input table and worker into the new grid folder % split table with inputs - TableInput = PinputALL(Vstart(irun):Vend(irun)); %#ok + TableInput = PinputALL(Vstart(irun):Vend(irun)); % get individual solver from Evaluator % TODO: how to loop on the available solvers??? - Xworker = Xobj.CXsolvers{1}; %#ok + Xworker = Xobj.Solvers(1); save (fullfile(OpenCossan.getCossanWorkingPath, Sfoldername, 'workerInput.mat'),'TableInput','Xworker'); end @@ -97,7 +97,7 @@ % Check status of job every second. % TODO: Get this value from JobManager pause(1); - Cstatus = Xjob.getJobStatus('CSjobID',CSjobID); %#ok % check the status of the one that are not yet completed only + Cstatus = Xjob.getJobStatus('CSjobID',CSjobID); % check the status of the one that are not yet completed only % TODO: Use properties of JobManagerInterface % It should be @@ -145,17 +145,17 @@ % Load results form output files [PoutputALL, Vresults] = Xmio.retrieveResults(Vresults,Vstart,Vend,PoutputALL,Xjob); % In case some results are missing, try to reload -if any(Vresults==0), +if any(Vresults==0) % if not all the simulation were readed correctly try againg [PoutputALL(Vresults==0), Vresults] = Xmio.retrieveResults(Vresults,Vstart,Vend,PoutputALL,Xjob); end % Manage results that could not be loaded -if any(Vresults==0), +if any(Vresults==0) % Include NaN in the output if the results of the simulation can not be % retrieved Vpos = find(Vresults==0); %determine results that could not be retrieved - for ij=1:length(Vpos), - if isa(PoutputALL,'struct'), + for ij=1:length(Vpos) + if isa(PoutputALL,'struct') for isim = Vstart(Vpos(ij)):Vend(Vpos(ij)) for ifield = 1:length(Xmio.Coutputnames) PoutputALL(isim).(Xmio.Coutputnames{ifield}) = NaN; @@ -172,10 +172,10 @@ end %% Export results - create output object -if isa(PoutputALL,'struct'), +if isa(PoutputALL,'struct') Xsimtmp=SimulationData('Tvalues',PoutputALL); -elseif isa(PoutputALL,'double'), - Xsimtmp=SimulationData('Mvalues',PoutputALL,'Cvariablenames',Xmio.Coutputnames); +elseif isa(PoutputALL,'double') + Xsimtmp=SimulationData('Mvalues',PoutputALL,'Cvariablenames',Xmio.OutputNames); else error('openCOSSAN:Mio:runJob','The output of a compiled MIO has to be a structure or a matrix'); end diff --git a/+opencossan/+workers/@Evaluator/executeWorkersHorizontal.m b/+opencossan/+workers/@Evaluator/executeWorkersHorizontal.m index 03bdd001..36f32eeb 100644 --- a/+opencossan/+workers/@Evaluator/executeWorkersHorizontal.m +++ b/+opencossan/+workers/@Evaluator/executeWorkersHorizontal.m @@ -48,9 +48,9 @@ TableOutputSolver=[]; % Evaluator execution -for n=1:length(Xobj.CXsolvers) +for n=1:length(Xobj.Solvers) OpenCossan.cossanDisp(['[Status:workers ] * Processing solver ' ... - num2str(n) '/' num2str(length(Xobj.CXsolvers))],3) + num2str(n) '/' num2str(length(Xobj.Solvers))],3) % Merge tableInput with output produced by the workers and then % pass only the inputs required by the specific worker. @@ -64,22 +64,22 @@ % Execute worker. % Only the input required by the worker are passes. No recheck % is needed in the evaluate method. - TableOutputSolverTmp=Xobj.CXsolvers{n}.evaluate(TableSolver(:,Xobj.CXsolvers{n}.InputNames)); + TableOutputSolverTmp=Xobj.Solvers(n).evaluate(TableSolver(:,Xobj.Solvers(n).InputNames)); %TableOutputSolverTmp=array2table(Xobj.CXsolvers{n}.evaluate(TableSolver(:,Xobj.CXsolvers{n}.Cinputnames)),'VariableNames',Xobj.Coutputnames); catch Exception warning('OpenCossan:Evaluator:executeWorkersHorizontal:workerFaild',... - sprintf('Unable to execute worker %i of type %s',n,class(Xobj.CXsolvers{n}))) + 'Unable to execute worker %i of type %s',n,class(Xobj.Solvers(n))) % Add meaningful error message msgID = 'OpenCossan:Evaluator:executeWorkersHorizontal:workerFaild'; - msg = sprintf('Unable to execute worker %i of type %s',n,class(Xobj.CXsolvers{n})); + msg = sprintf('Unable to execute worker %i of type %s',n,class(Xobj.Solvers(n))); causeException = MException(msgID,msg); % Store Exception in the Analysis object Xanalysis.ErrorsStack{end+1} = addCause(Exception,causeException); % Construct a tableOutputSolverTmp with NaN - TableOutputSolverTmp=array2table(NaN(height(TableSolver),length(Xobj.CXsolvers{n}.OutputNames)),... - 'VariableNames',Xobj.CXsolvers{n}.OutputNames); + TableOutputSolverTmp=array2table(NaN(height(TableSolver),length(Xobj.Solvers(n).OutputNames)),... + 'VariableNames',Xobj.Solvers{n}.OutputNames); end % Merge workers output for the currenct analysis TableOutputSolver=[TableOutputSolver, TableOutputSolverTmp]; %#ok diff --git a/+opencossan/+workers/@MatlabWorker/MatlabWorker.m b/+opencossan/+workers/@MatlabWorker/MatlabWorker.m new file mode 100644 index 00000000..0af4ea74 --- /dev/null +++ b/+opencossan/+workers/@MatlabWorker/MatlabWorker.m @@ -0,0 +1,124 @@ +classdef MatlabWorker < opencossan.workers.Worker + % Matlab Input/Output interface + % See also: Worker, Evaluator, Connector + % + % Author: Edoardo Patelli, Matteo Broggi, Marco De Angelis + % Institute for Risk and Uncertainty, University of Liverpool, UK + % email address: openengine@cossan.co.uk + % Website: http://www.cossan.co.uk + + %{ + This file is part of OpenCossan . + Copyright (C) 2006-2020 COSSAN WORKING GROUP + + OpenCossan is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License or, + (at your option) any later version. + + OpenCossan is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with OpenCossan. If not, see . + %} + + %% Properties of the object + properties % Public access + FullFileName(1,:) char % Default Full path of the matlab file + Format(1,1) string = 'table' % Define interface to the matlab file + IsFunction(1,1) logical % Define if the matlab script is a function + AdditionalPath(1,1) % Folder that contains additional files required for compiling MatlabWorker + Script(1,:) char % field that contains the script + end + + properties (SetAccess=private,Hidden=true)% Define private propeties + IsCompiled logical = false % compiled status + FunctionHandle % function handle for external file + end + + properties (Constant,Hidden) + % Available format + FormatTypes= ["table","structure","matrix","vectors"]; + end + %% Methods of the class + methods + + [XSimOut, Poutput] = deterministicAnalysis(Xmio,Xinput) % Evaluates the target function/script using the nominal values + + [XSimOut,Pout] = run(Xobj,varargin) % Evaluates the target function/script + + [XSimOut,Pout] = runJob(Xobj,varargin) %Evaluates the target function using the JobManager + + [Xobj] = compile(Xobj,varargin) %This method allows compiling the m-function within the MatlabWorker + + tableOutput = evaluate(thisObject,tableInput) + %% Constructor + function obj = MatlabWorker(varargin) + %% MatlabWorker Matlab Input/Output mapper + % + % MatlabWorker defines an object type, which can be used to map an + % input to the corresponding output, using a Matlab + % function defined in an m-file + % + % See also: Worker, + + + %% Process inputs + if nargin == 0 + superArg={}; + else + + % Define optional arguments with default values + OptionalsArguments={"FullFileName", [];... + "Format","table";... + "IsFunction",false;... + "AdditionalPath","";... + "Script",[];... + "FunctionHandle",[]}; + + % Parse the optional values + [optionalArg, superArg] = opencossan.common.utilities.parseOptionalNameValuePairs(... + [OptionalsArguments{:,1}],{OptionalsArguments{:,2}}, varargin{:}); + end + + % Now we define all the inputs not filtered out by the parsers + obj@opencossan.workers.Worker(superArg{:}); + + if nargin>0 + + obj.FullFileName = optionalArg.fullfilename; + obj.Format = optionalArg.format; + obj.IsFunction = optionalArg.isfunction; + obj.AdditionalPath = optionalArg.additionalpath; + obj.Script = optionalArg.script; + obj.FunctionHandle = optionalArg.functionhandle; + % Check the object + obj = obj.validateConstructor; + end + + end %of constructor + + PinputMio=prepareInput(Xobj,varargin); % Prepare the input file for the execution of MatlabWorker Object + + end %of methods + + %% Private methods + methods (Access = private) + Poutput = checkPinput(Xobj,Pinput) %this method checks the correctness of the objects given to the method run of MatlabWorker + XsimOut = createSimulationData(Xobj,Poutput) %this method checks the correctness of the matrix/structure output from MatlabWorker and create a simulationData + [PoutputALL, Vresults] = retrieveResults(Xobj,Vresults,Vstart,Vend,PoutputALL,Xjob) %method to retrieve results after evaluation of MatlabWorker + end %of private methods + + methods (Access = protected) + Xobj=validateConstructor(Xobj); % Validate the constructor + Pout=runScript(Xobj,Pinput); % evaluate a Matlab script. + Pout=runFunction(Xobj,Pinput); % evaluate a Matlab function. + [XSimOut,Pout] = runJobMatlab(Xobj,varargin) + [XSimOut,Pout] = runJobCompiled(Xobj,varargin) + end + + +end %of class definition diff --git a/+opencossan/+workers/@Mio/checkPinput.m b/+opencossan/+workers/@MatlabWorker/checkPinput.m similarity index 100% rename from +opencossan/+workers/@Mio/checkPinput.m rename to +opencossan/+workers/@MatlabWorker/checkPinput.m diff --git a/+opencossan/+workers/@Mio/compile.m b/+opencossan/+workers/@MatlabWorker/compile.m similarity index 100% rename from +opencossan/+workers/@Mio/compile.m rename to +opencossan/+workers/@MatlabWorker/compile.m diff --git a/+opencossan/+workers/@Mio/createSimulationData.m b/+opencossan/+workers/@MatlabWorker/createSimulationData.m similarity index 100% rename from +opencossan/+workers/@Mio/createSimulationData.m rename to +opencossan/+workers/@MatlabWorker/createSimulationData.m diff --git a/+opencossan/+workers/@Mio/createWrapper.m b/+opencossan/+workers/@MatlabWorker/createWrapper.m similarity index 100% rename from +opencossan/+workers/@Mio/createWrapper.m rename to +opencossan/+workers/@MatlabWorker/createWrapper.m diff --git a/+opencossan/+workers/@Mio/deterministicAnalysis.m b/+opencossan/+workers/@MatlabWorker/deterministicAnalysis.m similarity index 100% rename from +opencossan/+workers/@Mio/deterministicAnalysis.m rename to +opencossan/+workers/@MatlabWorker/deterministicAnalysis.m diff --git a/+opencossan/+workers/@Mio/evaluate.m b/+opencossan/+workers/@MatlabWorker/evaluate.m similarity index 100% rename from +opencossan/+workers/@Mio/evaluate.m rename to +opencossan/+workers/@MatlabWorker/evaluate.m diff --git a/+opencossan/+workers/@Mio/execute.m b/+opencossan/+workers/@MatlabWorker/execute.m old mode 100755 new mode 100644 similarity index 100% rename from +opencossan/+workers/@Mio/execute.m rename to +opencossan/+workers/@MatlabWorker/execute.m diff --git a/+opencossan/+workers/@Mio/prepareInput.m b/+opencossan/+workers/@MatlabWorker/prepareInput.m similarity index 100% rename from +opencossan/+workers/@Mio/prepareInput.m rename to +opencossan/+workers/@MatlabWorker/prepareInput.m diff --git a/+opencossan/+workers/@Mio/retrieveResults.m b/+opencossan/+workers/@MatlabWorker/retrieveResults.m similarity index 100% rename from +opencossan/+workers/@Mio/retrieveResults.m rename to +opencossan/+workers/@MatlabWorker/retrieveResults.m diff --git a/+opencossan/+workers/@Mio/run.m b/+opencossan/+workers/@MatlabWorker/run.m similarity index 100% rename from +opencossan/+workers/@Mio/run.m rename to +opencossan/+workers/@MatlabWorker/run.m diff --git a/+opencossan/+workers/@Mio/runFunction.m b/+opencossan/+workers/@MatlabWorker/runFunction.m similarity index 100% rename from +opencossan/+workers/@Mio/runFunction.m rename to +opencossan/+workers/@MatlabWorker/runFunction.m diff --git a/+opencossan/+workers/@Mio/runJob.m b/+opencossan/+workers/@MatlabWorker/runJob.m similarity index 100% rename from +opencossan/+workers/@Mio/runJob.m rename to +opencossan/+workers/@MatlabWorker/runJob.m diff --git a/+opencossan/+workers/@Mio/runJobCompiled.m b/+opencossan/+workers/@MatlabWorker/runJobCompiled.m similarity index 100% rename from +opencossan/+workers/@Mio/runJobCompiled.m rename to +opencossan/+workers/@MatlabWorker/runJobCompiled.m diff --git a/+opencossan/+workers/@Mio/runJobMatlab.m b/+opencossan/+workers/@MatlabWorker/runJobMatlab.m similarity index 100% rename from +opencossan/+workers/@Mio/runJobMatlab.m rename to +opencossan/+workers/@MatlabWorker/runJobMatlab.m diff --git a/+opencossan/+workers/@Mio/runScript.m b/+opencossan/+workers/@MatlabWorker/runScript.m similarity index 93% rename from +opencossan/+workers/@Mio/runScript.m rename to +opencossan/+workers/@MatlabWorker/runScript.m index fe44a712..e52aeb9e 100644 --- a/+opencossan/+workers/@Mio/runScript.m +++ b/+opencossan/+workers/@MatlabWorker/runScript.m @@ -50,9 +50,9 @@ Moutput = zeros(size(Minput,1),length(Xmio.OutputNames)); case 'table' % Rename input - TableInput=Psamples; - TableOutput=table(zeros(height(TableInput),length(Xmio.OutputNames)),... - 'VariableNames',Xmio.OutputNames); + TableInput=Psamples; + TableOutput = array2table(zeros(height(TableInput),length(Xmio.OutputNames))); + TableOutput.Properties.VariableNames = Xmio.OutputNames; otherwise error('OpenCossan:Mio:runScript:wrongFormat',... 'Format %s is not valid in runScript',Xmio.Format) @@ -91,8 +91,8 @@ case 'table' Poutput = TableOutput; case 'structure' - Poutput = Toutput; + Poutput = Toutput; case 'matrix' - Poutput = Moutput; + Poutput = Moutput; end diff --git a/+opencossan/+workers/@Mio/validateConstructor.m b/+opencossan/+workers/@MatlabWorker/validateConstructor.m similarity index 100% rename from +opencossan/+workers/@Mio/validateConstructor.m rename to +opencossan/+workers/@MatlabWorker/validateConstructor.m diff --git a/+opencossan/+workers/@Mio/Mio.m b/+opencossan/+workers/@Mio/Mio.m deleted file mode 100644 index e626e463..00000000 --- a/+opencossan/+workers/@Mio/Mio.m +++ /dev/null @@ -1,148 +0,0 @@ -classdef Mio < opencossan.workers.Worker - % Matlab Input/Output interface - % See also: https://cossan.co.uk/wiki/index.php/@Mio - % - % Author: Edoardo Patelli, Matteo Broggi, Marco De Angelis - % Institute for Risk and Uncertainty, University of Liverpool, UK - % email address: openengine@cossan.co.uk - % Website: http://www.cossan.co.uk - - % ===================================================================== - % This file is part of openCOSSAN. The open general purpose matlab - % toolbox for numerical analysis, risk and uncertainty quantification. - % - % openCOSSAN is free software: you can redistribute it and/or modify - % it under the terms of the GNU General Public License as published by - % the Free Software Foundation, either version 3 of the License. - % - % openCOSSAN is distributed in the hope that it will be useful, - % but WITHOUT ANY WARRANTY; without even the implied warranty of - % MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - % GNU General Public License for more details. - % - % You should have received a copy of the GNU General Public License - % along with openCOSSAN. If not, see . - % ===================================================================== - - %% Properties of the object - properties % Public access - FullFileName(1,:) char % Default Full path of the matlab file - Format(1,1) string = 'table' % Define interface to the matlab file - IsFunction(1,1) logical = false % Define if the matlab script is a function - AdditionalPath(1,1) % Folder that contains additional files required for compiling Mio - Script(1,:) char % field that contains the script - end - - properties (SetAccess=private,Hidden=true)% Define private propeties - IsCompiled logical = false % compiled status - FunctionHandle % function handle for external file - end - - properties (Constant,Hidden) - % Available format - FormatTypes={'table','structure','matrix','vectors'}; - end - %% Methods of the class - methods - - [XSimOut, Poutput] = deterministicAnalysis(Xmio,Xinput) % Evaluates the target function/script using the nominal values - - [XSimOut,Pout] = run(Xobj,varargin) % Evaluates the target function/script - - [XSimOut,Pout] = runJob(Xobj,varargin) %Evaluates the target function using the JobManager - - [Xobj] = compile(Xobj,varargin) %This method allows compiling the m-function within the Mio - - tableOutput = evaluate(thisObject,tableInput) - %% Constructor - function Xobj = Mio(varargin) - %% Mio Matlab Input/Output mapper - % - % Mio defines an object type, which can be used to map an - % input to the corresponding output, using a Matlab - % function defined in an m-file - % - % See also: https://cossan.co.uk/wiki/index.php/@Mio - % - % Author: Edoardo Patelli and Matteo Broggi - % COSSAN WORKING GROUP - % email address: openengine@cossan.co.uk - % Website: http://www.cossan.co.uk - - % ===================================================================== - % This file is part of openCOSSAN. The open general purpose matlab - % toolbox for numerical analysis, risk and uncertainty quantification. - % - % openCOSSAN is free software: you can redistribute it and/or modify - % it under the terms of the GNU General Public License as published by - % the Free Software Foundation, either version 3 of the License. - % - % openCOSSAN is distributed in the hope that it will be useful, - % but WITHOUT ANY WARRANTY; without even the implied warranty of - % MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - % GNU General Public License for more details. - % - % You should have received a copy of the GNU General Public License - % along with openCOSSAN. If not, see . - % ===================================================================== - - %% Process inputs - if nargin == 0 - return % Create empty object - else - % Process inputs via inputParser - p = inputParser; - p.FunctionName = 'opencossan.workers.Mio'; - - % Use default values - p.addParameter('Description',Xobj.Description); - p.addParameter('FullFileName',Xobj.FullFileName); - p.addParameter('Format',Xobj.Format); - p.addParameter('IsFunction',Xobj.IsFunction); - p.addParameter('AdditionalPath',Xobj.AdditionalPath); - p.addParameter('Script',Xobj.Script); - % From the superclass - p.addParameter('OutputNames',Xobj.OutputNames); - p.addParameter('InputNames',Xobj.InputNames); - p.addParameter('IsKeepSimulationFiles',Xobj.IsKeepSimulationFiles); - p.addParameter('FunctionHandle',Xobj.FunctionHandle); - - p.parse(varargin{:}); - - % Assign input to objects properties - Xobj.Description = p.Results.Description; - Xobj.FullFileName = p.Results.FullFileName; - Xobj.Format = p.Results.Format; - Xobj.OutputNames = p.Results.OutputNames; - Xobj.InputNames = p.Results.InputNames; - Xobj.Script = p.Results.Script; - Xobj.IsFunction = p.Results.IsFunction; - Xobj.AdditionalPath = p.Results.AdditionalPath; - Xobj.IsKeepSimulationFiles = p.Results.IsKeepSimulationFiles; - Xobj.FunctionHandle = p.Results.FunctionHandle; - - Xobj = Xobj.validateConstructor; - end - end %of constructor - - PinputMio=prepareInput(Xobj,varargin); % Prepare the input file for the execution of Mio Object - - end %of methods - - %% Private methods - methods (Access = private) - Poutput = checkPinput(Xobj,Pinput) %this method checks the correctness of the objects given to the method run of Mio - XsimOut = createSimulationData(Xobj,Poutput) %this method checks the correctness of the matrix/structure output from Mio and create a simulationData - [PoutputALL, Vresults] = retrieveResults(Xobj,Vresults,Vstart,Vend,PoutputALL,Xjob) %method to retrieve results after evaluation of Mio - end %of private methods - - methods (Access = protected) - Xobj=validateConstructor(Xobj); % Validate the constructor - Pout=runScript(Xobj,Pinput); % evaluate a Matlab script. - Pout=runFunction(Xobj,Pinput); % evaluate a Matlab function. - [XSimOut,Pout] = runJobMatlab(Xobj,varargin) - [XSimOut,Pout] = runJobCompiled(Xobj,varargin) - end - - -end %of class definition diff --git a/+opencossan/+workers/@Worker/Worker.m b/+opencossan/+workers/@Worker/Worker.m index e29fe33d..7b447366 100644 --- a/+opencossan/+workers/@Worker/Worker.m +++ b/+opencossan/+workers/@Worker/Worker.m @@ -1,9 +1,9 @@ -classdef Worker < opencossan.common.CossanObject - %WORKERS Abstract class to define the COSSAN workers. +classdef Worker < matlab.mixin.Heterogeneous & opencossan.common.CossanObject + %WORKERS Abstract class to define the OpenCossan workers. % The class worker provides a common interface for the Worker % objects. % - % See Also: http://cossan.co.uk/wiki/index.php/@Worker + % See Also: SolutionSequence, Mio, Evaluator, Connector % % % Author: Edoardo Patelli @@ -31,13 +31,30 @@ properties OutputNames cell {opencossan.common.utilities.isunique(OutputNames)} InputNames cell {opencossan.common.utilities.isunique(InputNames)} - IsKeepSimulationFiles(1,1) logical = false % Keep simulation files + KeepSimulationFiles(1,1) logical = false % Keep simulation files end methods (Abstract) % Evaluate the workerObject based on the realizations provided in a % Table object TableOutput=evaluate(workerObject,TableInput); - end + end + + methods + function obj = Worker(varargin) + + [requiredArgs, varargin] = opencossan.common.utilities.parseRequiredNameValuePairs(... + ["InputNames","OutputNames"], varargin{:}); + + [optionalArgs, superArg] = opencossan.common.utilities.parseOptionalNameValuePairs(... + "KeepSimulationFiles",{false}, varargin{:}); + + obj@opencossan.common.CossanObject(superArg{:}); + + obj.InputNames=requiredArgs.inputnames; + obj.OutputNames=requiredArgs.outputnames; + obj.KeepSimulationFiles=optionalArgs.keepsimulationfiles; + end + end end diff --git a/+opencossan/@OpenCossan/validateCossanInputs.m b/+opencossan/@OpenCossan/validateCossanInputs.m index f2989042..9341f8ed 100644 --- a/+opencossan/@OpenCossan/validateCossanInputs.m +++ b/+opencossan/@OpenCossan/validateCossanInputs.m @@ -22,7 +22,7 @@ function validateCossanInputs(varargin) along with OpenCossan. If not, see . %} -warning('This feature is obsolete. Use Validate Property Values. /n Use https://uk.mathworks.com/help/matlab/matlab_oop/validate-property-values.html') +warning('This feature is obsolete. Use Validate Property Values. \n Use https://uk.mathworks.com/help/matlab/matlab_oop/validate-property-values.html') %% Check if the arguments has been passed as pairs of PropertyName/Value if mod(length(varargin),2) diff --git a/docs/tutorials/CantileverBeam/TutorialCantileverBeam.m b/docs/tutorials/CantileverBeam/TutorialCantileverBeam.m index 88178608..b15e237d 100644 --- a/docs/tutorials/CantileverBeam/TutorialCantileverBeam.m +++ b/docs/tutorials/CantileverBeam/TutorialCantileverBeam.m @@ -5,28 +5,24 @@ % % % The documentation and the problem description of this example is available -% here: -% -% -%

Copyright 2006-2014: COSSAN working group

-% Author: Edoardo-Patelli
-% Institute for Risk and Uncertainty, University of Liverpool, UK -%
COSSAN web site: http://www.cossan.co.uk -%

-% This file is part of openCOSSAN. The open source general purpose matlab toolbox -% for numerical analysis, risk and uncertainty quantification (http://www.cossan.co.uk). -%
-% openCOSSAN is free software: you can redistribute it and/or modify -% it under the terms of the GNU General Public License as published by -% the Free Software Foundation, either version 3 of the License. -% openCOSSAN is distributed in the hope that it will be useful, -% but WITHOUT ANY WARRANTY; without even the implied warranty of -% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -% GNU General Public License for more details. -% You should have received a copy of the GNU General Public License -% along with openCOSSAN. If not, see http://www.gnu.org/licenses/". -% +% here: + + +%{ +This file is part of OpenCossan . +Copyright (C) 2006-2018 COSSAN WORKING GROUP +OpenCossan is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License or, +(at your option) any later version. + +OpenCossan is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. +You should have received a copy of the GNU General Public License +along with OpenCossan. If not, see . +%} %% MATLAB solver: % See also: diff --git a/docs/tutorials/CantileverBeam/TutorialCantileverBeamAnsys.m b/docs/tutorials/CantileverBeam/TutorialCantileverBeamAnsys.m index b181d112..1c5c124d 100644 --- a/docs/tutorials/CantileverBeam/TutorialCantileverBeamAnsys.m +++ b/docs/tutorials/CantileverBeam/TutorialCantileverBeamAnsys.m @@ -2,10 +2,24 @@ % Model Definition and Uncertainty Quantification % This script run the Cantilever Beam Tutorial for Ansys in the COSSAN-X Engine % -% See Also http://cossan.cfd.liv.ac.uk/wiki/index.php/Cantilever_Beam -% -% $Copyright~1993-2011,~COSSAN~Working~Group,~University~of~Innsbruck,~Austria$ -% $Author: Edoardo-Patelli$ +% See Also http://cossan.co.uk/wiki/index.php/Cantilever_Beam + + +%{ +This file is part of OpenCossan . +Copyright (C) 2006-2018 COSSAN WORKING GROUP +OpenCossan is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License or, +(at your option) any later version. + +OpenCossan is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. +You should have received a copy of the GNU General Public License +along with OpenCossan. If not, see . +%} %% Input files manipulation diff --git a/docs/tutorials/CossanObjects/Files4Mio/ExampleMioFunction.m b/docs/tutorials/CossanObjects/Files4MatlabWorker/ExampleMatlabWorkerFunction.m similarity index 100% rename from docs/tutorials/CossanObjects/Files4Mio/ExampleMioFunction.m rename to docs/tutorials/CossanObjects/Files4MatlabWorker/ExampleMatlabWorkerFunction.m diff --git a/docs/tutorials/CossanObjects/Files4Mio/ExampleMioMatrix.m b/docs/tutorials/CossanObjects/Files4MatlabWorker/ExampleMatlabWorkerMatrix.m similarity index 100% rename from docs/tutorials/CossanObjects/Files4Mio/ExampleMioMatrix.m rename to docs/tutorials/CossanObjects/Files4MatlabWorker/ExampleMatlabWorkerMatrix.m diff --git a/docs/tutorials/CossanObjects/Files4Mio/ExampleMioStructure.m b/docs/tutorials/CossanObjects/Files4MatlabWorker/ExampleMatlabWorkerStructure.m similarity index 100% rename from docs/tutorials/CossanObjects/Files4Mio/ExampleMioStructure.m rename to docs/tutorials/CossanObjects/Files4MatlabWorker/ExampleMatlabWorkerStructure.m diff --git a/docs/tutorials/CossanObjects/Files4Mio/ExampleMioTable.m b/docs/tutorials/CossanObjects/Files4MatlabWorker/ExampleMatlabWorkerTable.m old mode 100755 new mode 100644 similarity index 100% rename from docs/tutorials/CossanObjects/Files4Mio/ExampleMioTable.m rename to docs/tutorials/CossanObjects/Files4MatlabWorker/ExampleMatlabWorkerTable.m diff --git a/docs/tutorials/CossanObjects/TutorialJobManagerSlurm.m b/docs/tutorials/CossanObjects/TutorialJobManagerSlurm.m new file mode 100644 index 00000000..6db2b65b --- /dev/null +++ b/docs/tutorials/CossanObjects/TutorialJobManagerSlurm.m @@ -0,0 +1,63 @@ +% Tutorial for the JobManagerSlurm Object +% This tutorial shows how to create and use a JobManager object + +% This tutorial shows only the most basic commands and features of +% JobMangarer. +clear +close all +clc; + +% Get queue from SLURM called partition +[a,partition]=system('sinfo --Format=partition'); +partion(1)=[]; + +Xjm = opencossan.highperformancecomputing.JobManagerSlurm; + +% This should call system('sinfo --Format=partition') +partion=Xjm.getPartition; + +% Get job list system('sqme') +queues=Xjm.getQueues; + +% Submit job system('sbactch ....') +[]=Xjm.submit() + + + + Xg = JobManager('Sdescription','test #1',... + 'Squeue','pizzas64.q',... + 'SpreExeCmd','echo preexecmd ', ... + 'SpostExeCmd','echo postexecmd '); + [~,SuserName] = system('whoami'); SuserName=SuserName(1:end-1); + CSjobID(1)=Xg.submitJob; + + + %% Run a model + + + RV1=RandomVariable('Sdistribution','normal', 'mean',0,'std',1); %#ok +RV2=RandomVariable('Sdistribution','normal', 'mean',0,'std',1); %#ok +% Define the RVset +Xrvs1=RandomVariableSet('Cmembers',{'RV1', 'RV2'}); +% Define Xinput +Xin = Input('Sdescription','Input satellite_inp'); +Xin = add(Xin,Xrvs1); + + +Xm=Mio( 'Sdescription', 'This is our Model', ... + 'Sscript','for j=1:length(Tinput), Toutput(j).out=-Tinput(j).RV1+Tinput(j).RV2; end', ... + 'Liostructure',true,... + 'Coutputnames',{'out'},... + 'Cinputnames',{'RV1','RV2'},... + 'Lfunction',false); % This flag specify if the .m file is a script or a function. + +%% Construct the Evaluator +% First mode (The object are passed by reference) + +Xeval1 = Evaluator('CXmembers',{Xm}, ... + 'Xjobmanagerinterface',Xjm,... + 'CSqueues',{'MatlabPool',''},... + 'Vconcurrent',[6]); + +Xmdl=Model('Xinput',Xin,'Xevaluator',Xeval1,'Sdescription','The Model'); + % Construct a Mio object diff --git a/docs/tutorials/CossanObjects/TutorialMio.m b/docs/tutorials/CossanObjects/TutorialMatlabWorker.m similarity index 66% rename from docs/tutorials/CossanObjects/TutorialMio.m rename to docs/tutorials/CossanObjects/TutorialMatlabWorker.m index 7e4ae173..ec61d16b 100644 --- a/docs/tutorials/CossanObjects/TutorialMio.m +++ b/docs/tutorials/CossanObjects/TutorialMatlabWorker.m @@ -1,5 +1,5 @@ -% Tutorial for Mio. -% This tutorial shows how to construct and use a Mio object. +% Tutorial for MatlabWorker. +% This tutorial shows how to construct and use a MatlabWorker object. % clear close all @@ -8,11 +8,11 @@ import opencossan.workers.* import opencossan.common.inputs.* % Author: Edoardo Patelli -StutorialPath = fileparts(which('TutorialMio.m')); +StutorialPath = fileparts(which('TutorialMatlabWorker.m')); %% Define additional objects -% In order to use a Mio, it is necessary to create an Input object and -% create a Model with such an input and the Mio. Please refer to the +% In order to use a MatlabWorker, it is necessary to create an Input object and +% create a Model with such an input and the MatlabWorker. Please refer to the % relevant objects tutorials for additional help. % Create random variables and random variable set @@ -20,11 +20,11 @@ RV=opencossan.common.inputs.random.NormalRandomVariable('mean',0,'std',1); % Define a set of Random Variables of Nrv independent, identically % distributed random variables -Xrvs1=opencossan.common.inputs.random.RandomVariableSet('names',{'RV'},'members',[RV],'Nrv',Nrv); +Xrvs1=opencossan.common.inputs.random.RandomVariableSet.fromIidRandomVariables('RandomVariable',RV,'Number',Nrv,'NamePrefix','RV_'); % Add a parameter % The parameter is unnecessary here but it is used to test the behaviour of -% the Mio object. +% the MatlabWorker object. Xpar=Parameter('value',5); % Define Input object @@ -35,8 +35,8 @@ % Add some samples to the Input Xinp = Xinp.sample('Nsamples',10); -%% Define MIO -% There are different ways to connect a Matlab function or a script with OpenCossan via the Mio class. +%% Define MatlabWorker +% There are different ways to connect a Matlab function or a script with OpenCossan via the MatlabWorker class. % % Define Matlab Format % The input and Output can be passed as: @@ -52,32 +52,32 @@ % % E.g.: Tinput(1).RV_1 contains the value of RV_1 for the first sample % -% The Matlab function used is found in 'Files4Mio/ExampleMioStructure.m' +% The Matlab function used is found in 'Files4MatlabWorker/ExampleMatlabWorkerStructure.m' -Xm = Mio('Description', 'Performance function', ... - 'FullFileName',fullfile(StutorialPath,'Files4Mio','ExampleMioStructure.m'), ... +Xm = MatlabWorker('Description', 'Performance function', ... + 'FullFileName',fullfile(StutorialPath,'Files4MatlabWorker','ExampleMatlabWorkerStructure.m'), ... 'Format','structure',... % This flag specify the type of I/O 'OutputNames',{'Out1';'Out2'},... % This field is mandatory 'InputNames',{'RV_1';'RV_2'},... % This field is mandatory 'IsFunction',true); % This flag specify if the .m file is a script or a function. -% The method evaluate is used yo execute the Mio. It requires a table +% The method evaluate is used yo execute the MatlabWorker. It requires a table % input. -% Test the Mio +% Test the MatlabWorker TableOutput = evaluate(Xm,Xinp.getTable); -% The run mio return a Matlab table containg only the results of the -% Mio +% The run MatlabWorker return a Matlab table containg only the results of the +% MatlabWorker display(TableOutput) %% First test - Use MonteCarlo simulation % Define Evaluator -Xev = Evaluator('Xmio',Xm); +Xev = Evaluator('Solvers',Xm); % Define probmodel -Xmodel = opencossan.common.Model('XInput',Xinp,'XEvaluator',Xev); +Xmodel = opencossan.common.Model('Input',Xinp,'Evaluator',Xev); % Apply -opencossan.OpenCossan.cossanDisp('Test 1: 100 samples (20 batches) - Monte Carlo Simulation - MIO I/O Structures'); +opencossan.OpenCossan.cossanDisp('Test 1: 100 samples (20 batches) - Monte Carlo Simulation - MatlabWorker I/O Structures'); Xmc = opencossan.simulations.MonteCarlo('Nsamples',100,'Nbatches',20); Xo = Xmc.apply(Xmodel); @@ -92,28 +92,28 @@ % Note that only mono-dimensional Parameters and Functions can be used with % this input/output type. % -% The Matlab function used is found in 'Files4Mio/ExampleMioFunction.m' -XmB = Mio('Description', 'Performance function', ... - 'FullFileName',fullfile(StutorialPath,'Files4Mio','ExampleMioFunction.m'), ... +% The Matlab function used is found in 'Files4MatlabWorker/ExampleMatlabWorkerFunction.m' +XmB = MatlabWorker('Description', 'Performance function', ... + 'FullFileName',fullfile(StutorialPath,'Files4MatlabWorker','ExampleMatlabWorkerFunction.m'), ... 'OutputNames',{'Out1' 'Out2'},... % This field is mandatory 'InputNames',{'RV_1' 'RV_2' 'Xpar'},... % This field is mandatory 'Format','vectors',... % This flag specify the type of I/O 'IsFunction',true); % This flag specify if the .m file is a script or a function. -% Test the Mio +% Test the MatlabWorker TableOutput = evaluate(XmB,Xinp.getTable); -% The run mio return a Matlab table containg only the results of the -% Mio +% The run MatlabWorker return a Matlab table containg only the results of the +% MatlabWorker display(TableOutput) %% Second - Use MonteCarlo simulation % Define Evaluator -Xev = Evaluator('Xmio',XmB); +Xev = Evaluator('Solvers',XmB); % Define probmodel -Xmodel = opencossan.common.Model('XInput',Xinp,'XEvaluator',Xev); +Xmodel = opencossan.common.Model('Input',Xinp,'Evaluator',Xev); % Apply -opencossan.OpenCossan.cossanDisp('Test 2: 100 samples (20 batches) - MCS - MIO Function'); +opencossan.OpenCossan.cossanDisp('Test 2: 100 samples (20 batches) - MCS - MatlabWorker Function'); Xmc = opencossan.simulations.MonteCarlo('Nsamples',100,'Nbatches',20); Xo = Xmc.apply(Xmodel); @@ -121,27 +121,27 @@ % Matlab matrices are used here for Input/Output. Each row of the matrix % correspond to a sample, and each column to an input quantity, in the % order specified by the CinputNames field. As an example, the elements -% (9,2) of the matrix in the Mio will contain the 9-th sampled value of +% (9,2) of the matrix in the MatlabWorker will contain the 9-th sampled value of % RV_2. % Please note that only random variables can be accessed with this % input/output strategy. % -% The Matlab function used is found in 'Files4Mio/ExampleMioMatrix.m' -XmC = Mio('Description', 'Performance function', ... - 'FullFileName',fullfile(StutorialPath,'Files4Mio','ExampleMioMatrix.m'), ... +% The Matlab function used is found in 'Files4MatlabWorker/ExampleMatlabWorkerMatrix.m' +XmC = MatlabWorker('Description', 'Performance function', ... + 'FullFileName',fullfile(StutorialPath,'Files4MatlabWorker','ExampleMatlabWorkerMatrix.m'), ... 'OutputNames',{'Out1';'Out2'},... % This field is mandatory 'InputNames',{'RV_1';'RV_2'},... % This field is mandatory 'Format','matrix',... % This flag specify the type of I/O 'IsFunction',true); % This flag specify if the .m file is a script or a function. -% Test the Mio +% Test the MatlabWorker TableOutput = evaluate(XmC,Xinp.getTable); -% The run mio return a Matlab table containg only the results of the -% Mio +% The run MatlabWorker return a Matlab table containg only the results of the +% MatlabWorker display(TableOutput) %% Define Matlab Script -% In this Mio, a script is used instead of a function. matlab script can be +% In this MatlabWorker, a script is used instead of a function. matlab script can be % either passed from a file (not shown here), or passed in a single-line % string, as shown here. % When using a script, either the structure or the matrix input/output can @@ -155,28 +155,28 @@ 'Toutput(i).Out2 = Tinput(i).RV_1;'... 'end']; -XmD = Mio('Description', 'Performance function', ... +XmD = MatlabWorker('Description', 'Performance function', ... 'Script',Sscript, ... % Define the script 'OutputNames',{'Out1';'Out2'},... % This field is mandatory 'InputNames',{'RV_1';'RV_2'},... % This field is mandatory 'Format','structure',... % This flag specify the type of I/O 'IsFunction',false); % This flag specify if the .m file is a script or a function. -% Test the Mio +% Test the MatlabWorker TableOutput = evaluate(XmD,Xinp.getTable); -% The run mio return a Matlab table containg only the results of the -% Mio +% The run MatlabWorker return a Matlab table containg only the results of the +% MatlabWorker display(TableOutput) % All the three different interface should produce %% Third test - Use MonteCarlo simulation %8.1. Define Evaluator -Xev = Evaluator('Xmio',XmD); +Xev = Evaluator('Solvers',XmD); %8.2. Define probmodel -Xmodel = opencossan.common.Model('XInput',Xinp,'XEvaluator',Xev); +Xmodel = opencossan.common.Model('Input',Xinp,'Evaluator',Xev); %8.3. Apply -opencossan.OpenCossan.cossanDisp('Test 3: 100 samples (20 batches) - MCS - MIO Matrix'); +opencossan.OpenCossan.cossanDisp('Test 3: 100 samples (20 batches) - MCS - MatlabWorker Matrix'); Xmc = opencossan.simulations.MonteCarlo('Nsamples',100,'Nbatches',20); Xo = Xmc.apply(Xmodel); @@ -184,18 +184,18 @@ %% CASE D % Matlab tables are used here for Input/Output. % -% The Matlab function used is found in 'Files4Mio/ExampleMioMatrix.m' -XmDfun = Mio('Description', 'Performance function', ... - 'FullFileName',fullfile(StutorialPath,'Files4Mio','ExampleMioTable.m'), ... +% The Matlab function used is found in 'Files4MatlabWorker/ExampleMatlabWorkerMatrix.m' +XmDfun = MatlabWorker('Description', 'Performance function', ... + 'FullFileName',fullfile(StutorialPath,'Files4MatlabWorker','ExampleMatlabWorkerTable.m'), ... 'OutputNames',{'Out1';'Out2'},... % This field is mandatory 'InputNames',{'RV_1';'RV_2'},... % This field is mandatory 'Format','table',... % This flag specify the type of I/O 'IsFunction',true); % This flag specify if the .m file is a script or a function. -% Test the Mio +% Test the MatlabWorker TableOutput = evaluate(XmDfun,Xinp.getTable); -% The run mio return a Matlab table containg only the results of the -% Mio +% The run MatlabWorker return a Matlab table containg only the results of the +% MatlabWorker display(TableOutput) @@ -203,26 +203,26 @@ 'Out2 = TableInput.RV_1;'... 'TableOutput=array2table([Out1 Out2],''VariableNames'',{''Out1'',''Out2''})']; - % The Matlab function used is found in 'Files4Mio/ExampleMioMatrix.m' -XmDscript = Mio('Description', 'Performance function', ... + % The Matlab function used is found in 'Files4MatlabWorker/ExampleMatlabWorkerMatrix.m' +XmDscript = MatlabWorker('Description', 'Performance function', ... 'Script',Sscript, ... % Define the script 'OutputNames',{'Out1';'Out2'},... % This field is mandatory 'InputNames',{'RV_1';'RV_2'},... % This field is mandatory 'Format','table',... % This flag specify the type of I/O 'IsFunction',false); % This flag specify if the .m file is a script or a function. -% Test the Mio +% Test the MatlabWorker TableOutput = evaluate(XmDscript,Xinp.getTable); -% The run mio return a Matlab table containg only the results of the -% Mio +% The run MatlabWorker return a Matlab table containg only the results of the +% MatlabWorker display(TableOutput) %% Third test - Use MonteCarlo simulation %8.1. Define Evaluator -Xev = Evaluator('Xmio',XmDfun); +Xev = Evaluator('Solvers',XmDfun); %8.2. Define probmodel -Xmodel = opencossan.common.Model('XInput',Xinp,'XEvaluator',Xev); +Xmodel = opencossan.common.Model('Input',Xinp,'Evaluator',Xev); %8.3. Apply opencossan.OpenCossan.cossanDisp('Test 3: 100 samples (20 batches) - MCS - MIO Matrix'); Xmc = opencossan.simulations.MonteCarlo('Nsamples',100,'Nbatches',20); diff --git a/docs/tutorials/CossanObjects/TutorialSensitivity.m b/docs/tutorials/CossanObjects/TutorialSensitivity.m index 8ebc97f5..4f6d4b99 100644 --- a/docs/tutorials/CossanObjects/TutorialSensitivity.m +++ b/docs/tutorials/CossanObjects/TutorialSensitivity.m @@ -7,32 +7,50 @@ % simply: $y=x_1^2+2x_2-x_3$ % % A more realistic example is provided in the TutorialInfectionDynamicModel. -% See Also: http://cossan.cfd.liv.ac.uk/wiki/index.php/Infection_Dynamic_Model +% See Also: http://cossan.co.uk/wiki/index.php/Infection_Dynamic_Model % -% +%{ +This file is part of OpenCossan . +Copyright (C) 2006-2018 COSSAN WORKING GROUP +OpenCossan is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License or, +(at your option) any later version. + +OpenCossan is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. +You should have received a copy of the GNU General Public License +along with OpenCossan. If not, see . +%} + % $Copyright~1993-2015,~COSSAN~Working~Group,~University~of~Liverpool,~UK$ % $Author: Edoardo-Patelli$ -close all -clear -clc; + +% Clean up enviroment +close('all'), clear, clc; +import opencossan.common.inputs.random.* +import opencossan.common.inputs.* +import opencossan.workers.* %% Problem setup % In this examples we consider only 3 uniform random variables -Xrv1 = opencossan.common.inputs.random.UniformRandomVariable('bounds',[-1,1]); -Xrv2 = opencossan.common.inputs.random.UniformRandomVariable('bounds',[-1,1]); -Xrv3 = opencossan.common.inputs.random.UniformRandomVariable('bounds',[-1,1]); -Xrv4 = opencossan.common.inputs.random.UniformRandomVariable('bounds',[-1,1]); -Xpar = opencossan.common.inputs.Parameter('value',0); -Xrvset = opencossan.common.inputs.random.RandomVariableSet('names',{'Xrv1','Xrv2','Xrv3' 'Xrv4'},'members',[Xrv1;Xrv2;Xrv3;Xrv4]); -Xin = opencossan.common.inputs.Input('randomVariableSet',Xrvset,'parameter',Xpar); +Xrv1 = UniformRandomVariable('bounds',[-1,1],'Description','First Random Variable'); +Xrv2 = UniformRandomVariable('bounds',[-1,1],'Description','Second Random Variable'); +Xrv3 = UniformRandomVariable('bounds',[-1,1],'Description','Thirth Random Variable'); +Xrv4 = UniformRandomVariable('bounds',[-1,1],'Description','Dummy Random Variable'); +Xpar = Parameter('value',0); +Xrvset = RandomVariableSet('names',{'Xrv1','Xrv2','Xrv3' 'Xrv4'},'members',[Xrv1;Xrv2;Xrv3;Xrv4]); +Xin = Input('RandomVariableSet',Xrvset,'Parameter',Xpar); % The model is defined using a Mio object -Xm = opencossan.workers.Mio('Script','for j=1:length(Tinput), Toutput(j).out1=Tinput(j).Xrv1^2+2*Tinput(j).Xrv2-Tinput(j).Xrv3; end', ... +Xm = Mio('Script','for j=1:length(Tinput), Toutput(j).out1=Tinput(j).Xrv1^2+2*Tinput(j).Xrv2-Tinput(j).Xrv3; end', ... 'OutputNames',{'out1'},... 'InputNames',{'Xrv1' 'Xrv2' 'Xrv3'},... 'Format','structure'); -Xev = opencossan.workers.Evaluator('Xmio',Xm); +Xev = Evaluator('Xmio',Xm); Xmdl = opencossan.common.Model('Xinput',Xin,'Xevaluator',Xev); Xperfun = PerformanceFunction('OutputName','vg','Demand','Xpar','Capacity','out1'); diff --git a/test/+workers/EvaluatorTest.m b/test/unit/+opencossan/+workers/EvaluatorTest.m similarity index 78% rename from test/+workers/EvaluatorTest.m rename to test/unit/+opencossan/+workers/EvaluatorTest.m index afa306bd..e8f9937f 100644 --- a/test/+workers/EvaluatorTest.m +++ b/test/unit/+opencossan/+workers/EvaluatorTest.m @@ -28,70 +28,71 @@ methods (Test) %% Constructor function emptyConstructor(testCase) - Xe = workers.Evaluator; - testCase.assertClass(Xe,'workers.Evaluator'); + Xe = opencossan.workers.Evaluator; + testCase.assertClass(Xe,'opencossan.workers.Evaluator'); end function constructorShouldSetDescription(testCase) - Xe = workers.Evaluator('Sdescription','Evaluator'); - testCase.assertEqual(Xe.Sdescription,'Evaluator'); + Xe = workers.Evaluator('Description','Evaluator'); + testCase.assertEqual(Xe.Description,'Evaluator'); end - function constructorShouldSetMio(testCase) - Xmio = workers.Mio('Coutputnames',{'output'},... + function constructorShouldSetMatlabWorker(testCase) + Xmio = workers.MatlabWorker('Coutputnames',{'output'},... 'Cinputnames',{'input'},'Sscript','%do nothing'); - Xe = workers.Evaluator('Xmio',Xmio); - testCase.assertEqual(Xe.CXsolvers{1},Xmio); + Xe = workers.Evaluator('Solvers',Xmio,'SolversName',"Xmio"); + testCase.assertEqual(Xe.Solvers{1},Xmio); end - function constructorShouldSetConnector(testCase) - Xcon = workers.Connector; - Xe = workers.Evaluator('Xconnector',Xcon); - testCase.assertEqual(Xe.CXsolvers{1},Xcon); + function constructorShouldFail(testCase) + Xmio = workers.MatlabWorker('Coutputnames',{'output'},... + 'Cinputnames',{'input'},'Sscript','%do nothing'); + + Xe = workers.Evaluator('Solvers',Xmio,'SolversName',["MatlabWorker" "ExtraName"]); + testCase.assertEqual(Xe.Solvers{1},XMatlabWorker); end + + function constructorShouldSetJobManagerInterface(testCase) Xjmi = highperformancecomputing.JobManagerInterface(); - Xe = workers.Evaluator('XjobManagerInterface',Xjmi); - testCase.assertEqual(Xe.XjobInterface,Xjmi); - Xe = workers.Evaluator('CXjobManagerInterface',{Xjmi}); + Xe = workers.Evaluator('JobManager',Xjmi); testCase.assertEqual(Xe.XjobInterface,Xjmi); end function constructorShouldSetLremoteInjectExtract(testCase) - Xe = workers.Evaluator('LremoteInjectExtract',true); - testCase.assertTrue(Xe.LremoteInjectExtract); + Xe = workers.Evaluator('RemoteInjectExtract',true); + testCase.assertTrue(Xe.RemoteInjectExtract); end function constructorShouldSetHostNames(testCase) Xcon = workers.Connector; - Xmio = workers.Mio('Coutputnames',{'output'},... + Xmio = workers.MatlabWorker('Coutputnames',{'output'},... 'Cinputnames',{'input'},'Sscript','%do nothing'); - CSqueues = {'Queue1' 'Queue2'}; - CShostnames = {'Host1','Host2'}; - Xe = workers.Evaluator('CXmembers',{Xcon Xmio},'CShostnames',CShostnames,... - 'CSqueues',CSqueues); - testCase.assertEqual(Xe.CShostnames,CShostnames); - + Squeues = ["Queue1" "Queue2"]; + Shostnames = ["Host1","Host2"]; + Xe = workers.Evaluator('Solvers',{Xcon Xmio},'Hostnames',Shostnames,... + 'Queues',Squeues); + testCase.assertEqual(Xe.CShostnames,CShostnames); end function constructorShouldSetQueues(testCase) Xcon = workers.Connector; Xmio = workers.Mio('Coutputnames',{'output'},... 'Cinputnames',{'input'},'Sscript','%do nothing'); - CSqueues = {'Queue1' 'Queue2'}; + Squeues = ["Queue1" "Queue2"]; Xe = workers.Evaluator('Xconnector',Xcon,'Xmio',... - Xmio,'CSqueues',CSqueues); - testCase.assertEqual(Xe.CSqueues,CSqueues); + Xmio,'Queues',Squeues); + testCase.assertEqual(Xe.Queues,Squeues); end function constructorShouldSetVconcurrent(testCase) Xcon = workers.Connector; Xmio = workers.Mio('Coutputnames',{'output'},... 'Cinputnames',{'input'},'Sscript','%do nothing'); - Xe = workers.Evaluator('Xconnector',Xcon,'Xmio',... - Xmio,'Vconcurrent',[Inf 4]); - testCase.assertEqual(Xe.Vconcurrent,[Inf 4]); + Xe = workers.Evaluator('Solvers',{Xcon Xmio},... + 'Concurrent',[Inf 4]); + testCase.assertEqual(Xe.Concurrent,[Inf 4]); end function constructorShouldSetMembers(testCase) diff --git a/test/unit/+opencossan/+workers/MioTest.m b/test/unit/+opencossan/+workers/MatlabWorkerTest.m similarity index 73% rename from test/unit/+opencossan/+workers/MioTest.m rename to test/unit/+opencossan/+workers/MatlabWorkerTest.m index 33e92711..cd595075 100644 --- a/test/unit/+opencossan/+workers/MioTest.m +++ b/test/unit/+opencossan/+workers/MatlabWorkerTest.m @@ -1,7 +1,7 @@ -classdef MioTest < matlab.unittest.TestCase - % MIOTEST Unit tests for the class - % opencossan.workers.Mio - % see http://cossan.co.uk/wiki/index.php/@Mio +classdef MatlabWorkerTest < matlab.unittest.TestCase + % MatlabWorkerTEST Unit tests for the class + % opencossan.workers.MatlabWorker + % % % @author Jasper Behrensdorf % @author Edoardo Patelli @@ -24,10 +24,10 @@ properties Xin; - workingDirectoryFunctions = fullfile(opencossan.OpenCossan.getRoot,'test','data','workers','Mio','functions'); - workingDirectoryFunctionsDeployed = fullfile(opencossan.OpenCossan.getRoot,'test','data','workers','Mio','functionsDeployed'); - workingDirectoryScripts = fullfile(opencossan.OpenCossan.getRoot,'test','data','workers','Mio','scripts'); - isdeployedDirectory = fullfile(opencossan.OpenCossan.getRoot,'test','data','workers','Mio','isdeployed'); + workingDirectoryFunctions = fullfile(opencossan.OpenCossan.getRoot,'test','data','workers','MatlabWorker','functions'); + workingDirectoryFunctionsDeployed = fullfile(opencossan.OpenCossan.getRoot,'test','data','workers','MatlabWorker','functionsDeployed'); + workingDirectoryScripts = fullfile(opencossan.OpenCossan.getRoot,'test','data','workers','MatlabWorker','scripts'); + isdeployedDirectory = fullfile(opencossan.OpenCossan.getRoot,'test','data','workers','MatlabWorker','isdeployed'); end methods (TestClassSetup) @@ -48,7 +48,7 @@ function createInput(testCase) methods (Test) %% constructor function constructorMinimal(testCase) - Xm = opencossan.workers.Mio('InputNames',{'x1','x2'},'OutputNames',{'out1','out2'},... + Xm = opencossan.workers.MatlabWorker('InputNames',{'x1','x2'},'OutputNames',{'out1','out2'},... 'Script','Toutput.out1=Tinput.x1;Toutput.out2=-Tinput.x2;'); testCase.assertEqual(Xm.InputNames,{'x1','x2'}); @@ -57,37 +57,37 @@ function constructorMinimal(testCase) end function constructorFile(testCase) - Xm = opencossan.workers.Mio('Description','Unit Test Mio',... + Xm = opencossan.workers.MatlabWorker('Description','Unit Test MatlabWorker',... 'OutputNames',{'diff1';'diff2'},... 'InputNames',{'Xrv1';'Xrv2'},... 'Path',testCase.workingDirectoryFunctions,... 'FullFileName','differenceStructure.m',... 'Format','structure'); - testCase.assertEqual(Xm.Description,'Unit Test Mio'); + testCase.assertEqual(Xm.Description,'Unit Test MatlabWorker'); testCase.assertEqual(Xm.Format,'structure'); testCase.assertEqual(Xm.Path,testCase.workingDirectoryFunctions); testCase.assertEqual(Xm.File,'differenceStructure.m'); end function WrongInputsToConstructor(testCase) % Checks that the command fails when the wrong input is passed to the constructor - testCase.verifyError(@() opencossan.workers.Mio('Sunexistingproperty',':-)'),... - 'OpenCossan:workers:Mio') + testCase.verifyError(@() opencossan.workers.MatlabWorker('Sunexistingproperty',':-)'),... + 'OpenCossan:workers:MatlabWorker') end function GiveNonExistingFileToConstructor(testCase) % Checks that the command fails due to incorrect file specified to be passed to the constructor - testCase.verifyError(@() opencossan.workers.Mio('FullFileName','thisfiledoesntexist.m', ... - 'Format','structure'),'OpenCossan:workers:Mio:NonExistingFunction') + testCase.verifyError(@() opencossan.workers.MatlabWorker('FullFileName','thisfiledoesntexist.m', ... + 'Format','structure'),'OpenCossan:workers:MatlabWorker:NonExistingFunction') end function GiveNonUniqueInputNames(testCase) % Checks that the command fails due to incorrect file specified to be passed to the constructor - testCase.verifyError(@() opencossan.workers.Mio('Inputnames',{'Xrv1';'Xrv1'}),... - 'OpenCossan:workers:Mio:NonUniqueName') + testCase.verifyError(@() opencossan.workers.MatlabWorker('Inputnames',{'Xrv1';'Xrv1'}),... + 'OpenCossan:workers:MatlabWorker:NonUniqueName') end %% run - function runMioWithStructurePassingInput(testCase) - Xm = opencossan.workers.Mio('Outputnames',{'diff1';'diff2'},... + function runMatlabWorkerWithStructurePassingInput(testCase) + Xm = opencossan.workers.MatlabWorker('Outputnames',{'diff1';'diff2'},... 'Inputnames',{'Xrv1';'Xrv2'},... 'FullFileName',fullfile(testCase.workingDirectoryFunctions,'differenceStructure.m'),... 'IsFunction',true,'Format','structure'); @@ -99,8 +99,8 @@ function runMioWithStructurePassingInput(testCase) testCase.assertSize(Pout.getValues('Names',{'diff1' 'diff2'}),[10 2]); end - function runMioWithStructurePassingSamples(testCase) - Xm = opencossan.workers.Mio('Outputnames',{'diff1';'diff2'},... + function runMatlabWorkerWithStructurePassingSamples(testCase) + Xm = opencossan.workers.MatlabWorker('Outputnames',{'diff1';'diff2'},... 'Inputnames',{'Xrv1';'Xrv2'},... 'FullFileName',fullfile(testCase.workingDirectoryFunctions,'differenceStructure.m'),... 'IsFunction',true,'Format','structure'); @@ -112,8 +112,8 @@ function runMioWithStructurePassingSamples(testCase) testCase.assertSize(Pout.getValues('Names',{'diff1' 'diff2'}),[10 2]); end - function runMioWithStructurePassingStructure(testCase) - Xm = opencossan.workers.Mio('OutputNames',{'diff1';'diff2'},... + function runMatlabWorkerWithStructurePassingStructure(testCase) + Xm = opencossan.workers.MatlabWorker('OutputNames',{'diff1';'diff2'},... 'OuputNames',{'Xrv1';'Xrv2'},... 'FullFileName',fullfile(testCase.workingDirectoryFunctions,'differenceStructure.m'),... 'IsFunction',true,'format','structure'); @@ -125,8 +125,8 @@ function runMioWithStructurePassingStructure(testCase) testCase.assertSize(Pout.getValues('Names',{'diff1' 'diff2'}),[10 2]); end - function runMioWithStructurePassingMatrix(testCase) - Xm = opencossan.workers.Mio('Outputnames',{'diff1';'diff2'},... + function runMatlabWorkerWithStructurePassingMatrix(testCase) + Xm = opencossan.workers.MatlabWorker('Outputnames',{'diff1';'diff2'},... 'Inputnames',{'Xrv1';'Xrv2'},... 'FullFileName',fullfile(testCase.workingDirectoryFunctions,'differenceStructure.m'),... 'IsFunction',true,'Format','structure'); @@ -138,8 +138,8 @@ function runMioWithStructurePassingMatrix(testCase) testCase.assertSize(Pout.getValues('Names',{'diff1' 'diff2'}),[10 2]); end - function runMioWithMatrixPassingInput(testCase) - Xm = opencossan.workers.Mio('OutputNames',{'diff1';'diff2'},... + function runMatlabWorkerWithMatrixPassingInput(testCase) + Xm = opencossan.workers.MatlabWorker('OutputNames',{'diff1';'diff2'},... 'InputNames',{'Xrv1';'Xrv2'},... 'FullFileName',fullfile(testCase.workingDirectoryFunctions,'differenceMatrix.m'),... 'IsFunction',true,'Format','matrix'); @@ -151,8 +151,8 @@ function runMioWithMatrixPassingInput(testCase) testCase.assertSize(Pout.getValues('Names',{'diff1' 'diff2'}),[10 2]); end - function runMioWithMatrixPassingSamples(testCase) - Xm = opencossan.workers.Mio('OutputNames',{'diff1';'diff2'},... + function runMatlabWorkerWithMatrixPassingSamples(testCase) + Xm = opencossan.workers.MatlabWorker('OutputNames',{'diff1';'diff2'},... 'InputNames',{'Xrv1';'Xrv2'},... 'FullFileName',fullfile(testCase.workingDirectoryFunctions,'differenceMatrix.m'),... 'IsFunction',true,'Format','matrix'); @@ -164,8 +164,8 @@ function runMioWithMatrixPassingSamples(testCase) testCase.assertSize(Pout.getValues('CSnames',{'diff1' 'diff2'}),[10 2]); end - function runMioWithMatrixPassingMatrix(testCase) - Xm = opencossan.workers.Mio('OutputNames',{'diff1';'diff2'},... + function runMatlabWorkerWithMatrixPassingMatrix(testCase) + Xm = opencossan.workers.MatlabWorker('OutputNames',{'diff1';'diff2'},... 'InputNames',{'Xrv1';'Xrv2'},... 'FullFileName',fullfile(testCase.workingDirectoryFunctions,'differenceMatrix.m'),.... 'IsFunction',true,'Format','matrix'); @@ -177,8 +177,8 @@ function runMioWithMatrixPassingMatrix(testCase) testCase.assertSize(Pout.getValues('Names',{'diff1' 'diff2'}),[10 2]); end - function runMioWithMatrixPassingStructure(testCase) - Xm = opencossan.workers.Mio('OutputNames',{'diff1';'diff2'},... + function runMatlabWorkerWithMatrixPassingStructure(testCase) + Xm = opencossan.workers.MatlabWorker('OutputNames',{'diff1';'diff2'},... 'InputNames',{'Xrv1';'Xrv2'},... 'FullFileName',fullfile(testCase.workingDirectoryFunctions,'differenceMatrix.m'),.... 'IsFunction',true,'Format','matrix'); @@ -190,10 +190,10 @@ function runMioWithMatrixPassingStructure(testCase) testCase.assertSize(Pout.getValues('Names',{'diff1' 'diff2'}),[10 2]); end - % TODO rumMioWithVector (Multiple Inputs Outputs) + % TODO rumMatlabWorkerWithVector (Multiple Inputs Outputs) - function runMioWithScriptAndStructurePassingInput(testCase) - Xm = opencossan.workers.Mio('OutputNames',{'diff1';'diff2'},... + function runMatlabWorkerWithScriptAndStructurePassingInput(testCase) + Xm = opencossan.workers.MatlabWorker('OutputNames',{'diff1';'diff2'},... 'InputNames',{'Xrv1';'Xrv2'},... 'FullFileName',fullfile(testCase.workingDirectoryFunctions,'differenceMatrix.m'),.... 'IsFunction',true,'Format','structure'); @@ -205,8 +205,8 @@ function runMioWithScriptAndStructurePassingInput(testCase) testCase.assertSize(Pout.getValues('Names',{'diff1' 'diff2'}),[10 2]); end - function runMioWithScriptAndStructurePassingSamples(testCase) - Xm = opencossan.workers.Mio('OutputNames',{'diff1';'diff2'},... + function runMatlabWorkerWithScriptAndStructurePassingSamples(testCase) + Xm = opencossan.workers.MatlabWorker('OutputNames',{'diff1';'diff2'},... 'InputNames',{'Xrv1';'Xrv2'},... 'FullFileName',fullfile(testCase.workingDirectoryFunctions,'differenceMatrix.m'),.... 'IsFunction',true,'Format','structure'); @@ -218,8 +218,8 @@ function runMioWithScriptAndStructurePassingSamples(testCase) testCase.assertSize(Pout.getValues('Names',{'diff1' 'diff2'}),[10 2]); end - function runMioWithScriptAndStructurePassingStructure(testCase) - Xm = opencossan.workers.Mio('OutputNames',{'diff1';'diff2'},... + function runMatlabWorkerWithScriptAndStructurePassingStructure(testCase) + Xm = opencossan.workers.MatlabWorker('OutputNames',{'diff1';'diff2'},... 'InputNames',{'Xrv1';'Xrv2'},... 'FullFileName',fullfile(testCase.workingDirectoryFunctions,'differenceStructure.m'),... 'Format','structure'); @@ -231,8 +231,8 @@ function runMioWithScriptAndStructurePassingStructure(testCase) testCase.assertSize(Pout.getValues('Names',{'diff1' 'diff2'}),[10 2]); end - function runMioWithScriptAndStructurePassingMatrix(testCase) - Xm = opencossan.workers.Mio('OutputNames',{'diff1';'diff2'},... + function runMatlabWorkerWithScriptAndStructurePassingMatrix(testCase) + Xm = opencossan.workers.MatlabWorker('OutputNames',{'diff1';'diff2'},... 'InputNames',{'Xrv1';'Xrv2'},... 'FullFileName',fullfile(testCase.workingDirectoryFunctions,'differenceStructure.m'),... 'Format','matrix'); @@ -244,8 +244,8 @@ function runMioWithScriptAndStructurePassingMatrix(testCase) testCase.assertSize(Pout.getValues('Names',{'diff1' 'diff2'}),[10 2]); end - function runMioWithScriptAndMatrixPassingInput(testCase) - Xm = opencossan.workers.Mio('OutputNames',{'diff1';'diff2'},... + function runMatlabWorkerWithScriptAndMatrixPassingInput(testCase) + Xm = opencossan.workers.MatlabWorker('OutputNames',{'diff1';'diff2'},... 'Inputnames',{'Xrv1';'Xrv2'},... 'FullFileName',fullfile(testCase.workingDirectoryFunctions,'differenceMatrix.m'),... 'Format','matrix'); @@ -257,8 +257,8 @@ function runMioWithScriptAndMatrixPassingInput(testCase) testCase.assertSize(Pout.getValues('Names',{'diff1' 'diff2'}),[10 2]); end - function runMioWithScriptAndMatrixPassingSamples(testCase) - Xm = opencossan.workers.Mio('OutputNames',{'diff1';'diff2'},... + function runMatlabWorkerWithScriptAndMatrixPassingSamples(testCase) + Xm = opencossan.workers.MatlabWorker('OutputNames',{'diff1';'diff2'},... 'InputNames',{'Xrv1';'Xrv2'},... 'FullFileName',fullfile(testCase.workingDirectoryFunctions,'differenceMatrix.m'),... 'Format','matrix'); @@ -270,8 +270,8 @@ function runMioWithScriptAndMatrixPassingSamples(testCase) testCase.assertSize(Pout.getValues('Names',{'diff1' 'diff2'}),[10 2]); end - function runMioWithScriptAndMatrixPassingStructure(testCase) - Xm = opencossan.workers.Mio('OutputNames',{'diff1';'diff2'},... + function runMatlabWorkerWithScriptAndMatrixPassingStructure(testCase) + Xm = opencossan.workers.MatlabWorker('OutputNames',{'diff1';'diff2'},... 'InputNames',{'Xrv1';'Xrv2'},... 'FullFileName',fullfile(testCase.workingDirectoryFunctions,'differenceMatrix.m'),... 'Format','matrix'); @@ -283,8 +283,8 @@ function runMioWithScriptAndMatrixPassingStructure(testCase) testCase.assertSize(Pout.getValues('Names',{'diff1' 'diff2'}),[10 2]); end - function runMioWithScriptAndMatrixPassingMatrix(testCase) - Xm = opencossan.workers.Mio('OutputNames',{'diff1';'diff2'},... + function runMatlabWorkerWithScriptAndMatrixPassingMatrix(testCase) + Xm = opencossan.workers.MatlabWorker('OutputNames',{'diff1';'diff2'},... 'InputNames',{'Xrv1';'Xrv2'},... 'FullFileName',fullfile(testCase.workingDirectoryFunctions,'differenceMatrix.m'),... 'Format','structure'); From a62a8c1de577f7c509efc0d7e929f005bc7124f4 Mon Sep 17 00:00:00 2001 From: Edoardo Patelli Date: Sat, 15 Feb 2020 23:21:36 +0000 Subject: [PATCH 3/9] Fix Unitest Evaluator --- .../PerformanceFunction.m | 4 +- +opencossan/+workers/@Connector/Connector.m | 6 - +opencossan/+workers/@Evaluator/Evaluator.m | 17 +- +opencossan/+workers/@Worker/Worker.m | 36 ++-- .../TutorialCantileverBeamMatlab.m | 4 +- ...TutorialCantileverBeamMatlabOptimization.m | 8 +- ...lCantileverBeamMatlabReliabilityAnalysis.m | 2 +- .../unit/+opencossan/+workers/EvaluatorTest.m | 171 ++++++++---------- 8 files changed, 119 insertions(+), 129 deletions(-) diff --git a/+opencossan/+reliability/@PerformanceFunction/PerformanceFunction.m b/+opencossan/+reliability/@PerformanceFunction/PerformanceFunction.m index 559d408e..5769d10a 100644 --- a/+opencossan/+reliability/@PerformanceFunction/PerformanceFunction.m +++ b/+opencossan/+reliability/@PerformanceFunction/PerformanceFunction.m @@ -1,4 +1,4 @@ -classdef PerformanceFunction < opencossan.workers.Mio +classdef PerformanceFunction < opencossan.workers.MatlabWorker % PERFORMANCEFUNCTION This class define the performance function for the % realiability analysis. It is a subclass of workers.Mio. % @@ -97,7 +97,7 @@ end end - obj = obj@opencossan.workers.Mio(super_args{:}); + obj = obj@opencossan.workers.MatlabWorker(super_args{:}); if nargin > 0 obj.Demand = p.Results.Demand; diff --git a/+opencossan/+workers/@Connector/Connector.m b/+opencossan/+workers/@Connector/Connector.m index efa2b91f..d8828a23 100644 --- a/+opencossan/+workers/@Connector/Connector.m +++ b/+opencossan/+workers/@Connector/Connector.m @@ -110,12 +110,6 @@ % along with openCOSSAN. If not, see . % ===================================================================== - - %% Argument Check - if opencossan.OpenCossan.getChecks - opencossan.OpenCossan.validateCossanInputs(varargin{:}); - end - if nargin==0 return end diff --git a/+opencossan/+workers/@Evaluator/Evaluator.m b/+opencossan/+workers/@Evaluator/Evaluator.m index 27087db9..552d6de5 100644 --- a/+opencossan/+workers/@Evaluator/Evaluator.m +++ b/+opencossan/+workers/@Evaluator/Evaluator.m @@ -24,7 +24,7 @@ %} properties - JobInterface opencossan.highperformancecomputing.JobManagerInterface % Define JobManager to submit job to grid/cluster computer + JobManager opencossan.highperformancecomputing.JobManagerInterface % Define JobManager to submit job to grid/cluster computer Solvers(1,:) %List of OpenCossan Workers opencossan.workers.Worker SolversName(1,:) string % Names of the workers (optional) Queues(1,:) cell % Where to submit solvers @@ -32,7 +32,7 @@ ParallelEnvironments(:,1) cell % Name of the parallel environment of each solver Slots(1,:) double {mustBeInteger} % Number of slots used in each job IsCompiled(1,:) logical % Number of slots used in each job - MaxCuncurrentJobs(1,:) double {mustBeInteger,mustBePositive} = 1 % Number of concurrent execution of each solver + MaxCuncurrentJobs(1,:) double {mustBePositive} = 1 % Number of concurrent execution of each solver RemoteInjectExtract = false %TODO: make it true by default VerticalSplit = false % if true split the analysis in vertical components (see wiki for more details) MaxNumberofJobs double {mustBeInteger,mustBePositive} = 1 % max number of jobs submitted for each analysis @@ -66,7 +66,7 @@ % Define optional arguments and default values OptionalsArguments={... - "JobInterface", opencossan.highperformancecomputing.JobManagerInterface.empty(1,0);... + "JobManager", opencossan.highperformancecomputing.JobManagerInterface.empty(1,0);... "Queues",[];... "Hostnames",[];... "ParallelEnvironments",[];... @@ -101,13 +101,14 @@ numel(obj.Solvers),numel(obj.SolversName)); end - obj.JobInterface = optionalArg.jobinterface; + obj.JobManager = optionalArg.jobmanager; obj.Queues = optionalArg.queues; obj.Hostnames = optionalArg.hostnames; obj.ParallelEnvironments = optionalArg.parallelenvironments; obj.Slots = optionalArg.slots; obj.IsCompiled = optionalArg.iscompiled; obj.MaxCuncurrentJobs = optionalArg.maxcuncurrentjobs; + obj.RemoteInjectExtract=optionalArg.remoteinjectextract; obj.VerticalSplit = optionalArg.verticalsplit; obj.MaxNumberofJobs = optionalArg.maxnumberofjobs; obj.WrapperMatlabInputName = optionalArg.wrappermatlabinputname; @@ -129,7 +130,7 @@ OutputNames={}; for n=1:length(Xobj.Solvers) if isrow(Xobj.Solvers(n).OutputNames) - Caddoutput=Xobj.Solvers{n}.OutputNames; + Caddoutput=Xobj.Solvers(n).OutputNames; else Caddoutput=transpose(Xobj.Solvers(n).OutputNames); end @@ -154,10 +155,10 @@ end CaddInputs(Vindex)=[]; - if isrow(Xobj.CXsolvers(n-1).Coutputnames) - Caddoutput=Xobj.CXsolvers(n-1).Coutputnames; + if isrow(Xobj.Solvers(n-1).OutputNames) + Caddoutput=Xobj.Solvers(n-1).OutputNames; else - Caddoutput=transpose(Xobj.CXsolvers(n-1).Coutputnames); + Caddoutput=transpose(Xobj.Solvers(n-1).OutputNames); end CoutEvaluator=[CoutEvaluator Caddoutput]; %#ok diff --git a/+opencossan/+workers/@Worker/Worker.m b/+opencossan/+workers/@Worker/Worker.m index 7b447366..2f93feec 100644 --- a/+opencossan/+workers/@Worker/Worker.m +++ b/+opencossan/+workers/@Worker/Worker.m @@ -30,31 +30,37 @@ properties OutputNames cell {opencossan.common.utilities.isunique(OutputNames)} - InputNames cell {opencossan.common.utilities.isunique(InputNames)} + InputNames cell {opencossan.common.utilities.isunique(InputNames)} KeepSimulationFiles(1,1) logical = false % Keep simulation files end methods (Abstract) % Evaluate the workerObject based on the realizations provided in a - % Table object - TableOutput=evaluate(workerObject,TableInput); - end + % Table object + TableOutput=evaluate(workerObject,TableInput); + end - methods - function obj = Worker(varargin) + methods + function obj = Worker(varargin) - [requiredArgs, varargin] = opencossan.common.utilities.parseRequiredNameValuePairs(... + if nargin == 0 + superArg = {}; + else + + [requiredArgs, varargin] = opencossan.common.utilities.parseRequiredNameValuePairs(... ["InputNames","OutputNames"], varargin{:}); - - [optionalArgs, superArg] = opencossan.common.utilities.parseOptionalNameValuePairs(... - "KeepSimulationFiles",{false}, varargin{:}); + [optionalArgs, superArg] = opencossan.common.utilities.parseOptionalNameValuePairs(... + "KeepSimulationFiles",{false}, varargin{:}); + end obj@opencossan.common.CossanObject(superArg{:}); - - obj.InputNames=requiredArgs.inputnames; - obj.OutputNames=requiredArgs.outputnames; - obj.KeepSimulationFiles=optionalArgs.keepsimulationfiles; - end + + if nargin > 0 + obj.InputNames=requiredArgs.inputnames; + obj.OutputNames=requiredArgs.outputnames; + obj.KeepSimulationFiles=optionalArgs.keepsimulationfiles; + end + end end end diff --git a/docs/tutorials/CantileverBeam/TutorialCantileverBeamMatlab.m b/docs/tutorials/CantileverBeam/TutorialCantileverBeamMatlab.m index 02102d7c..912293d7 100644 --- a/docs/tutorials/CantileverBeam/TutorialCantileverBeamMatlab.m +++ b/docs/tutorials/CantileverBeam/TutorialCantileverBeamMatlab.m @@ -65,11 +65,11 @@ %% Preparation of the Evaluator % Use of a matlab script to compute the Beam displacement currentFolder = fileparts(mfilename('fullpath')); -Xmio = opencossan.workers.Mio('FullFileName', fullfile(currentFolder, 'model', 'tipDisplacement.m'), ... +Xmio = opencossan.workers.MatlabWorker('FullFileName', fullfile(currentFolder, 'model', 'tipDisplacement.m'), ... 'InputNames', {'I', 'b', 'L', 'h', 'rho', 'P', 'E'}, ... 'OutputNames', {'w'}, 'Format', 'structure'); % Add the MIO object to an Evaluator object -Xevaluator = opencossan.workers.Evaluator('CXmembers', {Xmio}, 'CSmembers', {'Xmio'}); +Xevaluator = opencossan.workers.Evaluator('Solvers', Xmio, 'SolversName',"Xmio"); %% Preparation of the Physical Model % Define the Physical Model diff --git a/docs/tutorials/CantileverBeam/TutorialCantileverBeamMatlabOptimization.m b/docs/tutorials/CantileverBeam/TutorialCantileverBeamMatlabOptimization.m index e5356c6c..b4d791e0 100644 --- a/docs/tutorials/CantileverBeam/TutorialCantileverBeamMatlabOptimization.m +++ b/docs/tutorials/CantileverBeam/TutorialCantileverBeamMatlabOptimization.m @@ -2,10 +2,10 @@ % Perform optimization using Matlab evaluator % % -% See Also http://cossan.cfd.liv.ac.uk/wiki/index.php/Cantilever_Beam +% See Also http://cossan.co.uk/wiki/index.php/Cantilever_Beam % % -%

Copyright 2006-2014: COSSAN working group

+%

Copyright 2006-2020: COSSAN working group

% Author: Edoardo-Patelli
% Institute for Risk and Uncertainty, University of Liverpool, UK %
COSSAN web site: http://www.cossan.co.uk @@ -55,11 +55,11 @@ 'Cinputnames',{'I' 'b' 'L' 'h' 'rho' 'P','E'},'Sformat','structure', ... 'Coutputnames',{'w'}); % Add the MIO object to an Evaluator object -Xevaluator=Evaluator('CXmembers',{Xmio},'CSmembers',{'Xmio'}); +Xevaluator=Evaluator('Solvers',Xmio,'SolversName',"Xmio"); %% Preparation of the Physical Model % Define the Physical Model -Xmodel=Model('Xinput',XinputOptimization,'Xevaluator',Xevaluator); +Xmodel=Model('Input',XinputOptimization,'Evaluator',Xevaluator); %% Check feasibility of the optimization preoblem % The EesignOfExperiment analysis can be used to see if a feasible solution is diff --git a/docs/tutorials/CantileverBeam/TutorialCantileverBeamMatlabReliabilityAnalysis.m b/docs/tutorials/CantileverBeam/TutorialCantileverBeamMatlabReliabilityAnalysis.m index 982de2f9..ff9d7ce4 100644 --- a/docs/tutorials/CantileverBeam/TutorialCantileverBeamMatlabReliabilityAnalysis.m +++ b/docs/tutorials/CantileverBeam/TutorialCantileverBeamMatlabReliabilityAnalysis.m @@ -50,7 +50,7 @@ % Validate Solution assert(XfailureProbMC.pfhat == 0.06922,... 'CossanX:Tutorials:CantileverBeam',... - 'Reference Solution pf MCS not matched.'); + 'Computed solution does not match with the reference solution.'); %% Reliability Analysis via Latin Hypercube Sampling % Reset the random number generator to always produce the same results diff --git a/test/unit/+opencossan/+workers/EvaluatorTest.m b/test/unit/+opencossan/+workers/EvaluatorTest.m index e8f9937f..cf17299d 100644 --- a/test/unit/+opencossan/+workers/EvaluatorTest.m +++ b/test/unit/+opencossan/+workers/EvaluatorTest.m @@ -23,7 +23,29 @@ % ===================================================================== properties + MatWorker + MatWorker2 + ConWorker end + methods (TestClassSetup) + + + function defineWorker(testCase) + testCase.MatWorker = opencossan.workers.MatlabWorker( ... + 'Description','covariance function', ... + 'Format','structure',... + 'InputNames',{'TestInput'},... % Define the inputs + 'Script', "%Do Nothing",'OutputNames',{'TestOutput'}); % Define the outputs + + testCase.MatWorker2 = opencossan.workers.MatlabWorker('Script',"Toutput.out2=Tinput.out1+5;", ... + 'Format',"structure", ... + 'OutputNames',{'out3'},... + 'InputNames',{'X1' 'X2' 'X4' 'out1'}); + + testCase.ConWorker=opencossan.workers.Connector; + end + + end methods (Test) %% Constructor @@ -33,156 +55,123 @@ function emptyConstructor(testCase) end function constructorShouldSetDescription(testCase) - Xe = workers.Evaluator('Description','Evaluator'); - testCase.assertEqual(Xe.Description,'Evaluator'); + Xe = opencossan.workers.Evaluator('Description','Evaluator',... + 'Solver',testCase.MatWorker ); + testCase.assertEqual(Xe.Description,"Evaluator"); end function constructorShouldSetMatlabWorker(testCase) - Xmio = workers.MatlabWorker('Coutputnames',{'output'},... - 'Cinputnames',{'input'},'Sscript','%do nothing'); - Xe = workers.Evaluator('Solvers',Xmio,'SolversName',"Xmio"); - testCase.assertEqual(Xe.Solvers{1},Xmio); + Xe = opencossan.workers.Evaluator('Solvers',testCase.MatWorker,... + 'SolversName',{'Xmio'}); + testCase.assertEqual(Xe.Solvers(1),testCase.MatWorker); end function constructorShouldFail(testCase) - Xmio = workers.MatlabWorker('Coutputnames',{'output'},... - 'Cinputnames',{'input'},'Sscript','%do nothing'); - Xe = workers.Evaluator('Solvers',Xmio,'SolversName',["MatlabWorker" "ExtraName"]); - testCase.assertEqual(Xe.Solvers{1},XMatlabWorker); + Xe = opencossan.workers.Evaluator('Solvers',[testCase.MatWorker testCase.MatWorker2],... + 'SolversName',{'MatlabWorker' 'ExtraName'}); + testCase.assertEqual(Xe.Solvers(2),testCase.MatWorker2); end function constructorShouldSetJobManagerInterface(testCase) - Xjmi = highperformancecomputing.JobManagerInterface(); - Xe = workers.Evaluator('JobManager',Xjmi); - testCase.assertEqual(Xe.XjobInterface,Xjmi); + Xjmi = opencossan.highperformancecomputing.JobManagerInterface(); + Xe = opencossan.workers.Evaluator('Solvers',testCase.MatWorker,'JobManager',Xjmi); + testCase.assertEqual(Xe.JobManager,Xjmi); end function constructorShouldSetLremoteInjectExtract(testCase) - Xe = workers.Evaluator('RemoteInjectExtract',true); + Xe = opencossan.workers.Evaluator('Solvers',testCase.MatWorker,... + 'RemoteInjectExtract',true); testCase.assertTrue(Xe.RemoteInjectExtract); end function constructorShouldSetHostNames(testCase) - Xcon = workers.Connector; - Xmio = workers.MatlabWorker('Coutputnames',{'output'},... - 'Cinputnames',{'input'},'Sscript','%do nothing'); - Squeues = ["Queue1" "Queue2"]; - Shostnames = ["Host1","Host2"]; - Xe = workers.Evaluator('Solvers',{Xcon Xmio},'Hostnames',Shostnames,... + + Squeues = {'Queue1' 'Queue2'}; + Shostnames = {'Host1','Host2'}; + Xe = opencossan.workers.Evaluator('Solvers',[testCase.ConWorker testCase.MatWorker],... + 'Hostnames',Shostnames,... 'Queues',Squeues); - testCase.assertEqual(Xe.CShostnames,CShostnames); + testCase.assertEqual(Xe.Hostnames,Shostnames); end function constructorShouldSetQueues(testCase) - Xcon = workers.Connector; - Xmio = workers.Mio('Coutputnames',{'output'},... - 'Cinputnames',{'input'},'Sscript','%do nothing'); - Squeues = ["Queue1" "Queue2"]; - Xe = workers.Evaluator('Xconnector',Xcon,'Xmio',... - Xmio,'Queues',Squeues); + Squeues = {'Queue1' 'Queue2'}; + Xe = opencossan.workers.Evaluator('Solvers',... + [testCase.ConWorker testCase.MatWorker],'Queues',Squeues); testCase.assertEqual(Xe.Queues,Squeues); end function constructorShouldSetVconcurrent(testCase) - Xcon = workers.Connector; - Xmio = workers.Mio('Coutputnames',{'output'},... - 'Cinputnames',{'input'},'Sscript','%do nothing'); - Xe = workers.Evaluator('Solvers',{Xcon Xmio},... - 'Concurrent',[Inf 4]); - testCase.assertEqual(Xe.Concurrent,[Inf 4]); + Xe = opencossan.workers.Evaluator('Solvers',[testCase.ConWorker testCase.MatWorker],... + 'MaxCuncurrentJobs',[Inf 4]); + testCase.assertEqual(Xe.MaxCuncurrentJobs,[Inf 4]); end function constructorShouldSetMembers(testCase) - Xcon = workers.Connector; - Xmio = workers.Mio('Coutputnames',{'output'},... - 'Cinputnames',{'input'},'Sscript','%do nothing'); - Xe = workers.Evaluator('CXmembers',{Xcon Xmio}); - testCase.assertEqual(Xe.CXsolvers,{Xcon Xmio}); + Xe = opencossan.workers.Evaluator('Solvers',[testCase.ConWorker testCase.MatWorker]); + testCase.assertEqual(Xe.Solvers,[testCase.ConWorker testCase.MatWorker]); end function constructorShouldSetNames(testCase) - Xcon = workers.Connector; - Xmio = workers.Mio('Coutputnames',{'output'},... - 'Cinputnames',{'input'},'Sscript','%do nothing'); - CSnames = {'Connector Name', 'Mio name'}; - Xe = workers.Evaluator('Xconnector',Xcon,'Xmio',... - Xmio,'CSnames',CSnames); - testCase.assertEqual(Xe.CXsolvers,{Xcon Xmio}); - testCase.assertEqual(Xe.CSnames,CSnames); + CSnames = ["Connector Name", "Mio name"]; + Xe = opencossan.workers.Evaluator('Solvers',[testCase.ConWorker testCase.MatWorker],'SolversName',CSnames); + testCase.assertEqual(Xe.Solvers,[testCase.ConWorker testCase.MatWorker]); + testCase.assertEqual(Xe.SolversName,CSnames); end function constructorShouldSetSolutionSequence(testCase) - Xss = workers.SolutionSequence(); - Xe = workers.Evaluator('XsolutionSequence',Xss); - testCase.assertEqual(Xe.CXsolvers{1},Xss); + Xss = opencossan.workers.SolutionSequence(); + Xe = opencossan.workers.Evaluator('Solvers',Xss); + testCase.assertEqual(Xe.Solvers(1),Xss); end function constructorShouldSetMetaModel(testCase) - Xrs = metamodels.ResponseSurface(); - Xe = workers.Evaluator('XmetaModel',Xrs); - testCase.assertEqual(Xe.CXsolvers{1},Xrs); + Xrs = opencossan.metamodels.ResponseSurface(); + Xe = opencossan.workers.Evaluator('Solvers',Xrs); + testCase.assertEqual(Xe.Solvers(1),Xrs); end function constructorShouldSetParallelEnvironment(testCase) - Xcon = workers.Connector; - Xe = workers.Evaluator('Xconnector',Xcon,'CSparallelEnvironments',{'parallel'}); - testCase.assertEqual(Xe.CSparallelEnvironments,{'parallel'}); + Xe = opencossan.workers.Evaluator('Solvers',[testCase.ConWorker testCase.MatWorker],... + 'ParallelEnvironments',{'parallel'}); + testCase.assertEqual(Xe.ParallelEnvironments,{'parallel'}); end function constructorShouldSetVSlots(testCase) - Xcon = workers.Connector; - Xmio = workers.Mio('Coutputnames',{'output'},... - 'Cinputnames',{'input'},'Sscript','%do nothing'); - Xe = workers.Evaluator('Xconnector',Xcon,'Xmio',... - Xmio,'Vslots',[1 1]); - testCase.assertEqual(Xe.Vslots,[1 1]); + Xe = opencossan.workers.Evaluator('Solvers',[testCase.ConWorker testCase.MatWorker],... + 'Slots',[1 1]); + testCase.assertEqual(Xe.Slots,[1 1]); end function constructorTestInputNames(testCase) - Xmio1 = workers.Mio('Sscript','Toutput.out1=1;', ... - 'Sformat','structure', ... - 'Coutputnames',{'out1' 'out2'},... - 'Cinputnames',{'X1' 'X2' 'X3'}); - - Xmio2 = workers.Mio('Sscript','Toutput.out2=Tinput.out1+5;', ... - 'Sformat','structure', ... - 'Coutputnames',{'out3'},... - 'Cinputnames',{'X1' 'X2' 'X4' 'out1'}); - - Xe = workers.Evaluator('CXmembers',{Xmio1 Xmio2}); + + Xe = opencossan.workers.Evaluator('Solvers',[testCase.MatWorker testCase.MatWorker2]); - Cinput={'X1' 'X2' 'X3' 'X4'}; - testCase.assertEqual(Xe.Cinputnames,Cinput); + Cinput=[testCase.MatWorker.InputNames testCase.MatWorker2.InputNames]; + testCase.assertEqual(Xe.InputNames,Cinput); end function constructorTestOutputNames(testCase) - Xmio1 = workers.Mio('Sscript','Toutput.out1=1;', ... - 'Sformat','structure', ... - 'Coutputnames',{'out1' 'out2'},... - 'Cinputnames',{'X1' 'X2' 'X3'}); - - Xmio2 = workers.Mio('Sscript','Toutput.out2=Tinput.out1+5;', ... - 'Sformat','structure', ... - 'Coutputnames',{'out3'},... - 'Cinputnames',{'X1' 'X2' 'X4' 'out1'}); + - Xe = workers.Evaluator('CXmembers',{Xmio1 Xmio2}); + Xe = opencossan.workers.Evaluator('Solvers',[testCase.MatWorker testCase.MatWorker2]); - Coutput={'out1' 'out2' 'out3'}; - testCase.assertEqual(Xe.Coutputnames,Coutput); + Coutput=[testCase.MatWorker.OutputNames testCase.MatWorker2.OutputNames]; + testCase.assertEqual(Xe.OutputNames,Coutput); end %% Deterministic Analysis function deterministicAnalyis(testCase) - Xmio = workers.Mio('Sscript','Toutput.out1=Tinput.Xpar','CoutputNames',{'out1'},'CinputNames',{'Xpar'},'Sformat','structure'); - Xpar = common.inputs.Parameter('value',10.2); - Xinput = common.inputs.Input('Xparameter',Xpar); - Xe = workers.Evaluator('CXmembers',{Xmio},'CSnames',{'mio Name'}); + Xmio = opencossan.workers.MatlabWorker('Script',"Toutput.out1=Tinput.Xpar",... + 'OutputNames',{'out1'},'InputNames',{'Xpar'},'Format',"structure"); + Xpar = opencossan.common.inputs.Parameter('value',10.2); + Xinput = opencossan.common.inputs.Input('Parameter',Xpar); + Xe = opencossan.workers.Evaluator('Solvers',Xmio,'SolversName',"mio Name"); Xout=Xe.deterministicAnalysis(Xinput); - testCase.assertClass(Xout,'common.outputs.SimulationData'); + testCase.assertClass(Xout,'opencossan.common.outputs.SimulationData'); testCase.assertEqual(Xout.getValues('Sname','out1'),10.2); end From 2a6cfc7a517c9ed812f94f2ae2b7a5c9c7b740e8 Mon Sep 17 00:00:00 2001 From: Edoardo Patelli Date: Sun, 16 Feb 2020 21:43:13 +0000 Subject: [PATCH 4/9] All unit test and integration tutorials affected by the changes are working --- .../@CovarianceFunction/CovarianceFunction.m | 4 +- .../+stochasticprocess/StochasticProcess.m | 2 +- +opencossan/+common/@Model/Model.m | 4 +- .../@MetaModel/validateConstructor.m | 2 +- .../@ProbabilisticModel/ProbabilisticModel.m | 21 +-- +opencossan/+workers/@Evaluator/Evaluator.m | 60 +++---- +opencossan/+workers/@Evaluator/add.m | 109 ++++++------- +opencossan/+workers/@Evaluator/apply.m | 62 +++----- .../+workers/@Evaluator/executeWorkersGrid.m | 22 +-- .../@Evaluator/executeWorkersHorizontal.m | 16 +- .../+workers/@Evaluator/validateObject.m | 147 +++++++----------- .../TutorialCantileverBeamMatlab.m | 2 +- .../TutorialIntervalPredictorModel.m | 8 +- .../TutorialPolyharmonicSplines.m | 4 +- .../CossanObjects/TutorialResponseSurface.m | 11 +- .../+stochasticprocess/KarhunenLoeveTest.m | 2 +- .../unit/+opencossan/+workers/EvaluatorTest.m | 60 +++---- .../+opencossan/+workers/MatlabWorkerTest.m | 4 +- 18 files changed, 235 insertions(+), 305 deletions(-) diff --git a/+opencossan/+common/+inputs/+stochasticprocess/@CovarianceFunction/CovarianceFunction.m b/+opencossan/+common/+inputs/+stochasticprocess/@CovarianceFunction/CovarianceFunction.m index eb66b93a..0e5c9833 100644 --- a/+opencossan/+common/+inputs/+stochasticprocess/@CovarianceFunction/CovarianceFunction.m +++ b/+opencossan/+common/+inputs/+stochasticprocess/@CovarianceFunction/CovarianceFunction.m @@ -1,4 +1,4 @@ -classdef CovarianceFunction < opencossan.workers.Mio +classdef CovarianceFunction < opencossan.workers.MatlabWorker %COVARIANCEFUNCTION This class defines the covariance function for a %stochastic process. % @@ -22,7 +22,7 @@ % Call constructor of opencossan.workers.Mio - Xobj = Xobj@opencossan.workers.Mio(varargin{:}); + Xobj = Xobj@opencossan.workers.MatlabWorker(varargin{:}); if nargin==0 return; diff --git a/+opencossan/+common/+inputs/+stochasticprocess/StochasticProcess.m b/+opencossan/+common/+inputs/+stochasticprocess/StochasticProcess.m index 9a294427..ff3123b4 100644 --- a/+opencossan/+common/+inputs/+stochasticprocess/StochasticProcess.m +++ b/+opencossan/+common/+inputs/+stochasticprocess/StochasticProcess.m @@ -1,4 +1,4 @@ -classdef (Abstract) StochasticProcess < opencossan.workers.Mio +classdef (Abstract) StochasticProcess < opencossan.workers.MatlabWorker %STOCHASTICPROCESS The abstract class defines a random process (or a %random field) to represent the evolution of some random value, or %system, over time or space diff --git a/+opencossan/+common/@Model/Model.m b/+opencossan/+common/@Model/Model.m index 3d70e52d..01b700a2 100644 --- a/+opencossan/+common/@Model/Model.m +++ b/+opencossan/+common/@Model/Model.m @@ -62,11 +62,11 @@ obj = setGridProperties(obj, varargin); % Add execution details (i.e. Grid configuration) function names = get.OutputNames(obj) - names = obj.Evaluator.Coutputnames; + names = obj.Evaluator.OutputNames; end function names = get.InputNames(obj) - names = obj.Input.Names; + names = obj.Evaluator.InputNames; end end end diff --git a/+opencossan/+metamodels/@MetaModel/validateConstructor.m b/+opencossan/+metamodels/@MetaModel/validateConstructor.m index 8c2631fd..16681345 100644 --- a/+opencossan/+metamodels/@MetaModel/validateConstructor.m +++ b/+opencossan/+metamodels/@MetaModel/validateConstructor.m @@ -70,7 +70,7 @@ assert(all(ismember(Xobj.InputNames,Callinputnames)),... 'openCOSSAN:ResponseSurface',... strcat('Not all the required inputs factors are present in the ',... - 'Model or in the calibration point\n Model inputs: %s\n Required inputs: \s'),... + ' Model or in the calibration point\n Model inputs: %s\n Required inputs: %s'),... sprintf('"%s" ',Callinputnames{:}),sprintf('"%s" ',Xobj.InputNames{:})); end diff --git a/+opencossan/+reliability/@ProbabilisticModel/ProbabilisticModel.m b/+opencossan/+reliability/@ProbabilisticModel/ProbabilisticModel.m index ca6814af..0025af1f 100644 --- a/+opencossan/+reliability/@ProbabilisticModel/ProbabilisticModel.m +++ b/+opencossan/+reliability/@ProbabilisticModel/ProbabilisticModel.m @@ -41,6 +41,7 @@ [required, super_args] = ... opencossan.common.utilities.parseRequiredNameValuePairs(... ["model", "performancefunction"], varargin{:}); + % TODO: Why Input arguments???? super_args = [super_args; {'input', required.model.Input, ... 'evaluator', required.model.Evaluator}]; end @@ -49,21 +50,21 @@ if nargin > 0 % Add PerformanceFunction to Evaluator - if isempty(obj.Evaluator.CXsolvers) - obj.Evaluator = obj.Evaluator.add('Xmember',required.performancefunction); + if isempty(obj.Evaluator.Solver) + obj.Evaluator = obj.Evaluator.add('Solver',required.performancefunction); else % TODO: What does this do? - obj.Evaluator = obj.Evaluator.add('Xmember',required.performancefunction,'Sname','N/A','Nslots',Inf,... - 'Nconcurrent',Inf,'Shostname','localhost','Squeue','','SparallelEnvironment',''); + obj.Evaluator = obj.Evaluator.add('Solver',required.performancefunction,'SolverName','N/A','Slots',Inf,... + 'MaxCuncurrentJobs',Inf,'Hostname','localhost','Queue','','ParallelEnvironment',''); end end end function variable = get.PerformanceFunctionVariable(obj) % Return the name of the output of the performance function - for i = 1:numel(obj.Evaluator.CXsolvers) - if isa(obj.Evaluator.CXsolvers{i},'opencossan.reliability.PerformanceFunction') - variable = obj.Evaluator.CXsolvers{i}.OutputNames{:}; + for i = 1:numel(obj.Evaluator.Solver) + if isa(obj.Evaluator.Solver(i),'opencossan.reliability.PerformanceFunction') + variable = obj.Evaluator.Solver(i).OutputNames{:}; return; end end @@ -72,9 +73,9 @@ function indicatorFunction = get.StdDeviationIndicatorFunction(obj) % Return the standard deviation indicator function of the % performance function - for i = 1:numel(obj.Evaluator.CXsolvers) - if isa(obj.Evaluator.CXsolvers{i},'opencossan.reliability.PerformanceFunction') - indicatorFunction = obj.Evaluator.CXsolvers{i}.StdDeviationIndicatorFunction; + for i = 1:numel(obj.Evaluator.Solver) + if isa(obj.Evaluator.Solver(i),'opencossan.reliability.PerformanceFunction') + indicatorFunction = obj.Evaluator.Solver(i).StdDeviationIndicatorFunction; return; end end diff --git a/+opencossan/+workers/@Evaluator/Evaluator.m b/+opencossan/+workers/@Evaluator/Evaluator.m index 552d6de5..1047326d 100644 --- a/+opencossan/+workers/@Evaluator/Evaluator.m +++ b/+opencossan/+workers/@Evaluator/Evaluator.m @@ -25,18 +25,18 @@ properties JobManager opencossan.highperformancecomputing.JobManagerInterface % Define JobManager to submit job to grid/cluster computer - Solvers(1,:) %List of OpenCossan Workers opencossan.workers.Worker - SolversName(1,:) string % Names of the workers (optional) - Queues(1,:) cell % Where to submit solvers - Hostnames(1,:) cell % Names of hostnames where to evaluate workers - ParallelEnvironments(:,1) cell % Name of the parallel environment of each solver - Slots(1,:) double {mustBeInteger} % Number of slots used in each job + Solver(1,:) %List of OpenCossan Workers opencossan.workers.Worker + SolverName(1,:) string % Names of the workers (optional) + Queues(1,:) string % Where to submit solvers + Hostnames(1,:) string % Names of hostnames where to evaluate workers + ParallelEnvironments(:,1) string % Name of the parallel environment of each solver + Slots(1,:) double {mustBePositive} % Number of slots used in each job IsCompiled(1,:) logical % Number of slots used in each job MaxCuncurrentJobs(1,:) double {mustBePositive} = 1 % Number of concurrent execution of each solver RemoteInjectExtract = false %TODO: make it true by default VerticalSplit = false % if true split the analysis in vertical components (see wiki for more details) - MaxNumberofJobs double {mustBeInteger,mustBePositive} = 1 % max number of jobs submitted for each analysis - WrapperMatlabInputName(1,1) string % Name of the input Matlab file loaded by the job + MaxNumberofJobs double {mustBePositive} = 1 % max number of jobs submitted for each analysis + WrapperMatlabInputName(1,1) string % Name of the input Matlab file loaded by the job WrapperMatlabOutputName(1,1) string % Name of the output Matlab file create by the job end @@ -61,7 +61,7 @@ superArg={}; else [requiredArgs, varargin] = opencossan.common.utilities.parseRequiredNameValuePairs(... - "Solvers", varargin{:}); + "Solver", varargin{:}); % Define optional arguments and default values @@ -78,7 +78,7 @@ "MaxNumberofJobs",[];... "WrapperMatlabInputName","";... "WrapperMatlabOutputName","";... - "SolversName",[]}; + "SolverName",[]}; [optionalArg, superArg] = opencossan.common.utilities.parseOptionalNameValuePairs(... [OptionalsArguments{:,1}],{OptionalsArguments{:,2}}, varargin{:}); @@ -90,15 +90,15 @@ if nargin>0 - obj.Solvers = requiredArgs.solvers; + obj.Solver = requiredArgs.solver; - obj.SolversName = optionalArg.solversname; + obj.SolverName = optionalArg.solvername; - if ~isempty(obj.SolversName) - assert(numel(obj.Solvers) == numel(obj.SolversName),... + if ~isempty(obj.SolverName) + assert(numel(obj.Solver) == numel(obj.SolverName),... 'Evaluator:IllegalArguments',... - 'Number of solvers (%i) specified must be the same size of SolversName (%i)',... - numel(obj.Solvers),numel(obj.SolversName)); + 'Number of solvers (%i) specified must be the same size of SolverName (%i)',... + numel(obj.Solver),numel(obj.SolverName)); end obj.JobManager = optionalArg.jobmanager; @@ -113,7 +113,11 @@ obj.MaxNumberofJobs = optionalArg.maxnumberofjobs; obj.WrapperMatlabInputName = optionalArg.wrappermatlabinputname; obj.WrapperMatlabOutputName = optionalArg.wrappermatlaboutputname; + + obj=validateObject(obj); end + + end %end constructor Xout=apply(Xobj,Pinput) % Run the analysis @@ -124,15 +128,15 @@ function OutputNames=get.OutputNames(Xobj) % Extract output names from the target object - if isempty(Xobj.Solvers) + if isempty(Xobj.Solver) OutputNames={}; else OutputNames={}; - for n=1:length(Xobj.Solvers) - if isrow(Xobj.Solvers(n).OutputNames) - Caddoutput=Xobj.Solvers(n).OutputNames; + for n=1:length(Xobj.Solver) + if isrow(Xobj.Solver(n).OutputNames) + Caddoutput=Xobj.Solver(n).OutputNames; else - Caddoutput=transpose(Xobj.Solvers(n).OutputNames); + Caddoutput=transpose(Xobj.Solver(n).OutputNames); end OutputNames=[OutputNames Caddoutput]; %#ok end @@ -141,13 +145,13 @@ function InputNames=get.InputNames(Xobj) % Extract output names from the target object - if isempty(Xobj.Solvers) + if isempty(Xobj.Solver) InputNames={}; else - InputNames=Xobj.Solvers(1).InputNames; + InputNames=Xobj.Solver(1).InputNames; CoutEvaluator={}; - for n=2:length(Xobj.Solvers) - CaddInputs=Xobj.Solvers(n).InputNames; % tmp variable + for n=2:length(Xobj.Solver) + CaddInputs=Xobj.Solver(n).InputNames; % tmp variable % Remove already present inputs Vindex=false(length(CaddInputs),1); for j=1:length(CaddInputs) @@ -155,10 +159,10 @@ end CaddInputs(Vindex)=[]; - if isrow(Xobj.Solvers(n-1).OutputNames) - Caddoutput=Xobj.Solvers(n-1).OutputNames; + if isrow(Xobj.Solver(n-1).OutputNames) + Caddoutput=Xobj.Solver(n-1).OutputNames; else - Caddoutput=transpose(Xobj.Solvers(n-1).OutputNames); + Caddoutput=transpose(Xobj.Solver(n-1).OutputNames); end CoutEvaluator=[CoutEvaluator Caddoutput]; %#ok diff --git a/+opencossan/+workers/@Evaluator/add.m b/+opencossan/+workers/@Evaluator/add.m index 6e48cc95..30c460f8 100644 --- a/+opencossan/+workers/@Evaluator/add.m +++ b/+opencossan/+workers/@Evaluator/add.m @@ -1,73 +1,58 @@ -function Xev=add(Xev,varargin) +function obj=add(obj,varargin) %ADD This method adds a worker object to the current Evaluator % -% See also: https://cossan.co.uk/wiki/index.php/Add@Evaluator +% See also: Evaluator, Worker % % Author: Edoardo Patelli -% Institute for Risk and Uncertainty, University of Liverpool, UK % email address: openengine@cossan.co.uk % Website: http://www.cossan.co.uk -% ===================================================================== -% This file is part of openCOSSAN. The open general purpose matlab -% toolbox for numerical analysis, risk and uncertainty quantification. -% -% openCOSSAN is free software: you can redistribute it and/or modify -% it under the terms of the GNU General Public License as published by -% the Free Software Foundation, either version 3 of the License. -% -% openCOSSAN is distributed in the hope that it will be useful, -% but WITHOUT ANY WARRANTY; without even the implied warranty of -% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -% GNU General Public License for more details. -% -% You should have received a copy of the GNU General Public License -% along with openCOSSAN. If not, see . -% ===================================================================== -import opencossan.* + %{ +This file is part of OpenCossan . +Copyright (C) 2006-2020 COSSAN WORKING GROUP + +OpenCossan is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License or, +(at your option) any later version. + +OpenCossan is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with OpenCossan. If not, see . + %} +[requiredArgs, varargin] = opencossan.common.utilities.parseRequiredNameValuePairs(... + "Solver", varargin{:}); + +% Define optional arguments and default values + +OptionalsArguments={... + "Queues","";... + "Hostnames","";... + "ParallelEnvironments","";... + "Slots",[];... + "MaxCuncurrentJobs",[];... + "RemoteInjectExtract", false;... + "MaxNumberofJobs",[];... + "SolverName",""}; + +[optionalArg, ~] = opencossan.common.utilities.parseOptionalNameValuePairs(... + [OptionalsArguments{:,1}],{OptionalsArguments{:,2}}, varargin{:}); + +obj.Solver = [obj.Solver requiredArgs.solver]; -%% Process input argments -for k=1:2:length(varargin) - switch lower(varargin{k}) - case 'solver' - assert(isa(varargin{k+1},'opencossan.workers.SolutionSequence'), ... - 'openCOSSAN:Evaluator', ... - 'The object of class %s is not valid after the PropertyName %s', ... - class(varargin{k+1}),varargin{k}) - - Xev.Solvers(end+1)=varargin{k+1}; +obj.SolverName = [obj.SolverName optionalArg.solvername]; - case {'shostname','cshostnames'} - Xev.CShostnames{end+1}=varargin{k+1}; - case {'sparallelenvironment','spe','csparallelenviroments'} - Xev.CSparallelEnvironments{end+1}=varargin{k+1}; - case {'squeue','csqueues'} - Xev.CSqueues{end+1}=varargin{k+1}; - case {'sname','smember'} - Xev.CSnames{end+1}=varargin{k+1}; - case {'nconcurrent'} - Xev.Vconcurrent(end+1)=varargin{k+1}; - case {'nslots'} - Xev.Vslots(end+1)=varargin{k+1}; - case {'xmember'} - % The object are retrieved from the cell array - if any(ismember(superclasses(varargin{k+1}),'opencossan.workers.Worker')) - Xev.CXsolvers{end+1}=varargin{k+1}; - - elseif any(ismember(superclasses(varargin{k+1}),'opencossan.metamodels.Metamodels')) - Xev.CXsolvers{end+1}=varargin{k+1}; - elseif isa(varargin{k+1},'opencossan.highperformancecomputing.JobManagerInterface') - Xev.XjobInterface=varargin{k+1}; - else - error('openCOSSAN:Evaluator:add',... - 'The object of class %s is not allowed',class(varargin{k+1})); - end - - otherwise - error('openCOSSAN:Evaluator',... - 'PropertyName %s is not a valid PropertyName',varargin{k}); - end -end +obj.Queues = [obj.Queues optionalArg.queues]; +obj.Hostnames = [obj.Hostnames optionalArg.hostnames]; +obj.ParallelEnvironments = [obj.ParallelEnvironments optionalArg.parallelenvironments]; +obj.Slots = [obj.Slots optionalArg.slots]; +obj.MaxCuncurrentJobs = [obj.MaxCuncurrentJobs optionalArg.maxcuncurrentjobs]; +obj.MaxNumberofJobs = optionalArg.maxnumberofjobs; % Validate object -Xev=validateObject(Xev); +obj=validateObject(obj); +end \ No newline at end of file diff --git a/+opencossan/+workers/@Evaluator/apply.m b/+opencossan/+workers/@Evaluator/apply.m index 079bb87c..51f921b1 100644 --- a/+opencossan/+workers/@Evaluator/apply.m +++ b/+opencossan/+workers/@Evaluator/apply.m @@ -9,30 +9,30 @@ % % Usage: XSimout = Xev.apply(Pinput) % -% See Also: http://cossan.co.uk/wiki/index.php/Apply@Evaluator +% See Also: Evaluator, Worker % % % Author: Edoardo Patelli -% Institute for Risk and Uncertainty, University of Liverpool, UK % email address: openengine@cossan.co.uk % Website: http://www.cossan.co.uk -% ===================================================================== -% This file is part of openCOSSAN. The open general purpose matlab -% toolbox for numerical analysis, risk and uncertainty quantification. -% -% openCOSSAN is free software: you can redistribute it and/or modify -% it under the terms of the GNU General Public License as published by -% the Free Software Foundation, either version 3 of the License. -% -% openCOSSAN is distributed in the hope that it will be useful, -% but WITHOUT ANY WARRANTY; without even the implied warranty of -% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -% GNU General Public License for more details. -% -% You should have received a copy of the GNU General Public License -% along with openCOSSAN. If not, see . -% ===================================================================== + %{ +This file is part of OpenCossan . +Copyright (C) 2006-2020 COSSAN WORKING GROUP + +OpenCossan is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License or, +(at your option) any later version. + +OpenCossan is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with OpenCossan. If not, see . + %} import opencossan.common.outputs.SimulationData import opencossan.highperformancecomputing.* @@ -61,7 +61,7 @@ % Add input variables to the SimulationData object XSimInp=SimulationData('Table',TableInput); -if isempty(Xobj.Solvers) +if isempty(Xobj.Solver) XSimData=XSimInp; % Add input samples to the Simulation output object XSimData.Sdescription= 'created by the Evaluator with no workers'; @@ -70,34 +70,18 @@ if Xobj.VerticalSplit % Setting the JobManager - if ~isempty(Xobj.Queues) - % Setting the JobManager - Xjob=JobManager('XjobManagerInterface',Xobj.JobInterface, ... - 'Squeue',Xobj.Queues(1),'Shostname',Xobj.Hostnames(1), ... - 'SparallelEnvironment',Xobj.ParallelEnvironments{1},... - 'Nslots',Xobj.Slots(1),... - 'Nconcurrent',Xobj.MaxCuncurrentJobs(1),... - 'Sdescription','JobManager created by Evaluator.executeWorkers'); - + if ~isempty(Xobj.JobManager) % TODO: Not implemented - TableOutput=executeWorkersGrid(Xobj,XSimInp,Xjob); + TableOutput=executeWorkersGrid(Xobj,XSimInp); else TableOutput=executeWorkersVertical(Xobj,TableInput); end else % Setting the JobManager - if ~isempty(Xobj.Queues) - % Setting the JobManager - Xjob=JobManager('XjobManagerInterface',Xobj.XjobInterface, ... - 'Squeue',Xobj.Queues(1),'Shostname',Xobj.Hostnames{1}, ... - 'SparallelEnvironment',Xobj.ParallelEnvironments{1},... - 'Nslots',Xobj.Slots(1),... - 'Nconcurrent',Xobj.MaxCuncurrentJobs(1),... - 'Sdescription','JobManager created by Evaluator.executeWorkers'); - + if ~isempty(Xobj.JobManager) % TODO: Not implemented - TableOutput=executeWorkersGrid(Xobj,XSimInp,Xjob); + TableOutput=executeWorkersGrid(Xobj,XSimInp); else TableOutput=executeWorkersHorizontal(Xobj,TableInput); end diff --git a/+opencossan/+workers/@Evaluator/executeWorkersGrid.m b/+opencossan/+workers/@Evaluator/executeWorkersGrid.m index 02198ba9..d1e78238 100644 --- a/+opencossan/+workers/@Evaluator/executeWorkersGrid.m +++ b/+opencossan/+workers/@Evaluator/executeWorkersGrid.m @@ -1,4 +1,4 @@ -function XSimOut = executeWorkersGrid(Xobj,XSimInp,Xjob) +function SimOut = executeWorkersGrid(Xobj,PinputALL) % EXECUTEWORKERSGRID This is a protected method of evaluator to run the % analysis in vertical chunks using the Job Manager. % @@ -6,7 +6,7 @@ % % Usage: XSimout = executeWorkersGrid(Xobj,XSimInp,Xjob) % -% See Also: http://cossan.co.uk/wiki/index.php/executeWorkers@Evaluator +% See Also: Evaluator, JobManager % % % Author: Edoardo Patelli @@ -35,14 +35,14 @@ % split input samples between jobs -if (Xjob.Nconcurrent == Inf) % checks whether or not Njobs has been defined +if (Xobj.MaxCuncurrentJobs == Inf) % checks whether or not Njobs has been defined Njobs = size(PinputALL,1); % sets number of jobs equal to number of simulations to be performed Vsimxjobs = ones(size(PinputALL))'; % obviously, there is one simulation per job else - Njobs = min(Xjob.Nconcurrent,size(PinputALL,1)); % re-adjusts number of jobs (if required) - Vsimxjobs = floor(size(PinputALL,1)/Xjob.Nconcurrent)*ones(1,Xjob.Nconcurrent); - Vsimxjobs(1:rem(size(PinputALL,1),Xjob.Nconcurrent)) = ... - Vsimxjobs(1:rem(size(PinputALL,1),Xjob.Nconcurrent)) + 1; % sets number of simulations per job + Njobs = min(Xobj.MaxCuncurrentJobs,size(PinputALL,1)); % re-adjusts number of jobs (if required) + Vsimxjobs = floor(size(PinputALL,1)/Xobj.Nconcurrent)*ones(1,Xobj.MaxCuncurrentJobs); + Vsimxjobs(1:rem(size(PinputALL,1),Xobj.MaxCuncurrentJobs)) = ... + Vsimxjobs(1:rem(size(PinputALL,1),Xobj.MaxCuncurrentJobs)) + 1; % sets number of simulations per job % if there are less samples than concurrent jobs remove jobs with no samples Vsimxjobs(Vsimxjobs==0) = []; end @@ -58,11 +58,11 @@ % If required, displays information on which job is being processed OpenCossan.cossanDisp(['Preparing input file for Worker job #' num2str(irun) ' of ' num2str(Njobs) ],2); % Creates folder where the job will be executed - Sfoldername = [Xjob.Sfoldername '_job_' num2str(irun)]; % defines name of folder where the job will be executed + Sfoldername = [Xobj.JobManager.Sfoldername '_job_' num2str(irun)]; % defines name of folder where the job will be executed mkdir(fullfile(OpenCossan.getCossanWorkingPath,Sfoldername)); % creates the folder % set the execution command. This is a method/property depending on % whether you run worker compiled or not - Xjob.Sexecmd = ['cd ' fullfile(OpenCossan.getCossanWorkingPath,Sfoldername) '; '... + Xobj.JobManager.Sexecmd = ['cd ' fullfile(OpenCossan.getCossanWorkingPath,Sfoldername) '; '... strrep(fullfile(OpenCossan.getMatlabPath,'bin','matlab'),' ','\\ ') ... ' -r workers.remoteWorkerJob -nosplash -nodesktop']; % Copy input table and worker into the new grid folder @@ -181,9 +181,9 @@ end if exist('XSimOut','var') - XSimOut=XSimOut.merge(Xsimtmp); + SimOut=SimOut.merge(Xsimtmp); else - XSimOut=Xsimtmp; + SimOut=Xsimtmp; end diff --git a/+opencossan/+workers/@Evaluator/executeWorkersHorizontal.m b/+opencossan/+workers/@Evaluator/executeWorkersHorizontal.m index 36f32eeb..55e9177f 100644 --- a/+opencossan/+workers/@Evaluator/executeWorkersHorizontal.m +++ b/+opencossan/+workers/@Evaluator/executeWorkersHorizontal.m @@ -48,9 +48,9 @@ TableOutputSolver=[]; % Evaluator execution -for n=1:length(Xobj.Solvers) +for n=1:length(Xobj.Solver) OpenCossan.cossanDisp(['[Status:workers ] * Processing solver ' ... - num2str(n) '/' num2str(length(Xobj.Solvers))],3) + num2str(n) '/' num2str(length(Xobj.Solver))],3) % Merge tableInput with output produced by the workers and then % pass only the inputs required by the specific worker. @@ -64,22 +64,22 @@ % Execute worker. % Only the input required by the worker are passes. No recheck % is needed in the evaluate method. - TableOutputSolverTmp=Xobj.Solvers(n).evaluate(TableSolver(:,Xobj.Solvers(n).InputNames)); - %TableOutputSolverTmp=array2table(Xobj.CXsolvers{n}.evaluate(TableSolver(:,Xobj.CXsolvers{n}.Cinputnames)),'VariableNames',Xobj.Coutputnames); + TableOutputSolverTmp=Xobj.Solver(n).evaluate(TableSolver(:,Xobj.Solver(n).InputNames)); + %TableOutputSolverTmp=array2table(Xobj.CXSolver{n}.evaluate(TableSolver(:,Xobj.CXSolver{n}.Cinputnames)),'VariableNames',Xobj.Coutputnames); catch Exception warning('OpenCossan:Evaluator:executeWorkersHorizontal:workerFaild',... - 'Unable to execute worker %i of type %s',n,class(Xobj.Solvers(n))) + 'Unable to execute worker %i of type %s',n,class(Xobj.Solver(n))) % Add meaningful error message msgID = 'OpenCossan:Evaluator:executeWorkersHorizontal:workerFaild'; - msg = sprintf('Unable to execute worker %i of type %s',n,class(Xobj.Solvers(n))); + msg = sprintf('Unable to execute worker %i of type %s',n,class(Xobj.Solver(n))); causeException = MException(msgID,msg); % Store Exception in the Analysis object Xanalysis.ErrorsStack{end+1} = addCause(Exception,causeException); % Construct a tableOutputSolverTmp with NaN - TableOutputSolverTmp=array2table(NaN(height(TableSolver),length(Xobj.Solvers(n).OutputNames)),... - 'VariableNames',Xobj.Solvers{n}.OutputNames); + TableOutputSolverTmp=array2table(NaN(height(TableSolver),length(Xobj.Solver(n).OutputNames)),... + 'VariableNames',Xobj.Solver{n}.OutputNames); end % Merge workers output for the currenct analysis TableOutputSolver=[TableOutputSolver, TableOutputSolverTmp]; %#ok diff --git a/+opencossan/+workers/@Evaluator/validateObject.m b/+opencossan/+workers/@Evaluator/validateObject.m index 67bfe760..32b630c7 100644 --- a/+opencossan/+workers/@Evaluator/validateObject.m +++ b/+opencossan/+workers/@Evaluator/validateObject.m @@ -1,132 +1,89 @@ -function Xobj=validateObject(Xobj) +function obj=validateObject(obj) %VALIDATEOBJCECT This is a protected function of Evaluator used to validate %the inputs. -% See also: https://cossan.co.uk/wiki/index.php/@Evaluator +% See also: Evaluator, Worker % % Author: Edoardo Patelli -% Institute for Risk and Uncertainty, University of Liverpool, UK % email address: openengine@cossan.co.uk % Website: http://www.cossan.co.uk -% ===================================================================== -% This file is part of openCOSSAN. The open general purpose matlab -% toolbox for numerical analysis, risk and uncertainty quantification. -% -% openCOSSAN is free software: you can redistribute it and/or modify -% it under the terms of the GNU General Public License as published by -% the Free Software Foundation, either version 3 of the License. -% -% openCOSSAN is distributed in the hope that it will be useful, -% but WITHOUT ANY WARRANTY; without even the implied warranty of -% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -% GNU General Public License for more details. -% -% You should have received a copy of the GNU General Public License -% along with openCOSSAN. If not, see . -% ===================================================================== + %{ +This file is part of OpenCossan . +Copyright (C) 2006-2020 COSSAN WORKING GROUP + +OpenCossan is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License or, +(at your option) any later version. + +OpenCossan is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with OpenCossan. If not, see . + %} % Validate objects -if Xobj.LverticalSplit +if obj.VerticalSplit Nworkers=1; else - Nworkers=length(Xobj.CXsolvers); + Nworkers=length(obj.Solver); end -if ~isempty(Xobj.CSnames) - assert(length(Xobj.CSnames)==Nworkers,... - 'openCOSSAN:Evaluator',... - ['Length of CSnames (' num2str(length(Xobj.CSnames)) ... - ') must be equal to the length of CXsolvers (' ... +if ~isempty(obj.SolverName) + assert(length(obj.SolverName)==Nworkers,... + 'Evaluator:SolverName:wrongSize',... + ['Length of SolverNames (' num2str(length(obj.SolverName)) ... + ') must be equal to the length of Solver (' ... num2str(Nworkers) ')' ]) else - Xobj.CSnames=repmat({'N/A'},Nworkers,1); + obj.SolverName=repmat("N/A",Nworkers,1); end -if ~isempty(Xobj.Vconcurrent) - assert(length(Xobj.Vconcurrent)==Nworkers,... - 'openCOSSAN:Evaluator',... - ['Length of Vconcurrent (' num2str(length(Xobj.Vconcurrent)) ... - ') must be equal to the length of CXsolvers (' ... +if ~isempty(obj.MaxCuncurrentJobs) + assert(length(obj.MaxCuncurrentJobs)==Nworkers,... + 'Evaluator:MaxCuncurrentJob:wrongSize',... + ['Length of MaxCuncurrentJobs (' num2str(length(obj.MaxCuncurrentJobs)) ... + ') must be equal to the length of Solver (' ... num2str(Nworkers) ')' ]) else - Xobj.Vconcurrent=inf(size(Xobj.CXsolvers)); + obj.MaxCuncurrentJobs=inf(size(obj.Solver)); end -if ~isempty(Xobj.CSqueues) - assert(length(Xobj.CSqueues)==Nworkers,... - 'openCOSSAN:Evaluator',... - ['Length of CSqueues (' num2str(length(Xobj.CSqueues)) ... - ') must be equal to the length of CXsolvers (' ... - num2str(Nworkers) ')' ]) -else - for n=1:Nworkers - Xobj.CSqueues{n}=''; - Xobj.CShostnames{n}=''; - end +if ~isempty(obj.Queues) + assert(length(obj.Queues)==Nworkers,... + 'Evaluator:Queues:wrongSize',... + 'Length of Queues (%i) must be equal to the length of Solver (%i)',... + length(obj.Queues),Nworkers) end -if ~isempty(Xobj.CShostnames) - assert(length(Xobj.CShostnames)==Nworkers,... - 'openCOSSAN:Evaluator',... - ['Length of CShostnames (' num2str(length(Xobj.CShostnames)) ... - ') must be equal to the length of CXsolvers (' ... - num2str(Nworkers) ')' ]) -else - for n=1:Nworkers - Xobj.CShostnames{n}=''; - end +if ~isempty(obj.Hostnames) + assert(length(obj.Hostnames)==Nworkers,... + 'Evaluator:Hostnames:wrongSize',... + 'Length of Hostnames (%i) must be equal to the length of Solver (%i)',... + length(obj.Hostnames),Nworkers) end -if ~isempty(Xobj.CSparallelEnvironments) - assert(length(Xobj.CSparallelEnvironments)==Nworkers,... +if ~isempty(obj.ParallelEnvironments) + assert(length(obj.ParallelEnvironments)==Nworkers,... 'openCOSSAN:Evaluator',... - ['Length of CSparallelEnvironments (' num2str(length(Xobj.CSparallelEnvironments)) ... - ') must be equal to the length of CXsolvers (' ... + ['Length of ParallelEnvironments (' num2str(length(obj.ParallelEnvironments)) ... + ') must be equal to the length of Solver (' ... num2str(Nworkers) ')' ]) -else - for n=1:Nworkers - Xobj.CSparallelEnvironments{n}=''; - end end -if ~isempty(Xobj.Vslots) - assert(length(Xobj.Vslots)==Nworkers,... +if ~isempty(obj.Slots) + assert(length(obj.Slots)==Nworkers,... 'openCOSSAN:Evaluator',... - ['Length of Vslots (' num2str(length(Xobj.Vslots)) ... - ') must be equal to the length of CXsolvers (' ... + ['Length of Slots (' num2str(length(obj.Slots)) ... + ') must be equal to the length of Solver (' ... num2str(Nworkers) ')' ]) else - Xobj.Vslots=ones(size(Xobj.CXsolvers)); + obj.Slots=ones(size(obj.Solver)); end -% Check for unique SwrapperName in the Mio -for n=1:Nworkers - if isa(Xobj.CXsolvers{n},'Mio') - Ncount=1; - for i=1:n-1 - if isa(Xobj.CXsolvers{i},'Mio') - if(strcmp(Xobj.CXsolvers{i}.SwrapperName,Xobj.CXsolvers{n}.SwrapperName)) - Ncount = Ncount+1; - if Ncount==2 - Xobj.CXsolvers{n}.SwrapperName = [Xobj.CXsolvers{n}.SwrapperName num2str(Ncount)]; - elseif Ncount<=10 - Xobj.CXsolvers{n}.SwrapperName = [Xobj.CXsolvers{n}.SwrapperName(1:end-1) num2str(Ncount)]; - else - Xobj.CXsolvers{n}.SwrapperName = [Xobj.CXsolvers{n}.SwrapperName(1:end-2) num2str(Ncount)]; - end - end - end - end - end -end - -%% Check for duplicated Outputs -if ~isempty(Xobj.Coutputnames) - Couts=unique(Xobj.Coutputnames); - assert(length(Couts)==length(Xobj.Coutputnames),... - 'openCOSSAN:Evaluator:validateObject',... - strcat('Duplicated outputs present in the Evaluator \n',sprintf('%s ',Xobj.Coutputnames{:}))) -end diff --git a/docs/tutorials/CantileverBeam/TutorialCantileverBeamMatlab.m b/docs/tutorials/CantileverBeam/TutorialCantileverBeamMatlab.m index 912293d7..68085d6a 100644 --- a/docs/tutorials/CantileverBeam/TutorialCantileverBeamMatlab.m +++ b/docs/tutorials/CantileverBeam/TutorialCantileverBeamMatlab.m @@ -69,7 +69,7 @@ 'InputNames', {'I', 'b', 'L', 'h', 'rho', 'P', 'E'}, ... 'OutputNames', {'w'}, 'Format', 'structure'); % Add the MIO object to an Evaluator object -Xevaluator = opencossan.workers.Evaluator('Solvers', Xmio, 'SolversName',"Xmio"); +Xevaluator = opencossan.workers.Evaluator('Solver', Xmio, 'SolverName',"Xmio"); %% Preparation of the Physical Model % Define the Physical Model diff --git a/docs/tutorials/CossanObjects/TutorialIntervalPredictorModel.m b/docs/tutorials/CossanObjects/TutorialIntervalPredictorModel.m index 98c73ba7..47e1ee85 100644 --- a/docs/tutorials/CossanObjects/TutorialIntervalPredictorModel.m +++ b/docs/tutorials/CossanObjects/TutorialIntervalPredictorModel.m @@ -63,13 +63,13 @@ % displacement = load * length^3 / (3 * Young's modulus * Inertia) % %2.1. Definition of MIO object and construction of evaluator -Xmio = opencossan.workers.Mio('description','displacement at tip of cantilever beam', ... +Xmio = opencossan.workers.MatlabWorker('description','displacement at tip of cantilever beam', ... 'Script','for i=1:length(Tinput), Toutput(i).disp = Tinput(i).XP*Tinput(1).XL^3/(3*Tinput(1).XE*Tinput(i).XI);end',... 'InputNames',{'XP','XL','XE','XI'},... 'OutputNames',{'disp'},... 'Format','structure'); %2.2. Add MIO to evaluator -Xev = opencossan.workers.Evaluator('Sdescription','displacement at tip of cantilever beam','XMio',Xmio); +Xev = opencossan.workers.Evaluator('Description','displacement at tip of cantilever beam','Solver',Xmio); %2.3. Add Evaluator to a model Xmod = opencossan.common.Model('Evaluator',Xev,'Input',Xin); @@ -108,7 +108,7 @@ Xo_real = Xpm_real.computeFailureProbability(Xmc); %% Metamodel can also be also combined and used in a Evaluator -XevRS=opencossan.workers.Evaluator('Sdescription','displacement at tip of cantilever beam','Xmetamodel',Xipm); +XevRS=opencossan.workers.Evaluator('Description','displacement at tip of cantilever beam','Solver',Xipm); XmodRS= opencossan.common.Model('Evaluator',XevRS,'Input',Xin); Xpm_metamodel =opencossan.reliability.ProbabilisticModel('Model',XmodRS,'PerformanceFunction',Xpf); Xo_metamodel = Xpm_metamodel.computeFailureProbability(Xmc); @@ -116,7 +116,7 @@ assert(Xo_metamodel.pfhat>Xo_real.pfhat,'Upperbound on failure probability should be above the actual value') %Retry with the chance constraints -XevRS2=opencossan.workers.Evaluator('Sdescription','displacement at tip of cantilever beam','Xmetamodel',Xipm2); +XevRS2=opencossan.workers.Evaluator('Description','displacement at tip of cantilever beam','Solver',Xipm2); XmodRS2= opencossan.common.Model('Evaluator',XevRS2,'Input',Xin); Xpm_metamodel2 = opencossan.reliability.ProbabilisticModel('Model',XmodRS2,'PerformanceFunction',Xpf); Xo_metamodel2 = Xpm_metamodel2.computeFailureProbability(Xmc); diff --git a/docs/tutorials/CossanObjects/TutorialPolyharmonicSplines.m b/docs/tutorials/CossanObjects/TutorialPolyharmonicSplines.m index dfcfb377..d4b41df9 100644 --- a/docs/tutorials/CossanObjects/TutorialPolyharmonicSplines.m +++ b/docs/tutorials/CossanObjects/TutorialPolyharmonicSplines.m @@ -54,14 +54,14 @@ Xin = add(Xin,'Member',Xrvset,'Name','Xrvset'); %% Create Mio to Rosenbrock function -Xmio = Mio('FullFileName',[fullfile(opencossan.OpenCossan.getRoot),'/lib/MatlabFunctions/Rosenbrock/Rosenbrock.m'],... +Xmio = opencossan.workers.MatlabWorker('FullFileName',[fullfile(opencossan.OpenCossan.getRoot),'/lib/MatlabFunctions/Rosenbrock/Rosenbrock.m'],... 'IsFunction',true,... 'Format','matrix',... 'Inputnames',{'X1','X2','X3'},... 'Outputnames',{'out'}); %% Construct the Model -Xev = Evaluator('Xmio',Xmio); +Xev = Evaluator('Solver',Xmio); Xmod = Model('Evaluator',Xev,'Input',Xin); %% Construction and Calibration of Polyharmonic Splines diff --git a/docs/tutorials/CossanObjects/TutorialResponseSurface.m b/docs/tutorials/CossanObjects/TutorialResponseSurface.m index 389866d5..d195d795 100644 --- a/docs/tutorials/CossanObjects/TutorialResponseSurface.m +++ b/docs/tutorials/CossanObjects/TutorialResponseSurface.m @@ -1,8 +1,7 @@ %************************************************************************** % In this tutorial it is shown how to construct a ResponseSurface object % -% See Also: -% http://cossan.cfd.liv.ac.uk/wiki/index.php/@ResponseSurface +% See Also: ResponseSurface % Author: Matteo Broggi & Edoardo Patelli % Institute for Risk and Uncertainty, University of Liverpool, UK @@ -71,13 +70,13 @@ % displacement = load * length^3 / (3 * Young's modulus * Inertia) % %2.1. Definition of MIO object and construction of evaluator -Xmio = Mio('Description','displacement at tip of cantilever beam', ... +Xmio = opencossan.workers.MatlabWorker('Description','displacement at tip of cantilever beam', ... 'Script','for i=1:length(Tinput), Toutput(i).disp = Tinput(i).XP*Tinput(1).XL^3/(3*Tinput(1).XE*Tinput(i).XI);end',... 'InputNames',{'XP','XL','XE','XI'},... 'OutputNames',{'disp'},... 'Format','structure'); %2.2. Add MIO to evaluator -Xev = opencossan.workers.Evaluator('sdescription','displacement at tip of cantilever beam','XMio',Xmio); +Xev = opencossan.workers.Evaluator('Description','displacement at tip of cantilever beam','Solver',Xmio); %2.3. Add Evaluator to a model Xmod = Model('Evaluator',Xev,'Input',Xin); @@ -87,7 +86,7 @@ Xrs = opencossan.metamodels.ResponseSurface('sdescription',... 'response surface of tip displacement of cantilever beam',... 'XfullModel',Xmod,... %full model - 'Cinputnames',{'XP' 'Xh'},... + 'Cinputnames',{'XP' 'XI'},... 'Coutputnames',{'disp'},... %response to be extracted from full model 'Stype','custom',... 'Nmaximumexponent',4); %type of response surface @@ -109,7 +108,7 @@ Xo_real = Xpm_real.computeFailureProbability(Xmc) %% Metamodel can also be also combined and used in a Evaluator -XevRS=Evaluator('Sdescription','displacement at tip of cantilever beam','Xmetamodel',Xrs); +XevRS=Evaluator('Description','displacement at tip of cantilever beam','Solver',Xrs); XmodRS= Model('Evaluator',XevRS,'Input',Xin); Xpm_metamodel = ProbabilisticModel('Model',XmodRS,'PerformanceFunction',Xpf); Xo_metamodel = Xpm_metamodel.computeFailureProbability(Xmc) diff --git a/test/unit/+opencossan/+common/+inputs/+stochasticprocess/KarhunenLoeveTest.m b/test/unit/+opencossan/+common/+inputs/+stochasticprocess/KarhunenLoeveTest.m index 01b2e667..d62e29a5 100644 --- a/test/unit/+opencossan/+common/+inputs/+stochasticprocess/KarhunenLoeveTest.m +++ b/test/unit/+opencossan/+common/+inputs/+stochasticprocess/KarhunenLoeveTest.m @@ -100,7 +100,7 @@ function constructorNoNormalDistribution(testCase) function constructorInvalidCovFun(testCase) % should fail on initial input, can put in @Parameter object testCase.assertError(@()opencossan.common.inputs.stochasticprocess.KarhunenLoeve(... 'CovarianceFunction',opencossan.common.inputs.Parameter),... - 'MATLAB:UnableToConvert' ); + 'MATLAB:UnableToConvert'); end function constructorMcovariance(testCase) diff --git a/test/unit/+opencossan/+workers/EvaluatorTest.m b/test/unit/+opencossan/+workers/EvaluatorTest.m index cf17299d..cfa49a95 100644 --- a/test/unit/+opencossan/+workers/EvaluatorTest.m +++ b/test/unit/+opencossan/+workers/EvaluatorTest.m @@ -61,94 +61,94 @@ function constructorShouldSetDescription(testCase) end function constructorShouldSetMatlabWorker(testCase) - Xe = opencossan.workers.Evaluator('Solvers',testCase.MatWorker,... - 'SolversName',{'Xmio'}); - testCase.assertEqual(Xe.Solvers(1),testCase.MatWorker); + Xe = opencossan.workers.Evaluator('Solver',testCase.MatWorker,... + 'SolverName',{'Xmio'}); + testCase.assertEqual(Xe.Solver(1),testCase.MatWorker); end function constructorShouldFail(testCase) - Xe = opencossan.workers.Evaluator('Solvers',[testCase.MatWorker testCase.MatWorker2],... - 'SolversName',{'MatlabWorker' 'ExtraName'}); - testCase.assertEqual(Xe.Solvers(2),testCase.MatWorker2); + Xe = opencossan.workers.Evaluator('Solver',[testCase.MatWorker testCase.MatWorker2],... + 'SolverName',["MatlabWorker" "ExtraName"]); + testCase.assertEqual(Xe.Solver(2),testCase.MatWorker2); end function constructorShouldSetJobManagerInterface(testCase) Xjmi = opencossan.highperformancecomputing.JobManagerInterface(); - Xe = opencossan.workers.Evaluator('Solvers',testCase.MatWorker,'JobManager',Xjmi); + Xe = opencossan.workers.Evaluator('Solver',testCase.MatWorker,'JobManager',Xjmi); testCase.assertEqual(Xe.JobManager,Xjmi); end function constructorShouldSetLremoteInjectExtract(testCase) - Xe = opencossan.workers.Evaluator('Solvers',testCase.MatWorker,... + Xe = opencossan.workers.Evaluator('Solver',testCase.MatWorker,... 'RemoteInjectExtract',true); testCase.assertTrue(Xe.RemoteInjectExtract); end function constructorShouldSetHostNames(testCase) - Squeues = {'Queue1' 'Queue2'}; - Shostnames = {'Host1','Host2'}; - Xe = opencossan.workers.Evaluator('Solvers',[testCase.ConWorker testCase.MatWorker],... + Squeues = ["Queue1" "Queue2"]; + Shostnames = ["Host1","Host2"]; + Xe = opencossan.workers.Evaluator('Solver',[testCase.ConWorker testCase.MatWorker],... 'Hostnames',Shostnames,... 'Queues',Squeues); testCase.assertEqual(Xe.Hostnames,Shostnames); end function constructorShouldSetQueues(testCase) - Squeues = {'Queue1' 'Queue2'}; - Xe = opencossan.workers.Evaluator('Solvers',... + Squeues = ["Queue1" "Queue2"]; + Xe = opencossan.workers.Evaluator('Solver',... [testCase.ConWorker testCase.MatWorker],'Queues',Squeues); testCase.assertEqual(Xe.Queues,Squeues); end function constructorShouldSetVconcurrent(testCase) - Xe = opencossan.workers.Evaluator('Solvers',[testCase.ConWorker testCase.MatWorker],... + Xe = opencossan.workers.Evaluator('Solver',[testCase.ConWorker testCase.MatWorker],... 'MaxCuncurrentJobs',[Inf 4]); testCase.assertEqual(Xe.MaxCuncurrentJobs,[Inf 4]); end function constructorShouldSetMembers(testCase) - Xe = opencossan.workers.Evaluator('Solvers',[testCase.ConWorker testCase.MatWorker]); - testCase.assertEqual(Xe.Solvers,[testCase.ConWorker testCase.MatWorker]); + Xe = opencossan.workers.Evaluator('Solver',[testCase.ConWorker testCase.MatWorker]); + testCase.assertEqual(Xe.Solver,[testCase.ConWorker testCase.MatWorker]); end function constructorShouldSetNames(testCase) CSnames = ["Connector Name", "Mio name"]; - Xe = opencossan.workers.Evaluator('Solvers',[testCase.ConWorker testCase.MatWorker],'SolversName',CSnames); - testCase.assertEqual(Xe.Solvers,[testCase.ConWorker testCase.MatWorker]); - testCase.assertEqual(Xe.SolversName,CSnames); + Xe = opencossan.workers.Evaluator('Solver',[testCase.ConWorker testCase.MatWorker],'SolverName',CSnames); + testCase.assertEqual(Xe.Solver,[testCase.ConWorker testCase.MatWorker]); + testCase.assertEqual(Xe.SolverName,CSnames); end function constructorShouldSetSolutionSequence(testCase) Xss = opencossan.workers.SolutionSequence(); - Xe = opencossan.workers.Evaluator('Solvers',Xss); - testCase.assertEqual(Xe.Solvers(1),Xss); + Xe = opencossan.workers.Evaluator('Solver',Xss); + testCase.assertEqual(Xe.Solver(1),Xss); end function constructorShouldSetMetaModel(testCase) Xrs = opencossan.metamodels.ResponseSurface(); - Xe = opencossan.workers.Evaluator('Solvers',Xrs); - testCase.assertEqual(Xe.Solvers(1),Xrs); + Xe = opencossan.workers.Evaluator('Solver',Xrs); + testCase.assertEqual(Xe.Solver(1),Xrs); end function constructorShouldSetParallelEnvironment(testCase) - Xe = opencossan.workers.Evaluator('Solvers',[testCase.ConWorker testCase.MatWorker],... - 'ParallelEnvironments',{'parallel'}); - testCase.assertEqual(Xe.ParallelEnvironments,{'parallel'}); + Xe = opencossan.workers.Evaluator('Solver',[testCase.ConWorker testCase.MatWorker],... + 'ParallelEnvironments',["parallel" "parallel"]); + testCase.assertEqual(Xe.ParallelEnvironments,["parallel";"parallel"]); end function constructorShouldSetVSlots(testCase) - Xe = opencossan.workers.Evaluator('Solvers',[testCase.ConWorker testCase.MatWorker],... + Xe = opencossan.workers.Evaluator('Solver',[testCase.ConWorker testCase.MatWorker],... 'Slots',[1 1]); testCase.assertEqual(Xe.Slots,[1 1]); end function constructorTestInputNames(testCase) - Xe = opencossan.workers.Evaluator('Solvers',[testCase.MatWorker testCase.MatWorker2]); + Xe = opencossan.workers.Evaluator('Solver',[testCase.MatWorker testCase.MatWorker2]); Cinput=[testCase.MatWorker.InputNames testCase.MatWorker2.InputNames]; testCase.assertEqual(Xe.InputNames,Cinput); @@ -157,7 +157,7 @@ function constructorTestInputNames(testCase) function constructorTestOutputNames(testCase) - Xe = opencossan.workers.Evaluator('Solvers',[testCase.MatWorker testCase.MatWorker2]); + Xe = opencossan.workers.Evaluator('Solver',[testCase.MatWorker testCase.MatWorker2]); Coutput=[testCase.MatWorker.OutputNames testCase.MatWorker2.OutputNames]; testCase.assertEqual(Xe.OutputNames,Coutput); @@ -169,7 +169,7 @@ function deterministicAnalyis(testCase) 'OutputNames',{'out1'},'InputNames',{'Xpar'},'Format',"structure"); Xpar = opencossan.common.inputs.Parameter('value',10.2); Xinput = opencossan.common.inputs.Input('Parameter',Xpar); - Xe = opencossan.workers.Evaluator('Solvers',Xmio,'SolversName',"mio Name"); + Xe = opencossan.workers.Evaluator('Solver',Xmio,'SolverName',"mio Name"); Xout=Xe.deterministicAnalysis(Xinput); testCase.assertClass(Xout,'opencossan.common.outputs.SimulationData'); testCase.assertEqual(Xout.getValues('Sname','out1'),10.2); diff --git a/test/unit/+opencossan/+workers/MatlabWorkerTest.m b/test/unit/+opencossan/+workers/MatlabWorkerTest.m index cd595075..71ae67fe 100644 --- a/test/unit/+opencossan/+workers/MatlabWorkerTest.m +++ b/test/unit/+opencossan/+workers/MatlabWorkerTest.m @@ -38,8 +38,8 @@ function skip(testCase) methods (TestMethodSetup) function createInput(testCase) - Xrv1 = opencossan.common.inputs.RandomVariable('Sdistribution','uniform','par1',9,'par2',11); - Xrv2 = opencossan.common.inputs.RandomVariable('Sdistribution','uniform','par1',14,'par2',16); + Xrv1 = opencossan.common.inputs.random.Uniform('par1',9,'par2',11); + Xrv2 = opencossan.common.inputs.Uniform('par1',14,'par2',16); Xrvs = opencossan.common.inputs.RandomVariableSet('CSMembers',{'Xrv1','Xrv2'},'CXmembers',{Xrv1,Xrv2}); testCase.Xin = opencossan.common.inputs.Input('CXmembers',{Xrvs},'CSmembers',{'Xrvs'}); end From 8087351708e1c3527fb861ee2260a3de5ae509b1 Mon Sep 17 00:00:00 2001 From: Edoardo Patelli Date: Wed, 19 Feb 2020 19:26:50 +0000 Subject: [PATCH 5/9] Started integration of new JobManager and SLURM support --- .../+highperformancecomputing/@Job/Job.m | 59 ++++ .../@JobManager/JobManager.m | 267 +++++++----------- .../@JobManager/deleteJob.m | 45 --- .../@JobManager/display.m | 20 -- .../@JobManager/getJobStatus.m | 27 -- .../@JobManagerSlurm/JobManagerSlurm.m | 61 ++++ .../@JobManagerSlurm/cancelJob.m | 80 ++++++ .../@JobManagerSlurm/extractJobId.m | 35 +++ .../@JobManagerSlurm/getSubmitString.m | 7 + .../@JobManagerSlurm/nonshared/README | 143 ++++++++++ .../@JobManagerSlurm/nonshared/cancelJobFcn.m | 91 ++++++ .../nonshared/communicatingJobWrapper.sh | 53 ++++ .../nonshared/communicatingJobWrapperSmpd.sh | 258 +++++++++++++++++ .../nonshared/communicatingSubmitFcn.m | 190 +++++++++++++ .../nonshared/createSubmitScript.m | 30 ++ .../@JobManagerSlurm/nonshared/deleteJobFcn.m | 8 + .../@JobManagerSlurm/nonshared/extractJobId.m | 31 ++ .../nonshared/getCommonSubmitArgs.m | 18 ++ .../nonshared/getJobStateFcn.m | 161 +++++++++++ .../nonshared/getRemoteConnection.m | 220 +++++++++++++++ .../nonshared/getSubmitString.m | 7 + .../nonshared/independentJobWrapper.sh | 15 + .../nonshared/independentSubmitFcn.m | 195 +++++++++++++ .../@JobManagerSlurm/prepareScript.m | 131 +++++++++ .../@JobManagerSlurm/remote/README | 144 ++++++++++ .../@JobManagerSlurm/remote/cancelJobFcn.m | 78 +++++ .../remote/communicatingJobWrapper.sh | 257 +++++++++++++++++ .../remote/communicatingSubmitFcn.m | 142 ++++++++++ .../remote/createSubmitScript.m | 30 ++ .../@JobManagerSlurm/remote/deleteJobFcn.m | 8 + .../remote/getCommonSubmitArgs.m | 18 ++ .../@JobManagerSlurm/remote/getJobStateFcn.m | 119 ++++++++ .../remote/getRemoteConnection.m | 185 ++++++++++++ .../@JobManagerSlurm/remote/getSubmitString.m | 7 + .../remote/independentJobWrapper.sh | 15 + .../remote/independentSubmitFcn.m | 147 ++++++++++ .../@JobManagerSlurm/shared/README | 114 ++++++++ .../shared/communicatingJobWrapper.sh | 52 ++++ .../shared/communicatingJobWrapperSmpd.sh | 258 +++++++++++++++++ .../shared/communicatingSubmitFcn.m | 136 +++++++++ .../@JobManagerSlurm/shared/extractJobId.m | 33 +++ .../shared/getCommonSubmitArgs.m | 18 ++ .../@JobManagerSlurm/shared/getJobStateFcn.m | 110 ++++++++ .../shared/independentJobWrapper.sh | 15 + .../@JobManagerSlurm/submitJob.m | 122 ++++++++ 45 files changed, 3896 insertions(+), 264 deletions(-) create mode 100644 +opencossan/+highperformancecomputing/@Job/Job.m delete mode 100644 +opencossan/+highperformancecomputing/@JobManager/deleteJob.m delete mode 100644 +opencossan/+highperformancecomputing/@JobManager/display.m delete mode 100644 +opencossan/+highperformancecomputing/@JobManager/getJobStatus.m create mode 100644 +opencossan/+highperformancecomputing/@JobManagerSlurm/JobManagerSlurm.m create mode 100644 +opencossan/+highperformancecomputing/@JobManagerSlurm/cancelJob.m create mode 100644 +opencossan/+highperformancecomputing/@JobManagerSlurm/extractJobId.m create mode 100644 +opencossan/+highperformancecomputing/@JobManagerSlurm/getSubmitString.m create mode 100644 +opencossan/+highperformancecomputing/@JobManagerSlurm/nonshared/README create mode 100644 +opencossan/+highperformancecomputing/@JobManagerSlurm/nonshared/cancelJobFcn.m create mode 100644 +opencossan/+highperformancecomputing/@JobManagerSlurm/nonshared/communicatingJobWrapper.sh create mode 100644 +opencossan/+highperformancecomputing/@JobManagerSlurm/nonshared/communicatingJobWrapperSmpd.sh create mode 100644 +opencossan/+highperformancecomputing/@JobManagerSlurm/nonshared/communicatingSubmitFcn.m create mode 100644 +opencossan/+highperformancecomputing/@JobManagerSlurm/nonshared/createSubmitScript.m create mode 100644 +opencossan/+highperformancecomputing/@JobManagerSlurm/nonshared/deleteJobFcn.m create mode 100644 +opencossan/+highperformancecomputing/@JobManagerSlurm/nonshared/extractJobId.m create mode 100644 +opencossan/+highperformancecomputing/@JobManagerSlurm/nonshared/getCommonSubmitArgs.m create mode 100644 +opencossan/+highperformancecomputing/@JobManagerSlurm/nonshared/getJobStateFcn.m create mode 100644 +opencossan/+highperformancecomputing/@JobManagerSlurm/nonshared/getRemoteConnection.m create mode 100644 +opencossan/+highperformancecomputing/@JobManagerSlurm/nonshared/getSubmitString.m create mode 100644 +opencossan/+highperformancecomputing/@JobManagerSlurm/nonshared/independentJobWrapper.sh create mode 100644 +opencossan/+highperformancecomputing/@JobManagerSlurm/nonshared/independentSubmitFcn.m create mode 100644 +opencossan/+highperformancecomputing/@JobManagerSlurm/prepareScript.m create mode 100644 +opencossan/+highperformancecomputing/@JobManagerSlurm/remote/README create mode 100644 +opencossan/+highperformancecomputing/@JobManagerSlurm/remote/cancelJobFcn.m create mode 100644 +opencossan/+highperformancecomputing/@JobManagerSlurm/remote/communicatingJobWrapper.sh create mode 100644 +opencossan/+highperformancecomputing/@JobManagerSlurm/remote/communicatingSubmitFcn.m create mode 100644 +opencossan/+highperformancecomputing/@JobManagerSlurm/remote/createSubmitScript.m create mode 100644 +opencossan/+highperformancecomputing/@JobManagerSlurm/remote/deleteJobFcn.m create mode 100644 +opencossan/+highperformancecomputing/@JobManagerSlurm/remote/getCommonSubmitArgs.m create mode 100644 +opencossan/+highperformancecomputing/@JobManagerSlurm/remote/getJobStateFcn.m create mode 100644 +opencossan/+highperformancecomputing/@JobManagerSlurm/remote/getRemoteConnection.m create mode 100644 +opencossan/+highperformancecomputing/@JobManagerSlurm/remote/getSubmitString.m create mode 100644 +opencossan/+highperformancecomputing/@JobManagerSlurm/remote/independentJobWrapper.sh create mode 100644 +opencossan/+highperformancecomputing/@JobManagerSlurm/remote/independentSubmitFcn.m create mode 100644 +opencossan/+highperformancecomputing/@JobManagerSlurm/shared/README create mode 100644 +opencossan/+highperformancecomputing/@JobManagerSlurm/shared/communicatingJobWrapper.sh create mode 100644 +opencossan/+highperformancecomputing/@JobManagerSlurm/shared/communicatingJobWrapperSmpd.sh create mode 100644 +opencossan/+highperformancecomputing/@JobManagerSlurm/shared/communicatingSubmitFcn.m create mode 100644 +opencossan/+highperformancecomputing/@JobManagerSlurm/shared/extractJobId.m create mode 100644 +opencossan/+highperformancecomputing/@JobManagerSlurm/shared/getCommonSubmitArgs.m create mode 100644 +opencossan/+highperformancecomputing/@JobManagerSlurm/shared/getJobStateFcn.m create mode 100644 +opencossan/+highperformancecomputing/@JobManagerSlurm/shared/independentJobWrapper.sh create mode 100644 +opencossan/+highperformancecomputing/@JobManagerSlurm/submitJob.m diff --git a/+opencossan/+highperformancecomputing/@Job/Job.m b/+opencossan/+highperformancecomputing/@Job/Job.m new file mode 100644 index 00000000..41920e12 --- /dev/null +++ b/+opencossan/+highperformancecomputing/@Job/Job.m @@ -0,0 +1,59 @@ +classdef Job < opencossan.common.CossanObject + %JOB This class provide information of job prepared and submitted to a + %cluster or grid computing. + % + % See also: JobManager, Evaluator + + %{ + This file is part of OpenCossan . + Copyright (C) 2006-2020 COSSAN WORKING GROUP + + OpenCossan is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License or, + (at your option) any later version. + + OpenCossan is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with OpenCossan. If not, see . + %} + + properties + ID(1,:) {mustBeInteger} + Name + State + Dependences + ScriptName + end + + methods + function obj = Job(varargin) + + if nargin == 0 + superArg = {}; + else + + [requiredArgs, varargin] = opencossan.common.utilities.parseRequiredNameValuePairs(... + ["ID","Status"], varargin{:}); + + [optionalArgs, superArg] = opencossan.common.utilities.parseOptionalNameValuePairs(... + ["Name","Dependences"],{[],[]}, varargin{:}); + end + + obj@opencossan.common.CossanObject(superArg{:}); + + if nargin > 0 + obj.ID=requiredArgs.id; + obj.State=requiredArgs.state; + obj.Name=optionalArgs.name; + obj.Dependences=optionalArgs.dependences; + end + end + + end +end + diff --git a/+opencossan/+highperformancecomputing/@JobManager/JobManager.m b/+opencossan/+highperformancecomputing/@JobManager/JobManager.m index b7d8d742..dbf6687b 100644 --- a/+opencossan/+highperformancecomputing/@JobManager/JobManager.m +++ b/+opencossan/+highperformancecomputing/@JobManager/JobManager.m @@ -1,205 +1,128 @@ -classdef JobManager - % % Objects of this class are used by an Evaluator to run the - % analysis on a remote machine. The JobManagerInterface defines the - % interface with the Job Management software. The requested computation are - % automatically converted to jobs and submitted to the Job management - % software on the cluster. The results are then retrieved and - % processed by COSSAN (i.e., in a SimulationData object). - - % ===================================================================== - % This file is part of openCOSSAN. The open general purpose matlab - % toolbox for numerical analysis, risk and uncertainty quantification. - % - % openCOSSAN is free software: you can redistribute it and/or modify - % it under the terms of the GNU General Public License as published by - % the Free Software Foundation, either version 3 of the License. - % - % openCOSSAN is distributed in the hope that it will be useful, - % but WITHOUT ANY WARRANTY; without even the implied warranty of - % MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - % GNU General Public License for more details. +classdef(Abstract) JobManager < opencossan.common.CossanObject + % This class defines the interface with the cluster/cloud computing. + % The requested computation are automatically converted to jobs and + % submitted to the Job management software on the cluster. The results are then retrieved and + % processed by OpenCossan (i.e., in a SimulationData object). % - % You should have received a copy of the GNU General Public License - % along with openCOSSAN. If not, see . + % See also: JobManagerSlurm, Worker, Evaluator, + + %{ + This file is part of OpenCossan . + Copyright (C) 2006-2020 COSSAN WORKING GROUP + + OpenCossan is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License or, + (at your option) any later version. + + OpenCossan is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with OpenCossan. If not, see . + %} + % ===================================================================== properties - Squeue % Queue name - Shostname % Specific node (host) selection - Sdescription % Description of the JobManager - Sjobname = 'CossanJob' % Job name - Spreexecmd % prepocessor executeble cmd TODO: check with connector - Spostexecmd % postprocessor executeble cmd TODO: check with connector - Sworkingdirectory % directory when the job will be executed (better if not on NFS!) - Smaininputpath - Sfoldername % Name of the folder created by the JobManager for its execution - Xdependent = [] % Dependent object (no execution until this object job is finished) - TODO: which kind of object is? - Xjobmanagerinterface - Nconcurrent = Inf % Number of concurrent job execution (Default = unlimited) - SparallelEnvironment % Specific parallel environment (e.g. openmpi) and number of process - Nslots % Number of slots to be used in the simulation + PreExeCmd % prepocessor executeble cmd TODO: check with connector + PostExeCmd % postprocessor executeble cmd TODO: check with connector + WorkingPath % directory when the job will be executed (better if not on NFS!) + MainInputPath + SubFolder % Subfolder name for the batch execution + HasSharedFilesystem true % + ClusterMatlabRoot + OperatingSystem + JobStorageLocation + SSHconnection + LogFile(1,1) {string} end properties (Hidden) - Sexecmd % job submission executable cmd. Set by runJob of connector! - Sexeflags + ExeCmd % job submission executable cmd. Set by runJob of connector! + ExeFlags end properties (Hidden,SetAccess=protected) - SjobPrefixName='run' - SbatchIdentification='_job_' + ShellScriptPrefix='OpenCossan_job' % Prefix used in the shell script + BatchIdentification='_job_' % Batch identification name + JobNamePrefix = 'CossanJob' % Job name end properties (Dependent=true) - SjobScriptName % name of the script used to start the job - SuserName + JobName % name of the script used to start the job + Username + isRemoteCluster end methods - function Xobj=JobManager(varargin) - % ================================================================== - % COSSAN-X - The next generation of the computational stochastic analysis - % University of Innsbruck, Copyright 1993-2011 IfM - % ================================================================== - - % ===================================================================== - % This file is part of openCOSSAN. The open general purpose matlab - % toolbox for numerical analysis, risk and uncertainty quantification. - % - % openCOSSAN is free software: you can redistribute it and/or modify - % it under the terms of the GNU General Public License as published by - % the Free Software Foundation, either version 3 of the License. - % - % openCOSSAN is distributed in the hope that it will be useful, - % but WITHOUT ANY WARRANTY; without even the implied warranty of - % MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - % GNU General Public License for more details. - % - % You should have received a copy of the GNU General Public License - % along with openCOSSAN. If not, see . - % ===================================================================== - - %% Argument Check - opencossan.OpenCossan.validateCossanInputs(varargin{:}) - - % set the values of the public properties from the input - % parameters - for k=1:2:length(varargin) - switch lower(varargin{k}) - case ('sdescription') - Xobj.Sdescription = varargin{k+1}; - case ('squeue') - Xobj.Squeue = varargin{k+1}; - case ('shostname') - Xobj.Shostname = varargin{k+1}; - case ('sjobname') - Xobj.Sjobname = varargin{k+1}; - case ('spreexecmd') - Xobj.Spreexecmd = varargin{k+1}; - case ('spostexecmd') - Xobj.Spostexecmd = varargin{k+1}; - case ('xdependent') - if isa(varargin{k+1},'Evaluator') - if ~isempty(varargin{k+1}.Xjob) - Xobj.Xdependent = varargin{k+1}.Xjob; - else - error('openCOSSAN:JobManager:JobManager',... - 'Cannot create dependent job using an Evaluator without JobManager.'); - end - elseif isa(varargin{k+1},'JobManager') - Xobj.Xdependent = varargin{k+1}; - else - error('openCOSSAN:JobManager:JobManager',... - 'Dependent object must be of Evaluator or JobManager class'); - end - if isempty(Xobj.Xdependent.Sjobname) - error('openCOSSAN:JobManager:JobManager',... - 'A job name must be defined in the dependent object'); - end - case{'xjobmanagerinterface'} - Xobj.Xjobmanagerinterface = varargin{k+1}; - case{'cxjobmanagerinterface'} - Xobj.Xjobmanagerinterface = varargin{k+1}{1}; - case{'nconcurrent'} - Xobj.Nconcurrent = varargin{k+1}; - case ('sparallelenvironment') - Xobj.SparallelEnvironment = varargin{k+1}; - case ('nslots') - Xobj.Nslots = varargin{k+1}; - case ('sworkingdirectory') - Xobj.Sworkingdirectory = varargin{k+1}; - otherwise - error('openCOSSAN:JobManager',... - ['PropertyName name (' varargin{k} ') not allowed']); - end + function obj=JobManager(varargin) + %Constructor of JobManager + if nargin == 0 + superArg = {}; + else + % No mandatory parameters + + % Process optional paramets + OptionalsArguments={... + "PreExeCmd", opencossan.highperformancecomputing.JobManagerInterface.empty(1,0);... + "PostExeCmd",[];... + "WorkingPath",opencossan.OpenCossan.getWorkingPath;... + "MainInputPath",[];... + "SubFolder",datestr(now,30),... + "HasSharedFilesystem",true,... + "LogFile","OpenCossanJob.log"}; + + [optionalArgs, superArg] = opencossan.common.utilities.parseOptionalNameValuePairs(... + [OptionalsArguments{:,1}],{OptionalsArguments{:,2}}, varargin{:}); end + obj@opencossan.common.CossanObject(superArg{:}); - %% Check JobManager - - if isempty(Xobj.Sworkingdirectory) - % Use defailt working path defined in OpenCossan - Xobj.Sworkingdirectory=opencossan.OpenCossan.getWorkingPath; + if nargin > 0 + obj.WorkingPath=optionalArgs.workingpath; + obj.KeepSimulationFiles=optionalArgs.keepsimulationfiles; + obj.HasSharedFilesystem=optionalArgs.hassharedfilesystem; + obj.LogFile=optionalArgs.logfile; end - - % It is mandatory to have a JobManagerInterface object. If such - % object is missing, an error is given. - assert(isa(Xobj.Xjobmanagerinterface,'opencossan.highperformancecomputing.JobManagerInterface'), ... - 'openCOSSAN:JobManager', ... - 'An object of class JobManagerInterface is required.'); - - if ~isempty(Xobj.Squeue) - assert(logical(ismember(Xobj.Squeue,Xobj.Xjobmanagerinterface.getQueues)), ... - 'openCOSSAN:JobManager', ... - 'The selected queue %s is not available \nAvailable queues are: %s', ... - Xobj.Squeue,sprintf('%s ',Xobj.Xjobmanagerinterface.getQueues{:})); - end - - if ~isempty(Xobj.SparallelEnvironment) - assert(logical(ismember(Xobj.SparallelEnvironment,... - Xobj.Xjobmanagerinterface.getParallelEnvironments)), ... - 'openCOSSAN:JobManager', ... - ['The selected parallel environment %s is not available \n'... - 'Available parallel environments are: %s'], ... - Xobj.SparallelEnvironment,sprintf('%s ',Xobj.Xjobmanagerinterface.getParallelEnvironments{:})); - assert(~isempty(Xobj.Nslots),... - 'openCOSSAN:JobManager', ... - ['It is mandatory to specify the number of slots to be'... - ' used with the selected parallel environment.']); - end - - if ~isempty(Xobj.Shostname) - [Lavailable, Smess]=Xobj.Xjobmanagerinterface.checkHost( ... - 'SqueueName',Xobj.Squeue,'ShostName',Xobj.Shostname); - assert(Lavailable, 'openCOSSAN:JobManager', Smess) - end - - %% set the value of the private properties - Xobj.Sfoldername = datestr(now,30); - + end - - function SjobScriptName = get.SjobScriptName(Xobj) + + function JobName = get.JobName(obj) %TODO: This is not an unique name, better to use Sfoldername %instead SjobPrefixName - SjobScriptName = [Xobj.SjobPrefixName Xobj.Sjobname]; + JobName = [obj.JobNamePrefix obj.jobname]; end - SjobID = submitJob(Xobj,varargin) % submit single job - CSjobID = submitAnalysis(Xobj,Nsamples) % submit analysis (multiple jobs) + function isRemoteCluster = get.isRemoteCluster(obj) + % Check if a SSHConnection object has been defined + if isempty(obj.SSHconnection) + isRemoteCluster=false; + else + isRemoteCluster=true; + end + end - varargout=deleteJob(Xobj,varargin) - Lsuccessfull = checkSuccessfulJobs(Xobj,ScheckFileName) - display(Xobj) + jobID = submitJob(obj,varargin) % submit single job + jobID = submitAnalysis(obj,Samples) % submit analysis (multiple jobs) + + % Define function requirements for subclasses + + isOK=cancelJob(obj,JobObject) + State = getJobState(Xobj,JobObject) + Lsuccessfull = checkSuccessfulJobs(obj,ScheckFileName) + Status=checkCluster(obj,varargin + end - - methods (Access=private) - Sscriptname = prepareGridEngineScript(Xobj,NsimulationNumber) - Sscriptname = prepareLSFScript(Xobj,NsimulationNumber) - Sscriptname = prepareGridEngineArrayScript(Xobj) + + methods (static) + jobId=extractJobId(sbatchCommandOutput) end + end diff --git a/+opencossan/+highperformancecomputing/@JobManager/deleteJob.m b/+opencossan/+highperformancecomputing/@JobManager/deleteJob.m deleted file mode 100644 index f3e9b002..00000000 --- a/+opencossan/+highperformancecomputing/@JobManager/deleteJob.m +++ /dev/null @@ -1,45 +0,0 @@ -function varargout = deleteJob(Xobj,varargin) -% -% submit a job to a job manager -% ================================================================== -% COSSAN-X - The next generation of the computational stochastic analysis -% University of Innsbruck, Copyright 1993-2011 IfM -% ================================================================== - - -%% Process optional inputs -% NOTE: all the optional input parameters are properties of the jobmanager -% object -% - -OpenCossan.validateCossanInputs(varargin{:}) - -for iopt=1:2:length(varargin) - switch lower(varargin{iopt}) - case {'csjobid'} - CSjobID = varargin{iopt+1}; - otherwise - error('openCOSSAN:JobManager:deleteJob',... - ['PropertyName ' varargin{iopt} ' not allowed']); - end -end - -for n=1:length(CSjobID) - if ~isempty(CSjobID{n}) - if ~OpenCossan.hasSSHConnection - [~,CSstatus{n}]=system([Xobj.Xjobmanagerinterface.SdeleteJob ' ' CSjobID{n}]); %#ok - else - [~,CSstatus{n}]=OpenCossan.issueSSHcommand([Xobj.Xjobmanagerinterface.SdeleteJob ' ' CSjobID{n}]); %#ok - end - OpenCossan.cossanDisp(['Job ' CSjobID{n} ' canceled: ' CSstatus{n}],1) - end -end - - - -if nargout>0 - varargout{1}=CSstatus; -end - -end - diff --git a/+opencossan/+highperformancecomputing/@JobManager/display.m b/+opencossan/+highperformancecomputing/@JobManager/display.m deleted file mode 100644 index 50c34f75..00000000 --- a/+opencossan/+highperformancecomputing/@JobManager/display.m +++ /dev/null @@ -1,20 +0,0 @@ -function display(Xobj) -%DISPLAY Displays the object JobManager -% -% ================================================================== -% COSSAN-X - The next generation of the computational stochastic analysis -% University of Innsbruck, Copyright 1993-2011 IfM -% ================================================================== - -%% Output to Screen -% Name and description -OpenCossan.cossanDisp('===================================================================',3); -OpenCossan.cossanDisp([' JobManager Object - ' Xobj.Sdescription ],1); -OpenCossan.cossanDisp('===================================================================',3); - -% main paramenters -OpenCossan.cossanDisp(['Queue: ' Xobj.Squeue],2); -OpenCossan.cossanDisp(['Hostnanme: ' Xobj.Shostname],2); - -OpenCossan.cossanDisp(['Folder Name: ' Xobj.Sfoldername],3); -OpenCossan.cossanDisp(['Max Concurrent simulation: ' num2str(Xobj.Nconcurrent)],3); diff --git a/+opencossan/+highperformancecomputing/@JobManager/getJobStatus.m b/+opencossan/+highperformancecomputing/@JobManager/getJobStatus.m deleted file mode 100644 index 05a0febb..00000000 --- a/+opencossan/+highperformancecomputing/@JobManager/getJobStatus.m +++ /dev/null @@ -1,27 +0,0 @@ -function Cstatus = getJobStatus(Xobj,varargin) -%getJobStatus This function retrieves the status of a specific job from the -% JobManagerInterface -% -% $Copyright~1993-2012,~COSSAN~Working~Group,~University~of~Liverpool,~UK$ -% $Author: Matteo Broggi$ - -% ===================================================================== -% This file is part of openCOSSAN. The open general purpose matlab -% toolbox for numerical analysis, risk and uncertainty quantification. -% -% openCOSSAN is free software: you can redistribute it and/or modify -% it under the terms of the GNU General Public License as published by -% the Free Software Foundation, either version 3 of the License. -% -% openCOSSAN is distributed in the hope that it will be useful, -% but WITHOUT ANY WARRANTY; without even the implied warranty of -% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -% GNU General Public License for more details. -% -% You should have received a copy of the GNU General Public License -% along with openCOSSAN. If not, see . -% ===================================================================== - - -%% Process optional inputs -Cstatus = getJobStatus(Xobj.Xjobmanagerinterface,varargin{:}); diff --git a/+opencossan/+highperformancecomputing/@JobManagerSlurm/JobManagerSlurm.m b/+opencossan/+highperformancecomputing/@JobManagerSlurm/JobManagerSlurm.m new file mode 100644 index 00000000..5b4c6203 --- /dev/null +++ b/+opencossan/+highperformancecomputing/@JobManagerSlurm/JobManagerSlurm.m @@ -0,0 +1,61 @@ +classdef(Abstract) JobManagerSlurm < opencossan.highperformancecomputing.JobManager + % This class defines the interface with the cluster/cloud computing. + % The requested computation are automatically converted to jobs and + % submitted to the Job management software on the cluster. The results are then retrieved and + % processed by OpenCossan (i.e., in a SimulationData object). + % + % See also: JobManagerSlurm, Worker, Evaluator, + + %{ + This file is part of OpenCossan . + Copyright (C) 2006-2020 COSSAN WORKING GROUP + + OpenCossan is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License or, + (at your option) any later version. + + OpenCossan is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with OpenCossan. If not, see . + %} + + % ===================================================================== + + properties + + end + + + methods + + function obj=JobManagerSlurm(varargin) + %Constructor of JobManager + if nargin == 0 + superArg = {}; + else + % No mandatory parameters + + % Process optional paramets + OptionalsArguments={... + "Undefined", [];}; + + [~, superArg] = opencossan.common.utilities.parseOptionalNameValuePairs(... + [OptionalsArguments{:,1}],OptionalsArguments(:,2), varargin{:}); + end + + obj@opencossan.highperformancecomputing.JobManager(superArg{:}); + + end + + end + + methods (private) + jobID = extractJobId(obj,sbatchCommandOutput) + end + +end diff --git a/+opencossan/+highperformancecomputing/@JobManagerSlurm/cancelJob.m b/+opencossan/+highperformancecomputing/@JobManagerSlurm/cancelJob.m new file mode 100644 index 00000000..7f5726c5 --- /dev/null +++ b/+opencossan/+highperformancecomputing/@JobManagerSlurm/cancelJob.m @@ -0,0 +1,80 @@ +function isOK = cancelJob(obj, JobObject) +%CANCELJOB Cancels a job on Slurm. Adapted from cancelJobFnc. Available at +%https://github.com/cossan-working-group/matlab-roll +% +% Set your cluster's IntegrationScriptsLocation to the parent folder of this +% function to run it when you cancel a job. + +% Copyright 2010-2018 The MathWorks, Inc. + +% Store the current filename for the errors, warnings and dctSchedulerMessages +currFilename = mfilename; + +% This should work for all the cases +% +% if ~obj.HasSharedFilesystem +% error('JobManagerSlurm:GenericSLURM:NotSharedFileSystem', ... +% 'The function %s is for use with shared filesystems.', currFilename) +% end + + +% Get the information about the actual cluster used + +try + JobObject = getJobState(obj,JobObject); +catch err + ex = MException('JobManagerSlurm:GenericSLURM:FailedToRetrieveJobID', ... + 'Failed to retrieve clusters''s job IDs from the job cluster data.'); + ex = ex.addCause(err); + throw(ex); +end + +if isempty(JobObject.State) + % This indicates that the job has not been submitted, so return true + opencossan.OpenCossan.cossanDisp(sprintf('%s: Job cluster data was empty for job with ID %d.', currFilename, JobObject.ID),2); + isOK = true; + return +end + +% Only ask the cluster to cancel the job if it is hasn't reached a terminal +% state. +erroredJobAndCauseStrings = cell(size(JobObject.ID)); + +for ii = 1:length(jobIDs) + jobState = JobObject.State(ii); + jobID = JobObject.ID(ii); + if ~(strcmp(jobState, "finished") || strcmp(jobState, "failed")) + % Get the cluster to delete the job + commandToRun = sprintf('scancel ''%s''', jobID); + opencossan.OpenCossan.cossanDisp(sprintf('%s: Canceling job on cluster using command:\n\t%s.',... + currFilename, commandToRun),4); + try + if obj.isRemoteCluster + [cmdFailed, cmdOut] = obj.SSHconnection.runCommand(commandToRun); + else + % Make the shelled out call to run the command. + [cmdFailed, cmdOut] = system(commandToRun); + end + catch err + cmdFailed = true; + cmdOut = err.message; + end + + if cmdFailed + % Keep track of all jobs that errored when being cancelled. + % We'll report these later on. + erroredJobAndCauseStrings{ii} = sprintf('Job ID: %s\tReason: %s', jobID, strtrim(cmdOut)); + opencossan.OpenCossan.cossanDisp(sprintf('%s: Failed to cancel job %d on cluster. Reason:\n\t%s',... + currFilename, jobID, cmdOut),1); + end + end +end + +% Now warn about those jobs that we failed to cancel. +erroredJobAndCauseStrings = erroredJobAndCauseStrings(~cellfun(@isempty, erroredJobAndCauseStrings)); +if ~isempty(erroredJobAndCauseStrings) + warning('JobManagerSlurm:GenericSLURM:FailedToCancelJob', ... + 'Failed to cancel the following jobs on the cluster:\n%s', ... + sprintf(' %s\n', erroredJobAndCauseStrings{:})); +end +isOK = isempty(erroredJobAndCauseStrings); diff --git a/+opencossan/+highperformancecomputing/@JobManagerSlurm/extractJobId.m b/+opencossan/+highperformancecomputing/@JobManagerSlurm/extractJobId.m new file mode 100644 index 00000000..fe8baf40 --- /dev/null +++ b/+opencossan/+highperformancecomputing/@JobManagerSlurm/extractJobId.m @@ -0,0 +1,35 @@ +function jobID = extractJobId(sbatchCommandOutput) +% Extracts the job ID from the sbatch command output for Slurm +% Adapted from matlab-roll +% https://github.com/cossan-working-group/matlab-roll + +% Copyright 2015-2017 The MathWorks, Inc. + +% Output from sbatch expected to be in the following format: +% Submitted batch job 12345 +% +% sbatch could also attach a warning to the output, such as: +% +% sbatch: Warning: can't run 1 processes on 3 nodes, setting nnodes to 1 +% Submitted batch job 12346 + +% Trim sbatch command output for use in debug message +trimmedCommandOutput = strtrim(sbatchCommandOutput); + +% Ignore anything before or after 'Submitted batch job ###', and extract the numeric value. +searchPattern = '.*Submitted batch job ([0-9]+).*'; + +% When we match searchPattern, matchedTokens is a single entry cell array containing the jobID. +% Otherwise we failed to match searchPattern, so matchedTokens is an empty cell array. +matchedTokens = regexp(sbatchCommandOutput, searchPattern, 'tokens', 'once'); + +if isempty(matchedTokens) + % Callers check for error in extracting Job ID using isempty() on return value. + jobID = ''; + opencossan.OpenCossan.cossanDisp(sprintf('%s: Failed to extract Job ID from sbatch output: \n\t%s',... + mfilename, trimmedCommandOutput),1); +else + jobID = matchedTokens{1}; + opencossan.OpenCossan.cossanDisp(sprintf('%s: Job ID %s was extracted from sbatch output: \n\t%s',... + mfilename, jobID, trimmedCommandOutput),1); +end diff --git a/+opencossan/+highperformancecomputing/@JobManagerSlurm/getSubmitString.m b/+opencossan/+highperformancecomputing/@JobManagerSlurm/getSubmitString.m new file mode 100644 index 00000000..ec51b7a1 --- /dev/null +++ b/+opencossan/+highperformancecomputing/@JobManagerSlurm/getSubmitString.m @@ -0,0 +1,7 @@ +function submitString = getSubmitString(obj, quotedCommand, additionalSubmitArgs) +%GETSUBMITSTRING Gets the correct sbatch command for a Slurm cluster + +% Copyright 2010-2016 The MathWorks, Inc. + +submitString = sprintf('sbatch --job-name=%s --output=%s %s %s', ... + obj.jobName, obj.LogFile, additionalSubmitArgs, quotedCommand); diff --git a/+opencossan/+highperformancecomputing/@JobManagerSlurm/nonshared/README b/+opencossan/+highperformancecomputing/@JobManagerSlurm/nonshared/README new file mode 100644 index 00000000..98dbd9bb --- /dev/null +++ b/+opencossan/+highperformancecomputing/@JobManagerSlurm/nonshared/README @@ -0,0 +1,143 @@ +Copyright 2010-2016 The MathWorks, Inc. + +MATLAB and Simulink are registered trademarks of The MathWorks, Inc. +See www.mathworks.com/trademarks for a list of additional trademarks. +Other product or brand names may be trademarks or registered trademarks of their respective holders. + +This folder contains a number of files to allow Parallel Computing Toolbox +to be used with Slurm via the generic cluster interface. + +The files in this folder assume that the client and cluster do not share a file system +and that the client is not able to submit directly to the cluster using the +sbatch command. + +Note that all the files in this directory will work only for clusters that are +running on UNIX. + +1) Instructions for Use +======================= +On the Slurm cluster +-------------------- +Enable job accounting on the Slurm cluster. +This allows the sacct command to run. +The scripts use the sacct output to track the state of Slurm jobs. + +Read the documentation for using the generic cluster interface with +the Parallel Computing Toolbox and familiarize yourself with the different +properties that can be set for a generic cluster. + +In the MATLAB Client +-------------------- +1. Create a generic cluster object for your cluster. Set the IntegrationScriptsLocation +to this folder. For independent jobs, independentSubmitFcn is used as your +submit function. For communicating jobs, communicatingSubmitFcn is used as +your submit function. +Both of these functions require two character vector value fields on the cluster's AdditionalProperties property + a. ClusterHost - The name of the cluster host that will call the sbatch command. + b. RemoteJobStorageLocation - The directory used to store job information on the cluster. + This directory must be accessible by all the worker hosts. + +Example: +% Use a local folder as the JobStorageLocation +cluster = parallel.cluster.Generic( 'JobStorageLocation', '/tmp/JOB_STORAGE_LOCATION' ); +set(cluster, 'HasSharedFilesystem', false); +set(cluster, 'IntegrationScriptsLocation', '/parallel/slurm/nonshared'); +set(cluster, 'ClusterMatlabRoot', '/apps/matlab'); +set(cluster, 'OperatingSystem', 'unix'); +% Define the additional inputs to the submit functions +cluster.AdditionalProperties.ClusterHost = 'myHost1'; +cluster.AdditionalProperties.RemoteJobStorageLocation = '/share/JOB_STORAGE_LOCATION'; + +2. Create a job and some tasks, submit the job, and wait for it to finish before +getting the results. Do the same for communicating jobs if required. + +The first time you submit a job to the cluster from any client MATLAB session +you will be prompted to provide your username and other credential information for the +cluster host that will submit the job. If you choose to use an identity file, this must +be an ssh identity file. + +If you need to change your credentials, reset them using the following command +in the MATLAB command window: + cluster.UserData.RemoteConnection.disconnect + +You will be prompted to provide your credentials again when you next submit a job. + +As an alternative to these steps, create a profile that defines the appropriate +properties and run profile validation to verify that the profile +works correctly. + + +2) Description of Files +======================= +For more detail about these files, refer to the help and comments contained in the files themselves. + +MATLAB Functions Required for generic cluster +---------------------------------------------- +independentSubmitFcn.m + Submit function for independent jobs. Used as the IndependentSubmitFcn for your generic cluster object. +communicatingSubmitFcn.m + Submit function for communicating jobs. Used as the CommunicatingSubmitFcn for your generic cluster object. +deleteJobFcn.m + Delete a job on the cluster. Used as the DeleteJobFcn for your generic cluster object. +getJobStateFcn.m + Get the job state from the cluster. Used as the GetJobStateFcn for your generic cluster object. + +Other MATLAB Functions +----------------------- +extractJobId.m + Get the cluster's job ID from the submission output. +getSubmitString.m + Get the submission string for the cluster. +createSubmitScript.m + Create a script that is executed on the cluster host to perform job submission. +getRemoteConnection.m + Get or create a parallel.cluster.RemoteClusterAccess connection. If a new connection + is created, this function prompts for user credentials either through standard + MATLAB dialogs or the command line. It stores the connection in the UserData property of + the cluster object. If you provide incorrect credentials, you can reset them + using the following code: + >> cluster.UserData.RemoteConnection.disconnect(); + + Refer to the MATLAB help for parallel.cluster.RemoteClusterAccess for more information. + +Executable Scripts +------------------- +independentJobWrapper.sh + Script used by the cluster to launch the MATLAB worker processes for independent jobs. +communicatingJobWrapper.sh + Script used by the cluster to launch the MATLAB worker processes for communicating jobs. + + +3) Optional Customizations +========================== +The code customizations listed in this section are clearly marked in the relevant files. + +independentSubmitFcn.m +---------------------- +independentSubmitFcn provides the ability to supply additional submit arguments to the +sbatch command. Modify the AdditionalProperties.AdditionalSubmitArgs +property to include additional submit arguments that are appropriate to your cluster. +For more information, refer to the sbatch documentation provided with your cluster. + +communicatingSubmitFcn.m +------------------------ +communicatingSubmitFcn calculates the number of nodes to request from the cluster from the +NumWorkersRange property of the communicating job. Customize the number of +nodes requested to suit your cluster's requirements. + +communicatingSubmitFcn provides the ability to supply additional submit arguments to the +sbatch command. Modify the AdditionalProperties.AdditionalSubmitArgs +property to include additional submit arguments that are appropriate to your cluster. +For more information, refer to the sbatch documentation provided with your cluster. + +communicatingJobWrapper.sh +-------------------------- +communicatingJobWrapper.sh uses the StrictHostKeyChecking=no and UserKnownHostsFile=/dev/null options +for ssh. Customize the ssh options to suit your cluster's requirements. For +more information, refer to your operating system's ssh documentation. + +getRemoteConnection.m +--------------------- +getRemoteConnection.m uses either standard MATLAB dialogs or command line input to request +credentials from users. You may wish to provide default credentials or modify the manner +in which the credential information is requested. diff --git a/+opencossan/+highperformancecomputing/@JobManagerSlurm/nonshared/cancelJobFcn.m b/+opencossan/+highperformancecomputing/@JobManagerSlurm/nonshared/cancelJobFcn.m new file mode 100644 index 00000000..69396037 --- /dev/null +++ b/+opencossan/+highperformancecomputing/@JobManagerSlurm/nonshared/cancelJobFcn.m @@ -0,0 +1,91 @@ +function OK = cancelJobFcn(cluster, job) +%CANCELJOBFCN Cancels a job on Slurm +% +% Set your cluster's IntegrationScriptsLocation to the parent folder of this +% function to run it when you cancel a job. + +% Copyright 2010-2018 The MathWorks, Inc. + +% Store the current filename for the errors, warnings and dctSchedulerMessages +currFilename = mfilename; +if ~isa(cluster, 'parallel.Cluster') + error('parallelexamples:GenericSLURM:SubmitFcnError', ... + 'The function %s is for use with clusters created using the parcluster command.', currFilename) +end +if cluster.HasSharedFilesystem + error('parallelexamples:GenericSLURM:NotNonSharedFileSystem', ... + 'The function %s is for use with nonshared filesystems.', currFilename) +end +% Get the information about the actual cluster used +data = cluster.getJobClusterData(job); +if isempty(data) + % This indicates that the job has not been submitted, so return true + dctSchedulerMessage(1, '%s: Job cluster data was empty for job with ID %d.', currFilename, job.ID); + OK = true; + return +end +try + clusterHost = data.RemoteHost; + remoteJobStorageLocation = data.RemoteJobStorageLocation; + makeLocationUnique = false; +catch err + ex = MException('parallelexamples:GenericSLURM:FailedToRetrieveRemoteParameters', ... + 'Failed to retrieve remote parameters from the job cluster data.'); + ex = ex.addCause(err); + throw(ex); +end +remoteConnection = getRemoteConnection(cluster, clusterHost, remoteJobStorageLocation, makeLocationUnique); +try + jobIDs = data.ClusterJobIDs; +catch err + ex = MException('parallelexamples:GenericSLURM:FailedToRetrieveJobID', ... + 'Failed to retrieve clusters''s job IDs from the job cluster data.'); + ex = ex.addCause(err); + throw(ex); +end + +% Only ask the cluster to cancel the job if it is hasn't reached a terminal +% state. +erroredJobAndCauseStrings = cell(size(jobIDs)); +jobState = job.State; +if ~(strcmp(jobState, 'finished') || strcmp(jobState, 'failed')) + % Get the cluster to delete the job + for ii = 1:length(jobIDs) + jobID = jobIDs{ii}; + commandToRun = sprintf('scancel ''%s''', jobID); + dctSchedulerMessage(4, '%s: Canceling job on cluster using command:\n\t%s.', currFilename, commandToRun); + % Keep track of all jobs that were not canceled successfully - either through + % a bad exit code or if an error was thrown. We'll report these later on. + try + % Execute the command on the remote host. + [cmdFailed, cmdOut] = remoteConnection.runCommand(commandToRun); + catch err + cmdFailed = true; + cmdOut = err.message; + end + if cmdFailed + erroredJobAndCauseStrings{ii} = sprintf('Job ID: %s\tReason: %s', jobID, strtrim(cmdOut)); + dctSchedulerMessage(1, '%s: Failed to cancel job %d on cluster. Reason:\n\t%s', currFilename, jobID, cmdOut); + end + end +end + +% Only stop mirroring if we are actually mirroring +if remoteConnection.isJobUsingConnection(job.ID) + dctSchedulerMessage(5, '%s: Stopping the mirror for job %d.', currFilename, job.ID); + try + remoteConnection.stopMirrorForJob(job); + catch err + warning('parallelexamples:GenericSLURM:FailedToStopMirrorForJob', ... + 'Failed to stop the file mirroring for job %d.\nReason: %s', ... + job.ID, err.getReport); + end +end +% Now warn about those jobs that we failed to cancel. +erroredJobAndCauseStrings = erroredJobAndCauseStrings(~cellfun(@isempty, erroredJobAndCauseStrings)); +if ~isempty(erroredJobAndCauseStrings) + warning('parallelexamples:GenericSLURM:FailedToCancelJob', ... + 'Failed to cancel the following jobs on the cluster:\n%s', ... + sprintf(' %s\n', erroredJobAndCauseStrings{:})); +end +OK = isempty(erroredJobAndCauseStrings); diff --git a/+opencossan/+highperformancecomputing/@JobManagerSlurm/nonshared/communicatingJobWrapper.sh b/+opencossan/+highperformancecomputing/@JobManagerSlurm/nonshared/communicatingJobWrapper.sh new file mode 100644 index 00000000..70028348 --- /dev/null +++ b/+opencossan/+highperformancecomputing/@JobManagerSlurm/nonshared/communicatingJobWrapper.sh @@ -0,0 +1,53 @@ +#!/bin/sh +# This wrapper script is intended to be submitted to Slurm to support +# communicating jobs. +# +# This script uses the following environment variables set by the submit MATLAB code: +# MDCE_CMR - the value of ClusterMatlabRoot (may be empty) +# MDCE_MATLAB_EXE - the MATLAB executable to use +# MDCE_MATLAB_ARGS - the MATLAB args to use +# PARALLEL_SERVER_DEBUG - used to debug problems on the cluster +# +# The following environment variables are forwarded through mpiexec: +# MDCE_DECODE_FUNCTION - the decode function to use +# MDCE_STORAGE_LOCATION - used by decode function +# MDCE_STORAGE_CONSTRUCTOR - used by decode function +# MDCE_JOB_LOCATION - used by decode function +# +# The following environment variables are set by Slurm: +# SLURM_NODELIST - list of hostnames allocated to this Slurm job + +# Copyright 2015-2018 The MathWorks, Inc. +export PATH=/usr/bin:$PATH + +# Echo the nodes that the scheduler has allocated to this job: +echo The scheduler has allocated the following nodes to this job: ${SLURM_NODELIST:?"Node list undefined"} + +# Create full path to mw_mpiexec if needed. +FULL_MPIEXEC=${MDCE_CMR:+${MDCE_CMR}/bin/}mw_mpiexec + +# Label stdout/stderr with the rank of the process +MPI_VERBOSE=-l + +# Increase the verbosity of mpiexec if PARALLEL_SERVER_DEBUG or MDCE_DEBUG (for backwards compatibility) is true +#if [ "X${PARALLEL_SERVER_DEBUG}X" = "XtrueX" ] || [ "X${MDCE_DEBUG}X" = "XtrueX" ]; then +#MPI_VERBOSE="${MPI_VERBOSE} -v -print-all-exitcodes" +#fi + +# Construct the command to run. +CMD="\"${FULL_MPIEXEC}\" ${MPI_VERBOSE} -n ${MDCE_TOTAL_TASKS} \"${MDCE_MATLAB_EXE}\" ${MDCE_MATLAB_ARGS}" + +# Echo the command so that it is shown in the output log. +echo $CMD + +# Execute the command. +eval $CMD + +MPIEXEC_EXIT_CODE=${?} +if [ ${MPIEXEC_EXIT_CODE} -eq 42 ] ; then + # Get here if user code errored out within MATLAB. Overwrite this to zero in + # this case. + echo "Overwriting MPIEXEC exit code from 42 to zero (42 indicates a user-code failure)" + MPIEXEC_EXIT_CODE=0 +fi +exit ${MPIEXEC_EXIT_CODE} diff --git a/+opencossan/+highperformancecomputing/@JobManagerSlurm/nonshared/communicatingJobWrapperSmpd.sh b/+opencossan/+highperformancecomputing/@JobManagerSlurm/nonshared/communicatingJobWrapperSmpd.sh new file mode 100644 index 00000000..117b8967 --- /dev/null +++ b/+opencossan/+highperformancecomputing/@JobManagerSlurm/nonshared/communicatingJobWrapperSmpd.sh @@ -0,0 +1,258 @@ +#!/bin/sh +# This wrapper script is intended to be submitted to Slurm to support +# communicating jobs. +# +# This script uses the following environment variables set by the submit MATLAB code: +# MDCE_CMR - the value of ClusterMatlabRoot (might be empty) +# MDCE_MATLAB_EXE - the MATLAB executable to use +# MDCE_MATLAB_ARGS - the MATLAB args to use +# +# The following environment variables are forwarded through mpiexec: +# MDCE_DECODE_FUNCTION - the decode function to use +# MDCE_STORAGE_LOCATION - used by decode function +# MDCE_STORAGE_CONSTRUCTOR - used by decode function +# MDCE_JOB_LOCATION - used by decode function + +# The following environment variables are set by Slurm +# SLURM_JOB_ID - number of nodes allocated to Slurm job +# SLURM_JOB_NUM_NODES - number of hosts allocated to Slurm job +# SLURM_JOB_NODELIST - list of hostnames allocated to Slurm job +# SLURM_TASKS_PER_NODE - list containing number of tasks allocated per host to Slurm job + +# Copyright 2015-2018 The MathWorks, Inc. + +# Users of Slurm older than v1.1.34 should uncomment the following code +# to enable mapping from old Slurm environment variables: + +# SLURM_JOB_ID=${SLURM_JOBID} +# SLURM_JOB_NUM_NODES=${SLURM_NNODES} +# SLURM_JOB_NODELIST=${SLURM_NODELIST} + +# Create full paths to mw_smpd/mw_mpiexec if needed +FULL_SMPD=${MDCE_CMR:+${MDCE_CMR}/bin/}mw_smpd +FULL_MPIEXEC=${MDCE_CMR:+${MDCE_CMR}/bin/}mw_mpiexec + +######################################################################################### +# Work out where we need to launch SMPDs given our hosts file - defines SMPD_HOSTS +chooseSmpdHosts() { + + # SLURM_JOB_NODELIST is required: the following line either echoes the value, or aborts. + echo Node file: ${SLURM_JOB_NODELIST:?"Node file undefined"} + + # SMPD_HOSTS is a single line comma separated list of hostnames: + # node136,node138,node140,node141,node142,node143,node157 + # + # Our source of information is SLURM_JOB_NODELIST in the form: + # cnode[136,138],cnode[140-43],cnode157 + # + # 'scontrol show hostname ${SLURM_JOB_NODELIST}' produces multi-line list of hostnames: + # node136 + # node138 + # node140 + # ... + # + # Pipe through "tr" to convert newlines to spaces. + + SMPD_HOSTS=`scontrol show hostname ${SLURM_JOB_NODELIST} | tr '\n', ' '` +} + +######################################################################################### +# Work out which port to use for SMPD +chooseSmpdPort() { + + # Extract the numeric part of SLURM_JOB_ID using sed to choose unique port for SMPD to run on. + # Assumes SLURM_JOB_ID starts with a number, such as: 15.slurm-server-host.domain.com + JOB_NUM=`echo ${SLURM_JOB_ID:?"SLURM_JOB_ID undefined"} | sed 's#^\([0-9][0-9]*\).*$#\1#'` + # Base smpd_port on the numeric part of the above + SMPD_PORT=`expr $JOB_NUM % 10000 + 20000` +} + +######################################################################################### +# Work out how many processes to launch - set MACHINE_ARG +# +# Inputs: +# SLURM_JOB_NUM_NODES Slurm environment variable: Number of nodes allocated to Slurm job +# +# SMPD_HOSTS Comma separated list of hostnames of nodes set by chooseSmpdHosts +# +# SLURM_TASKS_PER_NODE Slurm environment variable: Number of tasks allocated per node. +# If two or more consecutive nodes have the same task count, +# that count is followed by "(x#)" where "#" is the repetition count. +# Output: +# MACHINE_ARG Arguments to pass to mpiexec in the form: +# -hosts host1 tasks_on_host1 host2 tasks_on_host2 +# +# Example +# ------- +# Inputs: +# SLURM_JOB_NUM_NODES 7 +# SMPD_HOSTS node136,node138,node140,node141,node42,node143,node157 +# SLURM_TASKS_PER_NODE 12(x4),7,9(x2) +# Output: +# -hosts 7 node136 12 node138 12 node140 12 node141 12 node142 7 node143 9 node157 9 +# +chooseMachineArg() { + + # Transform SLURM_TASKS_PER_NODE into TASKS_PER_NODE_LIST + # + # Examples: SLURM_TASKS_PER_NODE -> TASKS_PER_NODE_LIST + # ------- -------------------- ------------------- + # Single node has 12 tasks 12 -> 12 + # Three nodes have 12 tasks 12(x3) -> 12,12,12 + # First two nodes have 7 tasks, the third has 8 tasks 7(x2),8 -> 7,7,8 + + TASKS_PER_NODE_LIST='' + # Replace commas with spaces to create space delimited list to use with for loop + LIST_FROM_SLURM=`echo ${SLURM_TASKS_PER_NODE} | sed 's/,/ /g'` + for ITEM in ${LIST_FROM_SLURM} + do + if [ `echo ${ITEM} | grep -e '^[0-9][0-9]*$' -c` -eq 1 ] ; then + # "NUM_TASKS" == "NUM_TASKS(x1)" + NUM_NODES=1 + NUM_TASKS=${ITEM} + else + # "NUM_TASKS(xNUM_NODES)" + NUM_NODES=`echo $ITEM | sed 's/^[0-9][0-9]*(x\([0-9][0-9]*\))$/\1/'` + NUM_TASKS=`echo $ITEM | sed 's/^\([0-9][0-9]*\)(x[0-9][0-9]*)$/\1/'` + fi + + # Repeat NUM_NODES iterations: append NUM_TASKS to TASKS_PER_NODE_LIST + COUNT=0 + while [ ${COUNT} -lt ${NUM_NODES} ] + do + if [ -z "${TASKS_PER_NODE_LIST}" ] ; then + # List empty, therefore adding first item to list - avoid adding comma + TASKS_PER_NODE_LIST=${NUM_TASKS} + else + # Appending to list - add a comma to delimit entries + TASKS_PER_NODE_LIST="${TASKS_PER_NODE_LIST},${NUM_TASKS}" + fi + COUNT=`expr ${COUNT} + 1` + done + done + + # Add -hosts argument at start of MACHINE_ARG + MACHINE_ARG="-hosts ${SLURM_JOB_NUM_NODES}" + + # For each hostname in SMPD_HOSTS, append ' ' to MACHINE_ARG + INDEX=0 + for HOSTNAME in ${SMPD_HOSTS} + do + INDEX=`expr ${INDEX} + 1` + # Use cut to index the '${INDEX}th' item in TASKS_PER_NODE_LIST + TASKS_PER_NODE=`echo ${TASKS_PER_NODE_LIST} | cut -f ${INDEX} -d,` + MACHINE_ARG="${MACHINE_ARG} ${HOSTNAME} ${TASKS_PER_NODE}" + done + echo "Machine args: $MACHINE_ARG" +} + +######################################################################################### +# Shut down SMPDs and exit with the exit code of the last command executed +cleanupAndExit() { + EXIT_CODE=${?} + + echo "Stopping SMPD ..." + + STOP_SMPD_CMD="srun --ntasks-per-node=1 --ntasks=${SLURM_JOB_NUM_NODES} ${FULL_SMPD} -shutdown -phrase MATLAB -port ${SMPD_PORT}" + echo $STOP_SMPD_CMD + eval $STOP_SMPD_CMD + + echo "Exiting with code: ${EXIT_CODE}" + exit ${EXIT_CODE} +} + +######################################################################################### +# Use srun to launch the SMPD daemons on each processor +launchSmpds() { + + # Launch the SMPD processes on all hosts using srun + echo "Starting SMPD on ${SMPD_HOSTS} ..." + + START_SMPD_CMD="srun --ntasks-per-node=1 --ntasks=${SLURM_JOB_NUM_NODES} ${FULL_SMPD} -phrase MATLAB -port ${SMPD_PORT} -debug 0 &" + echo $START_SMPD_CMD + eval $START_SMPD_CMD + + # Check that the SMPD processes are running on all hosts + SUCCESS=0 + NUM_ATTEMPTS=60 + ATTEMPT=1 + while [ ${ATTEMPT} -le ${NUM_ATTEMPTS} ] + do + echo "Checking that SMPD processes are running (Attempt ${ATTEMPT} of ${NUM_ATTEMPTS})" + SMPD_LAUNCHED_HOSTS="" + NUM_HOSTS_FOUND=0 + for HOST in ${SMPD_HOSTS} + do + CHECK_SMPD_CMD="${FULL_SMPD} -phrase MATLAB -port ${SMPD_PORT} -status ${HOST} > /dev/null 2>&1" + echo $CHECK_SMPD_CMD + eval $CHECK_SMPD_CMD + EXIT_CODE=${?} + if [ $EXIT_CODE -ne 0 ]; then + echo "No SMPD process running on ${HOST}" + else + echo "SMPD process found running on ${HOST}" + NUM_HOSTS_FOUND=$((NUM_HOSTS_FOUND+1)) + + # Append HOST to SMPD_LAUNCHED_HOSTS if it does not already contain it. + case "${SMPD_LAUNCHED_HOSTS}" in + *$HOST* ) ;; + * ) SMPD_LAUNCHED_HOSTS="${SMPD_LAUNCHED_HOSTS} ${HOST}" ;; + esac + fi + done + if [ ${SLURM_JOB_NUM_NODES} -eq ${NUM_HOSTS_FOUND} ] ; then + SUCCESS=1 + break + elif [ ${ATTEMPT} -ne ${NUM_ATTEMPTS} ] ; then + sleep 1 + fi + ATTEMPT=$((ATTEMPT+1)) + done + if [ $SUCCESS -ne 1 ] ; then + if [ $NUM_HOSTS_FOUND -eq 0 ] ; then + echo "No SMPD processes were found running. Aborting." + else + echo "Found SMPD processes running on only ${NUM_HOSTS_FOUND} of ${SLURM_JOB_NUM_NODES} nodes. Aborting." + echo "Hosts found: ${SMPD_LAUNCHED_HOSTS}" + fi + exit 1 + fi + echo "All SMPDs launched" +} + +######################################################################################### +runMpiexec() { + + CMD="\"${FULL_MPIEXEC}\" -smpd -phrase MATLAB -port ${SMPD_PORT} \ + -l ${MACHINE_ARG} -genvlist \ + MDCE_DECODE_FUNCTION,MDCE_STORAGE_LOCATION,MDCE_STORAGE_CONSTRUCTOR,MDCE_JOB_LOCATION,MDCE_DEBUG,MDCE_LICENSE_NUMBER,MLM_WEB_LICENSE,MLM_WEB_USER_CRED,MLM_WEB_ID \ + \"${MDCE_MATLAB_EXE}\" ${MDCE_MATLAB_ARGS}" + + # As a debug stage: echo the command ... + echo $CMD + + # ... and then execute it. + eval $CMD + + MPIEXEC_CODE=${?} + if [ ${MPIEXEC_CODE} -ne 0 ] ; then + exit ${MPIEXEC_CODE} + fi +} + +######################################################################################### +# Define the order in which we execute the stages defined above +MAIN() { + # Install a trap to ensure that SMPDs are closed if something errors or the + # job is cancelled. + trap "cleanupAndExit" 0 1 2 15 + chooseSmpdHosts + chooseSmpdPort + launchSmpds + chooseMachineArg + runMpiexec + exit 0 # Explicitly exit 0 to trigger cleanupAndExit +} + +# Call the MAIN loop +MAIN diff --git a/+opencossan/+highperformancecomputing/@JobManagerSlurm/nonshared/communicatingSubmitFcn.m b/+opencossan/+highperformancecomputing/@JobManagerSlurm/nonshared/communicatingSubmitFcn.m new file mode 100644 index 00000000..02f6f4e5 --- /dev/null +++ b/+opencossan/+highperformancecomputing/@JobManagerSlurm/nonshared/communicatingSubmitFcn.m @@ -0,0 +1,190 @@ +function communicatingSubmitFcn(cluster, job, environmentProperties) +%COMMUNICATINGSUBMITFCN Submit a communicating MATLAB job to a Slurm cluster +% +% Set your cluster's IntegrationScriptsLocation to the parent folder of this +% function to run it when you submit a communicating job. +% +% See also parallel.cluster.generic.communicatingDecodeFcn. + +% Copyright 2010-2018 The MathWorks, Inc. + +% Store the current filename for the errors, warnings and dctSchedulerMessages +currFilename = mfilename; +if ~isa(cluster, 'parallel.Cluster') + error('parallelexamples:GenericSLURM:NotClusterObject', ... + 'The function %s is for use with clusters created using the parcluster command.', currFilename) +end + +decodeFunction = 'parallel.cluster.generic.communicatingDecodeFcn'; + +if cluster.HasSharedFilesystem + error('parallelexamples:GenericSLURM:NotNonSharedFileSystem', ... + 'The function %s is for use with nonshared filesystems.', currFilename) +end + +if ~isprop(cluster.AdditionalProperties, 'ClusterHost') + error('parallelexamples:GenericSLURM:MissingAdditionalProperties', ... + 'Required field %s is missing from AdditionalProperties.', 'ClusterHost'); +end +clusterHost = cluster.AdditionalProperties.ClusterHost; +if ~isprop(cluster.AdditionalProperties, 'RemoteJobStorageLocation') + error('parallelexamples:GenericSLURM:MissingAdditionalProperties', ... + 'Required field %s is missing from AdditionalProperties.', 'RemoteJobStorageLocation'); +end +remoteJobStorageLocation = cluster.AdditionalProperties.RemoteJobStorageLocation; +if isprop(cluster.AdditionalProperties, 'UseUniqueSubfolders') + makeLocationUnique = cluster.AdditionalProperties.UseUniqueSubfolders; +else + makeLocationUnique = false; +end + +if ~strcmpi(cluster.OperatingSystem, 'unix') + error('parallelexamples:GenericSLURM:UnsupportedOS', ... + 'The function %s only supports clusters with unix OS.', currFilename) +end +if ~ischar(clusterHost) + error('parallelexamples:GenericSLURM:IncorrectArguments', ... + 'ClusterHost must be a character vector'); +end +if ~ischar(remoteJobStorageLocation) + error('parallelexamples:GenericSLURM:IncorrectArguments', ... + 'RemoteJobStorageLocation must be a character vector'); +end +if ~islogical(makeLocationUnique) + error('parallelexamples:GenericSLURM:IncorrectArguments', ... + 'UseUniqueSubfolders must be a logical scalar'); +end + +remoteConnection = getRemoteConnection(cluster, clusterHost, remoteJobStorageLocation, makeLocationUnique); + +enableDebug = 'false'; +if isprop(cluster.AdditionalProperties, 'EnableDebug') ... + && islogical(cluster.AdditionalProperties.EnableDebug) ... + && cluster.AdditionalProperties.EnableDebug + enableDebug = 'true'; +end + +% The job specific environment variables +% Remove leading and trailing whitespace from the MATLAB arguments +matlabArguments = strtrim(environmentProperties.MatlabArguments); +variables = {'MDCE_DECODE_FUNCTION', decodeFunction; ... + 'MDCE_STORAGE_CONSTRUCTOR', environmentProperties.StorageConstructor; ... + 'MDCE_JOB_LOCATION', environmentProperties.JobLocation; ... + 'MDCE_MATLAB_EXE', environmentProperties.MatlabExecutable; ... + 'MDCE_MATLAB_ARGS', matlabArguments; ... + 'PARALLEL_SERVER_DEBUG', enableDebug; ... + 'MLM_WEB_LICENSE', environmentProperties.UseMathworksHostedLicensing; ... + 'MLM_WEB_USER_CRED', environmentProperties.UserToken; ... + 'MLM_WEB_ID', environmentProperties.LicenseWebID; ... + 'MDCE_LICENSE_NUMBER', environmentProperties.LicenseNumber; ... + 'MDCE_STORAGE_LOCATION', remoteConnection.JobStorageLocation; ... + 'MDCE_CMR', cluster.ClusterMatlabRoot; ... + 'MDCE_TOTAL_TASKS', num2str(environmentProperties.NumberOfTasks); ... + 'MDCE_NUM_THREADS', num2str(cluster.NumThreads)}; +% Trim the environment variables of empty values. +nonEmptyValues = cellfun(@(x) ~isempty(strtrim(x)), variables(:,2)); +variables = variables(nonEmptyValues, :); + +% Get the correct quote and file separator for the Cluster OS. +% This check is unnecessary in this file because we explicitly +% checked that the ClusterOsType is unix. This code is an example +% of how your integration code should deal with clusters that +% can be unix or pc. +if strcmpi(cluster.OperatingSystem, 'unix') + quote = ''''; + fileSeparator = '/'; +else + quote = '"'; + fileSeparator = '\'; +end + +% The local job directory +localJobDirectory = cluster.getJobFolder(job); +% How we refer to the job directory on the cluster +remoteJobDirectory = remoteConnection.getRemoteJobLocation(job.ID, cluster.OperatingSystem); +% Specify the job wrapper script to use. +if isprop(cluster.AdditionalProperties, 'UseSmpd') && cluster.AdditionalProperties.UseSmpd + scriptName = 'communicatingJobWrapperSmpd.sh'; +else + scriptName = 'communicatingJobWrapper.sh'; +end +% The wrapper script is in the same directory as this file +dirpart = fileparts(mfilename('fullpath')); +localScript = fullfile(dirpart, scriptName); +% Copy the local wrapper script to the job directory +copyfile(localScript, localJobDirectory); + +% The command that will be executed on the remote host to run the job. +remoteScriptName = sprintf('%s%s%s', remoteJobDirectory, fileSeparator, scriptName); +quotedScriptName = sprintf('%s%s%s', quote, remoteScriptName, quote); + +% Choose a file for the output. Please note that currently, JobStorageLocation refers +% to a directory on disk, but this may change in the future. +logFile = sprintf('%s%s%s', remoteJobDirectory, fileSeparator, sprintf('Job%d.log', job.ID)); +quotedLogFile = sprintf('%s%s%s', quote, logFile, quote); + +jobName = sprintf('Job%d', job.ID); +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% CUSTOMIZATION MAY BE REQUIRED %% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% You might want to customize this section to match your cluster, +% for example to limit the number of nodes for a single job. +%additionalSubmitArgs = sprintf('--ntasks=%d --cpus-per-task=%d', environmentProperties.NumberOfTasks, cluster.NumThreads); +additionalSubmitArgs = sprintf('--ntasks=%d --tasks-per-node=%d -t %s -A %s -p %s ',environmentProperties.NumberOfTasks,cluster.AdditionalProperties.Ppn,cluster.AdditionalProperties.Time, cluster.AdditionalProperties.Aname,cluster.AdditionalProperties.Queue); +commonSubmitArgs = getCommonSubmitArgs(cluster); +if ~isempty(commonSubmitArgs) && ischar(commonSubmitArgs) + additionalSubmitArgs = strtrim([additionalSubmitArgs, ' ', commonSubmitArgs]); +end +% Create a script to submit a Slurm job - this will be created in the job directory +dctSchedulerMessage(5, '%s: Generating script for job.', currFilename); +localScriptName = tempname(localJobDirectory); +[~, scriptName] = fileparts(localScriptName); +remoteScriptLocation = sprintf('%s%s%s', remoteJobDirectory, fileSeparator, scriptName); +createSubmitScript(localScriptName, jobName, quotedLogFile, quotedScriptName, ... + variables, additionalSubmitArgs); +% Create the command to run on the remote host. +commandToRun = sprintf('sh %s', remoteScriptLocation); +dctSchedulerMessage(4, '%s: Starting mirror for job %d.', currFilename, job.ID); +% Start the mirror to copy all the job files over to the cluster +remoteConnection.startMirrorForJob(job); + +% Now ask the cluster to run the submission command +dctSchedulerMessage(4, '%s: Submitting job using command:\n\t%s', currFilename, commandToRun); +% Execute the command on the remote host. +[cmdFailed, cmdOut] = remoteConnection.runCommand(commandToRun); +if cmdFailed + % Stop the mirroring if we failed to submit the job - this will also + % remove the job files from the remote location + % Only stop mirroring if we are actually mirroring + if remoteConnection.isJobUsingConnection(job.ID) + dctSchedulerMessage(5, '%s: Stopping the mirror for job %d.', currFilename, job.ID); + try + remoteConnection.stopMirrorForJob(job); + catch err + warning('parallelexamples:GenericSLURM:FailedToStopMirrorForJob', ... + 'Failed to stop the file mirroring for job %d.\nReason: %s', ... + job.ID, err.getReport); + end + end + error('parallelexamples:GenericSLURM:FailedToSubmitJob', ... + 'Failed to submit job to Slurm using command:\n\t%s.\nReason: %s', ... + commandToRun, cmdOut); +end + +jobIDs = extractJobId(cmdOut); +% jobIDs must be a cell array +if isempty(jobIDs) + warning('parallelexamples:GenericSLURM:FailedToParseSubmissionOutput', ... + 'Failed to parse the job identifier from the submission output: "%s"', ... + cmdOut); +end +if ~iscell(jobIDs) + jobIDs = {jobIDs}; +end + +% set the cluster host, remote job storage location and job ID on the job cluster data +jobData = struct('ClusterJobIDs', {jobIDs}, ... + 'RemoteHost', clusterHost, ... + 'RemoteJobStorageLocation', remoteConnection.JobStorageLocation, ... + 'HasDoneLastMirror', false); +cluster.setJobClusterData(job, jobData); diff --git a/+opencossan/+highperformancecomputing/@JobManagerSlurm/nonshared/createSubmitScript.m b/+opencossan/+highperformancecomputing/@JobManagerSlurm/nonshared/createSubmitScript.m new file mode 100644 index 00000000..ad339588 --- /dev/null +++ b/+opencossan/+highperformancecomputing/@JobManagerSlurm/nonshared/createSubmitScript.m @@ -0,0 +1,30 @@ +function createSubmitScript(outputFilename, jobName, quotedLogFile, quotedScriptName, ... + environmentVariables, additionalSubmitArgs) +% Create a script that sets the correct environment variables and then +% executes the Slurm sbatch command. + +% Copyright 2010-2016 The MathWorks, Inc. + +% Open file in binary mode to make it cross-platform. +fid = fopen(outputFilename, 'w'); +if fid < 0 + error('parallelexamples:GenericSLURM:FileError', ... + 'Failed to open file %s for writing', outputFilename); +end + +% Specify Shell to use +fprintf(fid, '#!/bin/sh\n'); + +% Write the commands to set and export environment variables +for ii = 1:size(environmentVariables, 1) + fprintf(fid, '%s=%s\n', environmentVariables{ii,1}, environmentVariables{ii,2}); + fprintf(fid, 'export %s\n', environmentVariables{ii,1}); +end + +% Generate the command to run and write it. +commandToRun = getSubmitString(jobName, quotedLogFile, quotedScriptName, ... + additionalSubmitArgs); +fprintf(fid, '%s\n', commandToRun); + +% Close the file +fclose(fid); diff --git a/+opencossan/+highperformancecomputing/@JobManagerSlurm/nonshared/deleteJobFcn.m b/+opencossan/+highperformancecomputing/@JobManagerSlurm/nonshared/deleteJobFcn.m new file mode 100644 index 00000000..113b5a65 --- /dev/null +++ b/+opencossan/+highperformancecomputing/@JobManagerSlurm/nonshared/deleteJobFcn.m @@ -0,0 +1,8 @@ +function deleteJobFcn(cluster, job) +%DELETEJOBFCN Deletes a job on cluster +% +% Set your cluster's IntegrationScriptsLocation to the parent folder of this +% function to run it when you delete a job. + +% Copyright 2017-2018 The MathWorks, Inc. +cancelJobFcn(cluster, job); diff --git a/+opencossan/+highperformancecomputing/@JobManagerSlurm/nonshared/extractJobId.m b/+opencossan/+highperformancecomputing/@JobManagerSlurm/nonshared/extractJobId.m new file mode 100644 index 00000000..091e2cb0 --- /dev/null +++ b/+opencossan/+highperformancecomputing/@JobManagerSlurm/nonshared/extractJobId.m @@ -0,0 +1,31 @@ +function jobID = extractJobId(sbatchCommandOutput) +% Extracts the job ID from the sbatch command output for Slurm + +% Copyright 2015-2017 The MathWorks, Inc. + +% Output from sbatch expected to be in the following format: +% Submitted batch job 12345 +% +% sbatch could also attach a warning to the output, such as: +% +% sbatch: Warning: can't run 1 processes on 3 nodes, setting nnodes to 1 +% Submitted batch job 12346 + +% Trim sbatch command output for use in debug message +trimmedCommandOutput = strtrim(sbatchCommandOutput); + +% Ignore anything before or after 'Submitted batch job ###', and extract the numeric value. +searchPattern = '.*Submitted batch job ([0-9]+).*'; + +% When we match searchPattern, matchedTokens is a single entry cell array containing the jobID. +% Otherwise we failed to match searchPattern, so matchedTokens is an empty cell array. +matchedTokens = regexp(sbatchCommandOutput, searchPattern, 'tokens', 'once'); + +if isempty(matchedTokens) + % Callers check for error in extracting Job ID using isempty() on return value. + jobID = ''; + dctSchedulerMessage(0, '%s: Failed to extract Job ID from sbatch output: \n\t%s', mfilename, trimmedCommandOutput); +else + jobID = matchedTokens{1}; + dctSchedulerMessage(0, '%s: Job ID %s was extracted from sbatch output: \n\t%s', mfilename, jobID, trimmedCommandOutput); +end diff --git a/+opencossan/+highperformancecomputing/@JobManagerSlurm/nonshared/getCommonSubmitArgs.m b/+opencossan/+highperformancecomputing/@JobManagerSlurm/nonshared/getCommonSubmitArgs.m new file mode 100644 index 00000000..8938cff7 --- /dev/null +++ b/+opencossan/+highperformancecomputing/@JobManagerSlurm/nonshared/getCommonSubmitArgs.m @@ -0,0 +1,18 @@ +function commonSubmitArgs = getCommonSubmitArgs(cluster) +% Get any additional submit arguments for the Slurm sbatch command +% that are common to both independent and communicating jobs. + +% Copyright 2016-2017 The MathWorks, Inc. + +commonSubmitArgs = ''; + +% Append any arguments provided by the AdditionalSubmitArgs field of cluster.AdditionalProperties. +if isprop(cluster.AdditionalProperties, 'AdditionalSubmitArgs') + extraArgs = cluster.AdditionalProperties.AdditionalSubmitArgs; + if ~isempty(extraArgs) && ischar(extraArgs) + commonSubmitArgs = strtrim([commonSubmitArgs, ' ', extraArgs]); + end +end + +% You may wish to support further cluster.AdditionalProperties fields here +% and modify the submission command arguments accordingly. diff --git a/+opencossan/+highperformancecomputing/@JobManagerSlurm/nonshared/getJobStateFcn.m b/+opencossan/+highperformancecomputing/@JobManagerSlurm/nonshared/getJobStateFcn.m new file mode 100644 index 00000000..c0ae457a --- /dev/null +++ b/+opencossan/+highperformancecomputing/@JobManagerSlurm/nonshared/getJobStateFcn.m @@ -0,0 +1,161 @@ +function state = getJobStateFcn(cluster, job, state) +%GETJOBSTATEFCN Gets the state of a job from Slurm +% +% Set your cluster's IntegrationScriptsLocation to the parent folder of this +% function to run it when you query the state of a job. + +% Copyright 2010-2018 The MathWorks, Inc. + +% Store the current filename for the errors, warnings and dctSchedulerMessages +currFilename = mfilename; +if ~isa(cluster, 'parallel.Cluster') + error('parallelexamples:GenericSLURM:SubmitFcnError', ... + 'The function %s is for use with clusters created using the parcluster command.', currFilename) +end +if cluster.HasSharedFilesystem + error('parallelexamples:GenericSLURM:NotNonSharedFileSystem', ... + 'The function %s is for use with nonshared filesystems.', currFilename) +end + +% Get the information about the actual cluster used +data = cluster.getJobClusterData(job); +if isempty(data) + % This indicates that the job has not been submitted, so just return + dctSchedulerMessage(1, '%s: Job cluster data was empty for job with ID %d.', currFilename, job.ID); + return +end +try + hasDoneLastMirror = data.HasDoneLastMirror; +catch err + ex = MException('parallelexamples:GenericSLURM:FailedToRetrieveRemoteParameters', ... + 'Failed to retrieve remote parameters from the job cluster data.'); + ex = ex.addCause(err); + throw(ex); +end +% Shortcut if the job state is already finished or failed +jobInTerminalState = strcmp(state, 'finished') || strcmp(state, 'failed'); +% and we have already done the last mirror +if jobInTerminalState && hasDoneLastMirror + return +end +try + clusterHost = data.RemoteHost; + remoteJobStorageLocation = data.RemoteJobStorageLocation; + makeLocationUnique = false; +catch err + ex = MException('parallelexamples:GenericSLURM:FailedToRetrieveRemoteParameters', ... + 'Failed to retrieve remote parameters from the job cluster data.'); + ex = ex.addCause(err); + throw(ex); +end +remoteConnection = getRemoteConnection(cluster, clusterHost, remoteJobStorageLocation, makeLocationUnique); +try + jobIDs = data.ClusterJobIDs; +catch err + ex = MException('parallelexamples:GenericSLURM:FailedToRetrieveJobID', ... + 'Failed to retrieve clusters''s job IDs from the job cluster data.'); + ex = ex.addCause(err); + throw(ex); +end + + +% Get the top level job state from sacct +% We are not using squeue because it omits information about completed/terminated jobs. +% We use the '--allocations' option to request cumulative statistics for each job. +% (Normal output of sacct includes intermediate Slurm job steps which we don't want.) +% If the Slurm JobID counter has been reset, there is a short period of time after a job +% has been submitted that sacct may return information on an old job with the same JobID. +% The option '--user=$USER' makes this less likely, as it will only happen if the old job +% was submitted by the same user. +jobList = sprintf('-j ''%s'' ', jobIDs{:}); +commandToRun = sprintf('sacct --allocations --user=$USER %s', jobList); +dctSchedulerMessage(4, '%s: Querying cluster for job state using command:\n\t%s', currFilename, commandToRun); + +try + % We will ignore the status returned from the state command because + % a non-zero status is returned if the job no longer exists + % Execute the command on the remote host. + [~, cmdOut] = remoteConnection.runCommand(commandToRun); +catch err + ex = MException('parallelexamples:GenericSLURM:FailedToGetJobState', ... + 'Failed to get job state from cluster.'); + ex = ex.addCause(err); + throw(ex); +end + +clusterState = iExtractJobState(cmdOut, numel(jobIDs)); +dctSchedulerMessage(6, '%s: State %s was extracted from cluster output.', currFilename, clusterState); + +% If we could determine the cluster's state, we'll use that, otherwise +% stick with MATLAB's job state. +if ~strcmp(clusterState, 'unknown') + state = clusterState; +end +% Decide what to do with mirroring based on the cluster's version of job state and whether or not +% the job is currently being mirrored: +% If job is not being mirrored, and job is not finished, resume the mirror +% If job is not being mirrored, and job is finished, do the last mirror +% If the job is being mirrored, and job is finished, do the last mirror. +% Otherwise (if job is not finished, and we are mirroring), do nothing +isBeingMirrored = remoteConnection.isJobUsingConnection(job.ID); +isJobFinished = strcmp(state, 'finished') || strcmp(state, 'failed'); +if ~isBeingMirrored && ~isJobFinished + % resume the mirror + dctSchedulerMessage(4, '%s: Resuming mirror for job %d.', currFilename, job.ID); + try + remoteConnection.resumeMirrorForJob(job); + catch err + warning('parallelexamples:GenericSLURM:FailedToResumeMirrorForJob', ... + 'Failed to resume mirror for job %d. Your local job files may not be up-to-date.\nReason: %s', ... + err.getReport); + end +elseif isJobFinished + dctSchedulerMessage(4, '%s: Doing last mirror for job %d.', currFilename, job.ID); + try + remoteConnection.doLastMirrorForJob(job); + % Store the fact that we have done the last mirror so we can shortcut in the future + data.HasDoneLastMirror = true; + cluster.setJobClusterData(job, data); + catch err + warning('parallelexamples:GenericSLURM:FailedToDoFinalMirrorForJob', ... + 'Failed to do last mirror for job %d. Your local job files may not be up-to-date.\nReason: %s', ... + err.getReport); + end +end + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +function state = iExtractJobState(sacctOut, numJobs) +% Function to extract the job state from the output of sacct + +numPending = numel(regexp(sacctOut, 'PENDING')); +numRunning = numel(regexp(sacctOut, 'RUNNING|SUSPENDED|COMPLETING|CONFIGURING')); +numFinished = numel(regexp(sacctOut, 'COMPLETED')); +numFailed = numel(regexp(sacctOut, 'CANCELLED|FAILED|TIMEOUT|PREEMPTED|NODE_FAIL')); + +% If all of the jobs that we asked about have finished, then we know the job has finished. +if numFinished == numJobs + state = 'finished'; + return +end + +% Any running indicates that the job is running +if numRunning > 0 + state = 'running'; + return +end + +% We know numRunning == 0 so if there are some still pending then the +% job must be queued again, even if there are some finished +if numPending > 0 + state = 'queued'; + return +end + +% Deal with any tasks that have failed +if numFailed > 0 + % Set this job to be failed + state = 'failed'; + return +end + +state = 'unknown'; diff --git a/+opencossan/+highperformancecomputing/@JobManagerSlurm/nonshared/getRemoteConnection.m b/+opencossan/+highperformancecomputing/@JobManagerSlurm/nonshared/getRemoteConnection.m new file mode 100644 index 00000000..034a1799 --- /dev/null +++ b/+opencossan/+highperformancecomputing/@JobManagerSlurm/nonshared/getRemoteConnection.m @@ -0,0 +1,220 @@ +function remoteConnection = getRemoteConnection(cluster, clusterHost, remoteJobStorageLocation, makeLocationUnique) +%GETREMOTECONNECTION Get a connected RemoteClusterAccess +% +% getRemoteConnection will either retrieve a RemoteClusterAccess from the +% cluster's UserData or it will create a new RemoteClusterAccess. + +% Copyright 2010-2017 The MathWorks, Inc. + +% Store the current filename for the dctSchedulerMessages +currFilename = mfilename; + +if ~ischar(clusterHost) + error('parallelexamples:GenericSLURM:IncorrectArguments', ... + 'Hostname must be a string'); +end +if ~ischar(remoteJobStorageLocation) + error('parallelexamples:GenericSLURM:RemoteJobStorageLocationType', ... + 'Remote Job Storage Location must be a character vector'); +end + +needToCreateNewConnection = false; +if isempty(cluster.UserData) + needToCreateNewConnection = true; +else + if ~isstruct(cluster.UserData) + error('parallelexamples:GenericSLURM:IncorrectUserData', ... + ['Failed to retrieve remote connection from cluster''s UserData.\n' ... + 'Expected cluster''s UserData to be a structure, but found %s'], ... + class(cluster.UserData)); + end + + if isfield(cluster.UserData, 'RemoteConnection') + % Get the remote connection out of the cluster user data + remoteConnection = cluster.UserData.RemoteConnection; + + % And check it is of the type that we expect + if isempty(remoteConnection) + needToCreateNewConnection = true; + else + clusterAccessClassname = 'parallel.cluster.RemoteClusterAccess'; + if ~isa(remoteConnection, clusterAccessClassname) + error('parallelexamples:GenericSLURM:IncorrectArguments', ... + ['Failed to retrieve remote connection from cluster''s UserData.\n' ... + 'Expected the RemoteConnection field of the UserData to contain an object of type %s, but found %s.'], ... + clusterAccessClassname, class(remoteConnection)); + end + + if makeLocationUnique + username = remoteConnection.Username; + expectedRemoteJobStorageLocation = iBuildUniqueSubfolder(remoteJobStorageLocation, ... + username, iGetFileSeparator(cluster)); + else + expectedRemoteJobStorageLocation = remoteJobStorageLocation; + end + + if ~remoteConnection.IsConnected + needToCreateNewConnection = true; + elseif ~(strcmpi(remoteConnection.Hostname, clusterHost) && ... + remoteConnection.IsFileMirrorSupported && ... + strcmpi(remoteConnection.JobStorageLocation, expectedRemoteJobStorageLocation)) + % The connection stored in the user data does not match the cluster host + % and remote location requested + warning('parallelexamples:GenericSLURM:DifferentRemoteParameters', ... + ['The current cluster is already using cluster host %s and remote job storage location %s.\n', ... + 'The existing connection to %s will be replaced.'], ... + remoteConnection.Hostname, remoteConnection.JobStorageLocation, remoteConnection.Hostname); + cluster.UserData.RemoteConnection = []; + needToCreateNewConnection = true; + end + end + else + needToCreateNewConnection = true; + end +end + +if ~needToCreateNewConnection + return +end + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% CUSTOMIZATION MAY BE REQUIRED %% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% Get the credential options from the user using simple +% MATLAB dialogs or command line input. You should change +% this section if you wish for users to provide their credential +% options in a different way. +% The pertinent options are: +% username - the username to use when running commands on the remote host +% useIdentityFile - whether or not to use an identity file (true/false). +% False means that a password is used +% identityFilename - the full path to the identity file +% fileHasPassphrase - whether or not the identity file requires a passphrase +% (true/false). +%if isempty(javachk('awt')) + % MATLAB has been started with a desktop, so use dialogs to get credential data. +% [username, useIdentityFile, identityFilename, fileHasPassphrase] = iGetCredentialsFromUI(clusterHost); +%else + % MATLAB has been started in nodisplay mode, so use command line to get credential data + %[username, useIdentityFile, identityFilename, fileHasPassphrase] = iGetCredentialsFromCommandLine(clusterHost); + username=cluster.AdditionalProperties.User + fileHasPassphrase=false + identityFilename=cluster.AdditionalProperties.Keyfile +%end + +% Establish a new connection +%if useIdentityFile +% dctSchedulerMessage(1, '%s: Identity file %s will be used for remote connections', ... +% currFilename, username, identityFilename); + userArgs = {username, ... + 'IdentityFilename', identityFilename, 'IdentityFileHasPassphrase', fileHasPassphrase}; +%else +% userArgs = {username}; +%end + +% Now connect and store the connection +dctSchedulerMessage(1, '%s: Connecting to remote host %s', ... + currFilename, clusterHost); +if makeLocationUnique + remoteJobStorageLocation = iBuildUniqueSubfolder(remoteJobStorageLocation, ... + username, iGetFileSeparator(cluster)); +end +remoteConnection = parallel.cluster.RemoteClusterAccess.getConnectedAccessWithMirror(clusterHost, remoteJobStorageLocation, userArgs{:}); +dctSchedulerMessage(5, '%s: Storing remote connection in cluster''s user data.', currFilename); +cluster.UserData.RemoteConnection = remoteConnection; + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +function [username, useIdentityFile, identityFilename, fileHasPassphrase] = iGetCredentialsFromUI(clusterHost) +% Function to get the user credentials using dialogs +identityFilename = ''; +fileHasPassphrase = false; + +%dlgTitle = 'User Credentials'; +%numlines = 1; +%dlgMessage = sprintf('Enter the username for %s', clusterHost); +%usernameResponse = inputdlg(dlgMessage, dlgTitle, numlines); +% Hitting cancel gives an empty cell array, but a user providing an empty string gives +% a (non-empty) cell array containing an empty string +%if isempty(usernameResponse) + % User hit cancel +% error('parallelexamples:GenericSLURM:UserCancelledOperation', 'User cancelled operation.'); +%end +%username = char(usernameResponse); + +%dlgMessage = sprintf('Use an identity file to login to %s?', clusterHost); +%identityFileResponse = questdlg(dlgMessage, dlgTitle); +%if strcmp(identityFileResponse, 'Cancel') +% % User hit cancel +% error('parallelexamples:GenericSLURM:UserCancelledOperation', 'User cancelled operation.'); +%end + +%useIdentityFile = strcmp(identityFileResponse, 'Yes'); +%if ~useIdentityFile +% return +%end + +%dlgMessage = 'Select Identity File to use'; +%[filename, pathname] = uigetfile({'*.*', 'All Files (*.*)'}, dlgMessage); +% If the user hit cancel, then filename and pathname will both be 0. +%if isequal(filename, 0) && isequal(pathname,0) +% error('parallelexamples:GenericSLURM:UserCancelledOperation', 'User cancelled operation.'); +%end + +%identityFilename = fullfile(pathname, filename); +%dlgMessage = 'Does the identity file require a password?'; +%passphraseResponse = questdlg(dlgMessage, dlgTitle); +%if strcmp(passphraseResponse, 'Cancel') +% % User hit cancel +% error('parallelexamples:GenericSLURM:UserCancelledOperation', 'User cancelled operation.'); +%end +%fileHasPassphrase = strcmp(passphraseResponse, 'Yes'); + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +function [username, useIdentityFile, identityFilename, fileHasPassphrase] = iGetCredentialsFromCommandLine(clusterHost) +% Function to get the user credentials from the command line +identityFilename = ''; +fileHasPassphrase = false; +validYesNoResponse = {'y', 'n'}; + +% Allow the username to be empty +username = input(sprintf('Enter the username for %s:\n', clusterHost), 's'); + +identityFileMessage = sprintf('Use an identity file to login to %s? (y or n)\n', clusterHost); +identityFileResponse = iLoopUntilValidStringInput(identityFileMessage, validYesNoResponse); +useIdentityFile = strcmpi(identityFileResponse, 'y'); +if ~useIdentityFile + return +end + +%identityFilename = ''; +%while isempty(identityFilename) +% identityFilename = input(sprintf('Please enter the full path to the Identity File to use:\n'), 's'); +%end + +%passphraseMessage = 'Does the identity file require a password? (y or n)\n'; +%passphraseResponse = iLoopUntilValidStringInput(passphraseMessage, validYesNoResponse); +%fileHasPassphrase = strcmpi(passphraseResponse, 'y'); + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +function returnValue = iLoopUntilValidStringInput(message, validValues) +% Function to loop until a valid response is obtained user input +returnValue = ''; + +while isempty(returnValue) || ~any(strcmpi(returnValue, validValues)) + returnValue = input(message, 's'); +end + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +function subfolder = iBuildUniqueSubfolder(remoteJobStorageLocation, username, fileSeparator) +% Function to build unique location using username and MATLAB release version +release = ['R' version('-release')]; +subfolder = [remoteJobStorageLocation fileSeparator username fileSeparator release]; + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +function fileSeparator = iGetFileSeparator(cluster) +% Function to return file separator for cluster operating system +if strcmpi(cluster.OperatingSystem, 'unix') + fileSeparator = '/'; +else + fileSeparator = '\'; +end diff --git a/+opencossan/+highperformancecomputing/@JobManagerSlurm/nonshared/getSubmitString.m b/+opencossan/+highperformancecomputing/@JobManagerSlurm/nonshared/getSubmitString.m new file mode 100644 index 00000000..a1520e27 --- /dev/null +++ b/+opencossan/+highperformancecomputing/@JobManagerSlurm/nonshared/getSubmitString.m @@ -0,0 +1,7 @@ +function submitString = getSubmitString(jobName, quotedLogFile, quotedCommand, additionalSubmitArgs) +%GETSUBMITSTRING Gets the correct sbatch command for a Slurm cluster + +% Copyright 2010-2016 The MathWorks, Inc. + +submitString = sprintf('sbatch --job-name=%s --output=%s %s %s', ... + jobName, quotedLogFile, additionalSubmitArgs, quotedCommand); diff --git a/+opencossan/+highperformancecomputing/@JobManagerSlurm/nonshared/independentJobWrapper.sh b/+opencossan/+highperformancecomputing/@JobManagerSlurm/nonshared/independentJobWrapper.sh new file mode 100644 index 00000000..e487e6a5 --- /dev/null +++ b/+opencossan/+highperformancecomputing/@JobManagerSlurm/nonshared/independentJobWrapper.sh @@ -0,0 +1,15 @@ +#!/bin/sh +# This wrapper script is intended to support independent execution. +# +# This script uses the following environment variables set by the submit MATLAB code: +# MDCE_MATLAB_EXE - the MATLAB executable to use +# MDCE_MATLAB_ARGS - the MATLAB args to use +# + +# Copyright 2010-2018 The MathWorks, Inc. + +echo "Executing: ${MDCE_MATLAB_EXE} ${MDCE_MATLAB_ARGS}" +eval "${MDCE_MATLAB_EXE}" ${MDCE_MATLAB_ARGS} +EXIT_CODE=${?} +echo "Exiting with code: ${EXIT_CODE}" +exit ${EXIT_CODE} diff --git a/+opencossan/+highperformancecomputing/@JobManagerSlurm/nonshared/independentSubmitFcn.m b/+opencossan/+highperformancecomputing/@JobManagerSlurm/nonshared/independentSubmitFcn.m new file mode 100644 index 00000000..71964847 --- /dev/null +++ b/+opencossan/+highperformancecomputing/@JobManagerSlurm/nonshared/independentSubmitFcn.m @@ -0,0 +1,195 @@ +function independentSubmitFcn(cluster, job, environmentProperties) +%INDEPENDENTSUBMITFCN Submit a MATLAB job to a Slurm cluster +% +% Set your cluster's IntegrationScriptsLocation to the parent folder of this +% function to run it when you submit an independent job. +% +% See also parallel.cluster.generic.independentDecodeFcn. + +% Copyright 2010-2018 The MathWorks, Inc. + +% Store the current filename for the errors, warnings and dctSchedulerMessages +currFilename = mfilename; +if ~isa(cluster, 'parallel.Cluster') + error('parallelexamples:GenericSLURM:NotClusterObject', ... + 'The function %s is for use with clusters created using the parcluster command.', currFilename) +end + +decodeFunction = 'parallel.cluster.generic.independentDecodeFcn'; + +if cluster.HasSharedFilesystem + error('parallelexamples:GenericSLURM:NotNonSharedFileSystem', ... + 'The function %s is for use with nonshared filesystems.', currFilename) +end + +if ~isprop(cluster.AdditionalProperties, 'ClusterHost') + error('parallelexamples:GenericSLURM:MissingAdditionalProperties', ... + 'Required field %s is missing from AdditionalProperties.', 'ClusterHost'); +end +clusterHost = cluster.AdditionalProperties.ClusterHost; +if ~isprop(cluster.AdditionalProperties, 'RemoteJobStorageLocation') + error('parallelexamples:GenericSLURM:MissingAdditionalProperties', ... + 'Required field %s is missing from AdditionalProperties.', 'RemoteJobStorageLocation'); +end +remoteJobStorageLocation = cluster.AdditionalProperties.RemoteJobStorageLocation; +if isprop(cluster.AdditionalProperties, 'UseUniqueSubfolders') + makeLocationUnique = cluster.AdditionalProperties.UseUniqueSubfolders; +else + makeLocationUnique = false; +end + +if ~strcmpi(cluster.OperatingSystem, 'unix') + error('parallelexamples:GenericSLURM:UnsupportedOS', ... + 'The function %s only supports clusters with unix OS.', currFilename) +end +if ~ischar(clusterHost) + error('parallelexamples:GenericSLURM:IncorrectArguments', ... + 'ClusterHost must be a character vector'); +end +if ~ischar(remoteJobStorageLocation) + error('parallelexamples:GenericSLURM:IncorrectArguments', ... + 'RemoteJobStorageLocation must be a character vector'); +end +if ~islogical(makeLocationUnique) + error('parallelexamples:GenericSLURM:IncorrectArguments', ... + 'UseUniqueSubfolders must be a logical scalar'); +end + +remoteConnection = getRemoteConnection(cluster, clusterHost, remoteJobStorageLocation, makeLocationUnique); + +enableDebug = 'false'; +if isprop(cluster.AdditionalProperties, 'EnableDebug') ... + && islogical(cluster.AdditionalProperties.EnableDebug) ... + && cluster.AdditionalProperties.EnableDebug + enableDebug = 'true'; +end + +% The job specific environment variables +% Remove leading and trailing whitespace from the MATLAB arguments +matlabArguments = strtrim(environmentProperties.MatlabArguments); +variables = {'MDCE_DECODE_FUNCTION', decodeFunction; ... + 'MDCE_STORAGE_CONSTRUCTOR', environmentProperties.StorageConstructor; ... + 'MDCE_JOB_LOCATION', environmentProperties.JobLocation; ... + 'MDCE_MATLAB_EXE', environmentProperties.MatlabExecutable; ... + 'MDCE_MATLAB_ARGS', matlabArguments; ... + 'PARALLEL_SERVER_DEBUG', enableDebug; ... + 'MLM_WEB_LICENSE', environmentProperties.UseMathworksHostedLicensing; ... + 'MLM_WEB_USER_CRED', environmentProperties.UserToken; ... + 'MLM_WEB_ID', environmentProperties.LicenseWebID; ... + 'MDCE_LICENSE_NUMBER', environmentProperties.LicenseNumber; ... + 'MDCE_STORAGE_LOCATION', remoteConnection.JobStorageLocation}; +% Trim the environment variables of empty values. +nonEmptyValues = cellfun(@(x) ~isempty(strtrim(x)), variables(:,2)); +variables = variables(nonEmptyValues, :); + +% Get the correct quote and file separator for the Cluster OS. +% This check is unnecessary in this file because we explicitly +% checked that the ClusterOsType is unix. This code is an example +% of how your integration code should deal with clusters that +% can be unix or pc. +if strcmpi(cluster.OperatingSystem, 'unix') + quote = ''''; + fileSeparator = '/'; +else + quote = '"'; + fileSeparator = '\'; +end + +% The local job directory +localJobDirectory = cluster.getJobFolder(job); +% How we refer to the job directory on the cluster +remoteJobDirectory = remoteConnection.getRemoteJobLocation(job.ID, cluster.OperatingSystem); + +% The script name is independentJobWrapper.sh +scriptName = 'independentJobWrapper.sh'; +% The wrapper script is in the same directory as this file +dirpart = fileparts(mfilename('fullpath')); +localScript = fullfile(dirpart, scriptName); +% Copy the local wrapper script to the job directory +copyfile(localScript, localJobDirectory); + +% The command that will be executed on the remote host to run the job. +remoteScriptName = sprintf('%s%s%s', remoteJobDirectory, fileSeparator, scriptName); +quotedScriptName = sprintf('%s%s%s', quote, remoteScriptName, quote); + +% Get the tasks for use in the loop +tasks = job.Tasks; +numberOfTasks = environmentProperties.NumberOfTasks; +jobIDs = cell(numberOfTasks, 1); +commandsToRun = cell(numberOfTasks, 1); +% Loop over every task we have been asked to submit +for ii = 1:numberOfTasks + taskLocation = environmentProperties.TaskLocations{ii}; + % Add the task location to the environment variables + environmentVariables = [variables; ... + {'MDCE_TASK_LOCATION', taskLocation}]; + + % Choose a file for the output. Please note that currently, JobStorageLocation refers + % to a directory on disk, but this may change in the future. + logFile = sprintf('%s%s%s', remoteJobDirectory, fileSeparator, sprintf('Task%d.log', tasks(ii).ID)); + quotedLogFile = sprintf('%s%s%s', quote, logFile, quote); + + % Submit one task at a time + jobName = sprintf('Job%d.%d', job.ID, tasks(ii).ID); + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + %% CUSTOMIZATION MAY BE REQUIRED %% + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% additionalSubmitArgs = sprintf('--ntasks=1 --cpus-per-task=%d', cluster.NumThreads); + additionalSubmitArgs = sprintf('--ntasks=1 -t %s -A %s -p %s ',cluster.AdditionalProperties.Time, cluster.AdditionalProperties.Aname,cluster.AdditionalProperties.Queue); + commonSubmitArgs = getCommonSubmitArgs(cluster); + if ~isempty(commonSubmitArgs) && ischar(commonSubmitArgs) + additionalSubmitArgs = strtrim([additionalSubmitArgs, ' ', commonSubmitArgs]); + end + % Create a script to submit a Slurm job - this will be created in the job directory + dctSchedulerMessage(5, '%s: Generating script for task %i', currFilename, ii); + localScriptName = tempname(localJobDirectory); + [~, scriptName] = fileparts(localScriptName); + remoteScriptLocation = sprintf('%s%s%s', remoteJobDirectory, fileSeparator, scriptName); + createSubmitScript(localScriptName, jobName, quotedLogFile, quotedScriptName, ... + environmentVariables, additionalSubmitArgs); + % Create the command to run on the remote host. + commandsToRun{ii} = sprintf('sh %s', remoteScriptLocation); +end +dctSchedulerMessage(4, '%s: Starting mirror for job %d.', currFilename, job.ID); +% Start the mirror to copy all the job files over to the cluster +remoteConnection.startMirrorForJob(job); +for ii = 1:numberOfTasks + commandToRun = commandsToRun{ii}; + + % Now ask the cluster to run the submission command + dctSchedulerMessage(4, '%s: Submitting job using command:\n\t%s', currFilename, commandToRun); + % Execute the command on the remote host. + [cmdFailed, cmdOut] = remoteConnection.runCommand(commandToRun); + if cmdFailed + % Stop the mirroring if we failed to submit the job - this will also + % remove the job files from the remote location + % Only stop mirroring if we are actually mirroring + if remoteConnection.isJobUsingConnection(job.ID) + dctSchedulerMessage(5, '%s: Stopping the mirror for job %d.', currFilename, job.ID); + try + remoteConnection.stopMirrorForJob(job); + catch err + warning('parallelexamples:GenericSLURM:FailedToStopMirrorForJob', ... + 'Failed to stop the file mirroring for job %d.\nReason: %s', ... + job.ID, err.getReport); + end + end + error('parallelexamples:GenericSLURM:FailedToSubmitJob', ... + 'Failed to submit job to Slurm using command:\n\t%s.\nReason: %s', ... + commandToRun, cmdOut); + end + jobIDs{ii} = extractJobId(cmdOut); + + if isempty(jobIDs{ii}) + warning('parallelexamples:GenericSLURM:FailedToParseSubmissionOutput', ... + 'Failed to parse the job identifier from the submission output: "%s"', ... + cmdOut); + end +end + +% set the cluster host, remote job storage location and job ID on the job cluster data +jobData = struct('ClusterJobIDs', {jobIDs}, ... + 'RemoteHost', clusterHost, ... + 'RemoteJobStorageLocation', remoteConnection.JobStorageLocation, ... + 'HasDoneLastMirror', false); +cluster.setJobClusterData(job, jobData); diff --git a/+opencossan/+highperformancecomputing/@JobManagerSlurm/prepareScript.m b/+opencossan/+highperformancecomputing/@JobManagerSlurm/prepareScript.m new file mode 100644 index 00000000..5da998a2 --- /dev/null +++ b/+opencossan/+highperformancecomputing/@JobManagerSlurm/prepareScript.m @@ -0,0 +1,131 @@ +function Sscriptname = prepareScript(obj,taskNumber) +% This method creates a job script for Slurm + +% OpenCossan.getCossanWorkingPath: working path (this should be folder where all +% the final analysis are stored). It should be on the local machine if +% SSHconnection is used otherwise is a shared folder +% +% SSHConnection.SremoteWorkFolder --> folder on the head node (shared?) +% +% JobManager.Sworkingdirectory --> This should be a local working folder (not +% necessary a shared folder) +% +% TODO: add -M valid email and -m be (the email are not sent by the cluster, +% yet). +% +% See Also: +% http://cossan.co.uk/wiki/index.php/prepareGridEngineScript@JobManager +% +% $Copyright~1993-2014,~COSSAN~Working~Group,~University~of~Liverpool,~UK$ +% $Author: Edoardo Patelli$ + + +% Preparing job script +if ~exist('NsimulationNumber','var') + Sscriptname = [obj.SjobScriptName '.sh']; +else + Sscriptname = [obj.SjobScriptName num2str(taskNumber) '.sh']; +end + +Xssh = OpenCossan.getSSHConnection; % do not change this! +% create a new ASCII file +[Nfid, Serror] = fopen(fullfile(OpenCossan.getCossanWorkingPath,Sscriptname),'w'); + +assert(isempty(Serror),'openCOSSAN:JobManager:submit','Error: %s\n Path: %s', ... + Serror,fullfile(OpenCossan.getCossanWorkingPath,Sscriptname)) + +OpenCossan.cossanDisp(['Open file ' fullfile(OpenCossan.getCossanWorkingPath,Sscriptname)],3); + + +if strcmp(obj.Sworkingdirectory,'./') || strcmp(obj.Sworkingdirectory,'.') + % Use current directort + obj.Sworkingdirectory=[]; +end + +% define shell +fprintf(Nfid,'%s\n','#!/bin/bash'); +fprintf(Nfid,'%s\n','#$ -S /bin/bash'); + +% use current working directory +fprintf(Nfid,'%s\n','#$ -cwd '); + +if ~isempty(obj.Squeue) + % select queue + fprintf(Nfid,'%s\n',['#$ -q ' obj.Squeue]); +end + +if ~isempty(obj.Shostname) + % select hostname + fprintf(Nfid,'%s\n',['#$ -l hostname=' obj.Shostname]); +end % define shell + +if ~isempty(obj.SparallelEnvironment) + % select parallel environment and slots + fprintf(Nfid,'%s\n',['#$ -pe ' obj.SparallelEnvironment... + ' ' num2str(obj.Nslots)]); +end + +% Save std error in the cwd +fprintf(Nfid,'%s\n',['#$ -e ' obj.Sfoldername '.err' ]); +% Save std output in the cwd +fprintf(Nfid,'%s\n',['#$ -o ' obj.Sfoldername '.out']); + +% get the cwd +fprintf(Nfid,'%s\n','START_DIR=`pwd` '); + +%% create subfolder and copy input files +if ~isempty(obj.Sfoldername) + if ~OpenCossan.hasSSHConnection + % change directory to the job work folder + fprintf(Nfid,'%s\n',['cd ' fullfileunix(obj.Sworkingdirectory,obj.Sfoldername) ';']); + else + % change directory to the job work folder on the cluster + fprintf(Nfid,'%s\n',['cd ' fullfileunix(Xssh.SremoteWorkFolder ,obj.Sfoldername) ';']); + end +end + +fprintf(Nfid,'%s\n','echo Script execution started; date'); + + +%% Add MCR Pre-execution command +if ~isempty(obj.Xjobmanagerinterface.SMCRpreexec) + fprintf(Nfid,'%s\n',obj.Xjobmanagerinterface.SMCRpreexec); +end + +%% Add preprocessor cmd +if ~isempty(obj.Spreexecmd) + fprintf(Nfid,'%s\n',obj.Spreexecmd); +end + +%% Write hostname on the out file +fprintf(Nfid,'%s\n','hostname'); +%% Main code +fprintf(Nfid,'%s\n',[obj.Sexecmd ' ' obj.Sexeflags]); + +%% Add Postprocessor +if ~isempty(obj.Spostexecmd) + fprintf(Nfid,'%s\n',obj.Spostexecmd); +end +%% Add MCR Post-execution command +if ~isempty(obj.Xjobmanagerinterface.SMCRpostexec) + fprintf(Nfid,'%s\n',obj.Xjobmanagerinterface.SMCRpostexec); +end + +fprintf(Nfid,'%s\n','echo Script execution finished; date'); + +%% move results to the old directory +if ~isempty(obj.Sfoldername) + % move stdout and stderr to the Sfoldername + if ~OpenCossan.hasSSHConnection + fprintf(Nfid,'%s\n',['mv $START_DIR/' obj.Sfoldername '.err ' fullfileunix(obj.Sworkingdirectory,obj.Sfoldername)]); + fprintf(Nfid,'%s\n',['mv $START_DIR/' obj.Sfoldername '.out ' fullfileunix(obj.Sworkingdirectory,obj.Sfoldername)]); + else + fprintf(Nfid,'%s\n',['mv $START_DIR/' obj.Sfoldername '.err ' fullfileunix(Xssh.SremoteWorkFolder,obj.Sfoldername)]); + fprintf(Nfid,'%s\n',['mv $START_DIR/' obj.Sfoldername '.out ' fullfileunix(Xssh.SremoteWorkFolder,obj.Sfoldername)]); + end +end + +fclose(Nfid); + +end + diff --git a/+opencossan/+highperformancecomputing/@JobManagerSlurm/remote/README b/+opencossan/+highperformancecomputing/@JobManagerSlurm/remote/README new file mode 100644 index 00000000..e036b6dc --- /dev/null +++ b/+opencossan/+highperformancecomputing/@JobManagerSlurm/remote/README @@ -0,0 +1,144 @@ +Copyright 2010-2016 The MathWorks, Inc. + +MATLAB and Simulink are registered trademarks of The MathWorks, Inc. +See www.mathworks.com/trademarks for a list of additional trademarks. +Other product or brand names may be trademarks or registered trademarks of their respective holders. + +This folder contains a number of files to allow Parallel Computing Toolbox +to be used with Slurm via the generic cluster interface. + +The files in this folder assume that the client and cluster share a file system +and that the client is not able to submit directly to the cluster using the +sbatch command. + +Note that all the files in this directory will work only for clusters that are +running on UNIX. + +1) Instructions for Use +======================= +On the Slurm cluster +-------------------- +Enable job accounting on the Slurm cluster. +This allows the sacct command to run. +The scripts use the sacct output to track the state of Slurm jobs. + +Read the documentation for using the generic cluster interface with +the Parallel Computing Toolbox and familiarize yourself with the different +properties that can be set for a generic cluster. + +In the MATLAB Client +-------------------- +1. Create a generic cluster object for your cluster. Set the IntegrationScriptsLocation +to this folder. For independent jobs, independentSubmitFcn is used as your +submit function. For communicating jobs, communicatingSubmitFcn is used as +your submit function. +Both of these functions require one character vector value field on the cluster's AdditionalProperties property + a. ClusterHost - The name of the cluster host that will call the sbatch command. + +Example: +% Use a folder that both the client and cluster can access +% as the JobStorageLocation. If your cluster and client use +% different operating systems, you should specify JobStorageLocation +% to be a structure. Refer to the documentation on +% generic cluster for more information. +cluster = parallel.cluster.Generic('JobStorageLocation', '/home/JOB_STORAGE_LOCATION'); +set(cluster, 'HasSharedFilesystem', true); +set(cluster, 'IntegrationScriptsLocation', '/parallel/slurm/remote'); +set(cluster, 'ClusterMatlabRoot', '/apps/matlab'); +set(cluster, 'OperatingSystem', 'unix'); +% Define the cluster host that will be used for job submission +cluster.AdditionalProperties.ClusterHost = 'myHost1'; + +2. Create a job and some tasks, submit the job, and wait for it to finish before +getting the results. Do the same for communicating jobs if required. + +The first time you submit a job to the cluster from any client MATLAB session +you will be prompted to provide your username and other credential information for the +cluster host that will submit the job. If you choose to use an identity file, this must +be an ssh identity file. + +If you need to change your credentials, reset them using the following command +in the MATLAB command window: + cluster.UserData.RemoteConnection.disconnect + +You will be prompted to provide your credentials again when you next submit a job. + +As an alternative to these steps, create a profile that defines the appropriate +properties and run profile validation to verify that the profile +works correctly. + + +2) Description of Files +======================= +For more detail about these files, refer to the help and comments contained in the files themselves. + +MATLAB Functions Required for generic cluster +---------------------------------------------- +independentSubmitFcn.m + Submit function for independent jobs. Used as the IndependentSubmitFcn for your generic cluster object. +communicatingSubmitFcn.m + Submit function for communicating jobs. Used as the CommunicatingSubmitFcn for your generic cluster object. +deleteJobFcn.m + Delete a job on the cluster. Used as the DeleteJobFcn for your generic cluster object. +getJobStateFcn.m + Get the job state from the cluster. Used as the GetJobStateFcn for your generic cluster object. + +Other MATLAB Functions +----------------------- +extractJobId.m + Get the cluster's job ID from the submission output. +getSubmitString.m + Get the submission string for the cluster. +createSubmitScript.m + Create a script that is executed on the cluster host to perform job submission. +getRemoteConnection.m + Get or create a parallel.cluster.RemoteClusterAccess connection. If a new connection + is created, this function prompts for user credentials either through standard + MATLAB dialogs or the command line. It stores the connection in the UserData property of + the cluster object. If you provide incorrect credentials, you can reset them + using the following code: + >> cluster.UserData.RemoteConnection.disconnect(); + + Refer to the MATLAB help for parallel.cluster.RemoteClusterAccess for more information. + +Executable Scripts +------------------- +independentJobWrapper.sh + Script used by the cluster to launch the MATLAB worker processes for independent jobs. +communicatingJobWrapper.sh + Script used by the cluster to launch the MATLAB worker processes for communicating jobs. + + +3) Optional Customizations +========================== +The code customizations listed in this section are clearly marked in the relevant files. + +independentSubmitFcn.m +---------------------- +independentSubmitFcn provides the ability to supply additional submit arguments to the +sbatch command. Modify the AdditionalProperties.AdditionalSubmitArgs +property to include additional submit arguments that are appropriate to your cluster. +For more information, refer to the sbatch documentation provided with your cluster. + +communicatingSubmitFcn.m +------------------------ +communicatingSubmitFcn calculates the number of nodes to request from the cluster from the +NumWorkersRange property of the communicating job. Customize the number of +nodes requested to suit your cluster's requirements. + +communicatingSubmitFcn provides the ability to supply additional submit arguments to the +sbatch command. Modify the AdditionalProperties.AdditionalSubmitArgs +property to include additional submit arguments that are appropriate to your cluster. +For more information, refer to the sbatch documentation provided with your cluster. + +communicatingJobWrapper.sh +-------------------------- +communicatingJobWrapper.sh uses the StrictHostKeyChecking=no and UserKnownHostsFile=/dev/null options +for ssh. Customize the ssh options to suit your cluster's requirements. For +more information, refer to your operating system's ssh documentation. + +getRemoteConnection.m +--------------------- +getRemoteConnection.m uses either standard MATLAB dialogs or command line input to request +credentials from users. You may wish to provide default credentials or modify the manner +in which the credential information is requested. diff --git a/+opencossan/+highperformancecomputing/@JobManagerSlurm/remote/cancelJobFcn.m b/+opencossan/+highperformancecomputing/@JobManagerSlurm/remote/cancelJobFcn.m new file mode 100644 index 00000000..6f457873 --- /dev/null +++ b/+opencossan/+highperformancecomputing/@JobManagerSlurm/remote/cancelJobFcn.m @@ -0,0 +1,78 @@ +function OK = cancelJobFcn(cluster, job) +%CANCELJOBFCN Cancels a job on Slurm +% +% Set your cluster's IntegrationScriptsLocation to the parent folder of this +% function to run it when you cancel a job. + +% Copyright 2010-2017 The MathWorks, Inc. + +% Store the current filename for the errors, warnings and dctSchedulerMessages +currFilename = mfilename; +if ~isa(cluster, 'parallel.Cluster') + error('parallelexamples:GenericSLURM:SubmitFcnError', ... + 'The function %s is for use with clusters created using the parcluster command.', currFilename) +end +if ~cluster.HasSharedFilesystem + error('parallelexamples:GenericSLURM:SubmitFcnError', ... + 'The submit function %s is for use with shared filesystems.', currFilename) +end +% Get the information about the actual cluster used +data = cluster.getJobClusterData(job); +if isempty(data) + % This indicates that the job has not been submitted, so return true + dctSchedulerMessage(1, '%s: Job cluster data was empty for job with ID %d.', currFilename, job.ID); + OK = true; + return +end +try + clusterHost = data.RemoteHost; +catch err + ex = MException('parallelexamples:GenericSLURM:FailedToRetrieveRemoteParameters', ... + 'Failed to retrieve remote parameters from the job cluster data.'); + ex = ex.addCause(err); + throw(ex); +end +remoteConnection = getRemoteConnection(cluster, clusterHost); +try + jobIDs = data.ClusterJobIDs; +catch err + ex = MException('parallelexamples:GenericSLURM:FailedToRetrieveJobID', ... + 'Failed to retrieve clusters''s job IDs from the job cluster data.'); + ex = ex.addCause(err); + throw(ex); +end + +% Only ask the cluster to cancel the job if it is hasn't reached a terminal +% state. +erroredJobs = cell(size(jobIDs)); +jobState = job.State; +if ~(strcmp(jobState, 'finished') || strcmp(jobState, 'failed')) + % Get the cluster to delete the job + for ii = 1:length(jobIDs) + jobID = jobIDs{ii}; + commandToRun = sprintf('scancel ''%s''', jobID); + dctSchedulerMessage(4, '%s: Canceling job on cluster using command:\n\t%s.', currFilename, commandToRun); + % Keep track of all jobs that were not canceled successfully - either through + % a bad exit code or if an error was thrown. We'll report these later on. + try + % Execute the command on the remote host. + [cmdFailed, cmdOut] = remoteConnection.runCommand(commandToRun); + catch err + cmdFailed = true; + cmdOut = err.message; + end + if cmdFailed + erroredJobs{ii} = jobID; + dctSchedulerMessage(1, '%s: Failed to cancel job %d on cluster. Reason:\n\t%s', currFilename, jobID, cmdOut); + end + end +end + +% Now warn about those jobs that we failed to cancel. +erroredJobs = erroredJobs(~cellfun(@isempty, erroredJobs)); +if ~isempty(erroredJobs) + warning('parallelexamples:GenericSLURM:FailedToCancelJob', ... + 'Failed to cancel the following jobs on the cluster:\n%s', ... + sprintf('\t%s\n', erroredJobs{:})); +end +OK = isempty(erroredJobs); diff --git a/+opencossan/+highperformancecomputing/@JobManagerSlurm/remote/communicatingJobWrapper.sh b/+opencossan/+highperformancecomputing/@JobManagerSlurm/remote/communicatingJobWrapper.sh new file mode 100644 index 00000000..baef1e1b --- /dev/null +++ b/+opencossan/+highperformancecomputing/@JobManagerSlurm/remote/communicatingJobWrapper.sh @@ -0,0 +1,257 @@ +#!/bin/sh +# This wrapper script is intended to be submitted to Slurm to support +# communicating jobs. +# +# This script uses the following environment variables set by the submit MATLAB code: +# MDCE_CMR - the value of ClusterMatlabRoot (might be empty) +# MDCE_MATLAB_EXE - the MATLAB executable to use +# MDCE_MATLAB_ARGS - the MATLAB args to use +# +# The following environment variables are forwarded through mpiexec: +# MDCE_DECODE_FUNCTION - the decode function to use +# MDCE_STORAGE_LOCATION - used by decode function +# MDCE_STORAGE_CONSTRUCTOR - used by decode function +# MDCE_JOB_LOCATION - used by decode function + +# The following environment variables are set by Slurm +# SLURM_JOB_ID - number of nodes allocated to Slurm job +# SLURM_JOB_NUM_NODES - number of hosts allocated to Slurm job +# SLURM_JOB_NODELIST - list of hostnames allocated to Slurm job +# SLURM_TASKS_PER_NODE - list containing number of tasks allocated per host to Slurm job + +# Copyright 2015-2017 The MathWorks, Inc. + +# Users of Slurm older than v1.1.34 should uncomment the following code +# to enable mapping from old Slurm environment variables: + +# SLURM_JOB_ID=${SLURM_JOBID} +# SLURM_JOB_NUM_NODES=${SLURM_NNODES} +# SLURM_JOB_NODELIST=${SLURM_NODELIST} + +# Create full paths to mw_smpd/mw_mpiexec if needed +FULL_SMPD=${MDCE_CMR:+${MDCE_CMR}/bin/}mw_smpd +FULL_MPIEXEC=${MDCE_CMR:+${MDCE_CMR}/bin/}mw_mpiexec + +######################################################################################### +# Work out where we need to launch SMPDs given our hosts file - defines SMPD_HOSTS +chooseSmpdHosts() { + + # SLURM_JOB_NODELIST is required: the following line either echoes the value, or aborts. + echo Node file: ${SLURM_JOB_NODELIST:?"Node file undefined"} + + # SMPD_HOSTS is a single line comma separated list of hostnames: + # node136,node138,node140,node141,node142,node143,node157 + # + # Our source of information is SLURM_JOB_NODELIST in the form: + # cnode[136,138],cnode[140-43],cnode157 + # + # 'scontrol show hostname ${SLURM_JOB_NODELIST}' produces multi-line list of hostnames: + # node136 + # node138 + # node140 + # ... + # + # Pipe through "tr" to convert newlines to spaces. + + SMPD_HOSTS=`scontrol show hostname ${SLURM_JOB_NODELIST} | tr '\n', ' '` +} + +######################################################################################### +# Work out which port to use for SMPD +chooseSmpdPort() { + + # Extract the numeric part of SLURM_JOB_ID using sed to choose unique port for SMPD to run on. + # Assumes SLURM_JOB_ID starts with a number, such as: 15.slurm-server-host.domain.com + JOB_NUM=`echo ${SLURM_JOB_ID:?"SLURM_JOB_ID undefined"} | sed 's#^\([0-9][0-9]*\).*$#\1#'` + # Base smpd_port on the numeric part of the above + SMPD_PORT=`expr $JOB_NUM % 10000 + 20000` +} + +######################################################################################### +# Work out how many processes to launch - set MACHINE_ARG +# +# Inputs: +# SLURM_JOB_NUM_NODES Slurm environment variable: Number of nodes allocated to Slurm job +# +# SMPD_HOSTS Comma separated list of hostnames of nodes set by chooseSmpdHosts +# +# SLURM_TASKS_PER_NODE Slurm environment variable: Number of tasks allocated per node. +# If two or more consecutive nodes have the same task count, +# that count is followed by "(x#)" where "#" is the repetition count. +# Output: +# MACHINE_ARG Arguments to pass to mpiexec in the form: +# -hosts host1 tasks_on_host1 host2 tasks_on_host2 +# +# Example +# ------- +# Inputs: +# SLURM_JOB_NUM_NODES 7 +# SMPD_HOSTS node136,node138,node140,node141,node42,node143,node157 +# SLURM_TASKS_PER_NODE 12(x4),7,9(x2) +# Output: +# -hosts 7 node136 12 node138 12 node140 12 node141 12 node142 7 node143 9 node157 9 +# +chooseMachineArg() { + + # Transform SLURM_TASKS_PER_NODE into TASKS_PER_NODE_LIST + # + # Examples: SLURM_TASKS_PER_NODE -> TASKS_PER_NODE_LIST + # ------- -------------------- ------------------- + # Single node has 12 tasks 12 -> 12 + # Three nodes have 12 tasks 12(x3) -> 12,12,12 + # First two nodes have 7 tasks, the third has 8 tasks 7(x2),8 -> 7,7,8 + + TASKS_PER_NODE_LIST='' + # Replace commas with spaces to create space delimited list to use with for loop + LIST_FROM_SLURM=`echo ${SLURM_TASKS_PER_NODE} | sed 's/,/ /g'` + for ITEM in ${LIST_FROM_SLURM} + do + if [ `echo ${ITEM} | grep -e '^[0-9][0-9]*$' -c` -eq 1 ] ; then + # "NUM_TASKS" == "NUM_TASKS(x1)" + NUM_NODES=1 + NUM_TASKS=${ITEM} + else + # "NUM_TASKS(xNUM_NODES)" + NUM_NODES=`echo $ITEM | sed 's/^[0-9][0-9]*(x\([0-9][0-9]*\))$/\1/'` + NUM_TASKS=`echo $ITEM | sed 's/^\([0-9][0-9]*\)(x[0-9][0-9]*)$/\1/'` + fi + + # Repeat NUM_NODES iterations: append NUM_TASKS to TASKS_PER_NODE_LIST + COUNT=0 + while [ ${COUNT} -lt ${NUM_NODES} ] + do + if [ -z "${TASKS_PER_NODE_LIST}" ] ; then + # List empty, therefore adding first item to list - avoid adding comma + TASKS_PER_NODE_LIST=${NUM_TASKS} + else + # Appending to list - add a comma to delimit entries + TASKS_PER_NODE_LIST="${TASKS_PER_NODE_LIST},${NUM_TASKS}" + fi + COUNT=`expr ${COUNT} + 1` + done + done + + # Add -hosts argument at start of MACHINE_ARG + MACHINE_ARG="-hosts ${SLURM_JOB_NUM_NODES}" + + # For each hostname in SMPD_HOSTS, append ' ' to MACHINE_ARG + INDEX=0 + for HOSTNAME in ${SMPD_HOSTS} + do + INDEX=`expr ${INDEX} + 1` + # Use cut to index the '${INDEX}th' item in TASKS_PER_NODE_LIST + TASKS_PER_NODE=`echo ${TASKS_PER_NODE_LIST} | cut -f ${INDEX} -d,` + MACHINE_ARG="${MACHINE_ARG} ${HOSTNAME} ${TASKS_PER_NODE}" + done + echo "Machine args: $MACHINE_ARG" +} + +######################################################################################### +# Shut down SMPDs and exit with the exit code of the last command executed +cleanupAndExit() { + EXIT_CODE=${?} + + echo "Stopping SMPD ..." + + echo "srun --ntasks-per-node=1 --ntasks=${SLURM_JOB_NUM_NODES} ${FULL_SMPD} -shutdown -phrase MATLAB -port ${SMPD_PORT}" + srun --ntasks-per-node=1 --ntasks=${SLURM_JOB_NUM_NODES} ${FULL_SMPD} -shutdown -phrase MATLAB -port ${SMPD_PORT} + + echo "Exiting with code: ${EXIT_CODE}" + exit ${EXIT_CODE} +} + +######################################################################################### +# Use srun to launch the SMPD daemons on each processor +launchSmpds() { + + # Launch the SMPD processes on all hosts using srun + echo "Starting SMPD on ${SMPD_HOSTS} ..." + + START_SMPD_CMD="srun --ntasks-per-node=1 --ntasks=${SLURM_JOB_NUM_NODES} ${FULL_SMPD} -phrase MATLAB -port ${SMPD_PORT} -debug 0 &" + echo $START_SMPD_CMD + eval $START_SMPD_CMD + + # Check that the SMPD processes are running on all hosts + SUCCESS=0 + NUM_ATTEMPTS=10 + SMPD_LAUNCHED_HOSTS="" + ATTEMPT=1 + while [ ${ATTEMPT} -le ${NUM_ATTEMPTS} ] + do + echo "Checking that SMPD processes are running (Attempt ${ATTEMPT} of ${NUM_ATTEMPTS})" + NUM_HOSTS_FOUND=0 + for HOST in ${SMPD_HOSTS} + do + CHECK_SMPD_CMD="${FULL_SMPD} -phrase MATLAB -port ${SMPD_PORT} -status ${HOST} > /dev/null 2>&1" + echo $CHECK_SMPD_CMD + eval $CHECK_SMPD_CMD + EXIT_CODE=${?} + if [ $EXIT_CODE -ne 0 ]; then + echo "No SMPD process running on ${HOST}" + else + echo "SMPD process found running on ${HOST}" + NUM_HOSTS_FOUND=$((NUM_HOSTS_FOUND+1)) + + # Append HOST to SMPD_LAUNCHED_HOSTS if it does not already contain it. + case "${SMPD_LAUNCHED_HOSTS}" in + *$HOST* ) ;; + * ) SMPD_LAUNCHED_HOSTS="${SMPD_LAUNCHED_HOSTS} ${HOST}" ;; + esac + fi + done + if [ ${SLURM_JOB_NUM_NODES} -eq ${NUM_HOSTS_FOUND} ] ; then + SUCCESS=1 + break + elif [ ${ATTEMPT} -ne ${NUM_ATTEMPTS} ] ; then + sleep 6 + fi + ATTEMPT=$((ATTEMPT+1)) + done + if [ $SUCCESS -ne 1 ] ; then + if [ $NUM_HOSTS_FOUND -eq 0 ] ; then + echo "No SMPD processes were found running. Aborting." + else + echo "Found SMPD processes running on only ${NUMHOSTS} of ${SLURM_JOB_NUM_NODES} nodes. Aborting." + echo "Hosts found: ${SMPD_LAUNCHED_HOSTS}" + fi + exit 1 + fi + echo "All SMPDs launched" +} + +######################################################################################### +runMpiexec() { + + CMD="${FULL_MPIEXEC} -phrase MATLAB -port ${SMPD_PORT} \ + -l ${MACHINE_ARG} -genvlist \ + MDCE_DECODE_FUNCTION,MDCE_STORAGE_LOCATION,MDCE_STORAGE_CONSTRUCTOR,MDCE_JOB_LOCATION,MDCE_DEBUG,MDCE_LICENSE_NUMBER,MLM_WEB_LICENSE,MLM_WEB_USER_CRED,MLM_WEB_ID \ + \"${MDCE_MATLAB_EXE}\" ${MDCE_MATLAB_ARGS}" + + # As a debug stage: echo the command line... + echo $CMD + + # ...and then execute it. + eval $CMD + + MPIEXEC_CODE=${?} + if [ ${MPIEXEC_CODE} -ne 0 ] ; then + exit ${MPIEXEC_CODE} + fi +} + +######################################################################################### +# Define the order in which we execute the stages defined above +MAIN() { + # Install a trap to ensure that SMPDs are closed if something errors or the + # job is cancelled. + trap "cleanupAndExit" 0 1 2 15 + chooseSmpdHosts + chooseSmpdPort + launchSmpds + chooseMachineArg + runMpiexec + exit 0 # Explicitly exit 0 to trigger cleanupAndExit +} + +# Call the MAIN loop +MAIN diff --git a/+opencossan/+highperformancecomputing/@JobManagerSlurm/remote/communicatingSubmitFcn.m b/+opencossan/+highperformancecomputing/@JobManagerSlurm/remote/communicatingSubmitFcn.m new file mode 100644 index 00000000..188f8824 --- /dev/null +++ b/+opencossan/+highperformancecomputing/@JobManagerSlurm/remote/communicatingSubmitFcn.m @@ -0,0 +1,142 @@ +function communicatingSubmitFcn(cluster, job, environmentProperties) +%COMMUNICATINGSUBMITFCN Submit a communicating MATLAB job to a Slurm cluster +% +% Set your cluster's IntegrationScriptsLocation to the parent folder of this +% function to run it when you submit a communicating job. +% +% See also parallel.cluster.generic.communicatingDecodeFcn. + +% Copyright 2010-2017 The MathWorks, Inc. + +% Store the current filename for the errors, warnings and dctSchedulerMessages +currFilename = mfilename; +if ~isa(cluster, 'parallel.Cluster') + error('parallelexamples:GenericSLURM:SubmitFcnError', ... + 'The function %s is for use with clusters created using the parcluster command.', currFilename) +end + +decodeFunction = 'parallel.cluster.generic.communicatingDecodeFcn'; + +if ~cluster.HasSharedFilesystem + error('parallelexamples:GenericSLURM:SubmitFcnError', ... + 'The submit function %s is for use with shared filesystems.', currFilename) +end + +if ~isprop(cluster.AdditionalProperties, 'ClusterHost') + error('parallelexamples:GenericSLURM:MissingAdditionalProperties', ... + 'Required field %s is missing from AdditionalProperties.', 'ClusterHost'); +end +clusterHost = cluster.AdditionalProperties.ClusterHost; + +if ~strcmpi(cluster.OperatingSystem, 'unix') + error('parallelexamples:GenericSLURM:SubmitFcnError', ... + 'The submit function %s only supports clusters with unix OS.', currFilename) +end +if ~ischar(clusterHost) + error('parallelexamples:GenericSLURM:IncorrectArguments', ... + 'Hostname must be a character vector'); +end + +remoteConnection = getRemoteConnection(cluster, clusterHost); + +% The job specific environment variables +% Remove leading and trailing whitespace from the MATLAB arguments +matlabArguments = strtrim(environmentProperties.MatlabArguments); +variables = {'MDCE_DECODE_FUNCTION', decodeFunction; ... + 'MDCE_STORAGE_CONSTRUCTOR', environmentProperties.StorageConstructor; ... + 'MDCE_JOB_LOCATION', environmentProperties.JobLocation; ... + 'MDCE_MATLAB_EXE', environmentProperties.MatlabExecutable; ... + 'MDCE_MATLAB_ARGS', matlabArguments; ... + 'MDCE_DEBUG', 'true'; ... + 'MLM_WEB_LICENSE', environmentProperties.UseMathworksHostedLicensing; ... + 'MLM_WEB_USER_CRED', environmentProperties.UserToken; ... + 'MLM_WEB_ID', environmentProperties.LicenseWebID; ... + 'MDCE_LICENSE_NUMBER', environmentProperties.LicenseNumber; ... + 'MDCE_STORAGE_LOCATION', environmentProperties.StorageLocation; ... + 'MDCE_CMR', cluster.ClusterMatlabRoot; ... + 'MDCE_TOTAL_TASKS', num2str(environmentProperties.NumberOfTasks)}; +% Trim the environment variables of empty values. +nonEmptyValues = cellfun(@(x) ~isempty(strtrim(x)), variables(:,2)); +variables = variables(nonEmptyValues, :); + +% Get the correct quote and file separator for the Cluster OS. +% This check is unnecessary in this file because we explicitly +% checked that the ClusterOsType is unix. This code is an example +% of how your integration code should deal with clusters that +% can be unix or pc. +if strcmpi(cluster.OperatingSystem, 'unix') + quote = ''''; + fileSeparator = '/'; +else + quote = '"'; + fileSeparator = '\'; +end + +% The local job directory +localJobDirectory = cluster.getJobFolder(job); +% Find out how we should refer to the job storage location on the cluster. +remoteJobDirectory = cluster.getJobFolderOnCluster(job); + +% The script name is communicatingJobWrapper.sh +scriptName = 'communicatingJobWrapper.sh'; +% The wrapper script is in the same directory as this file +dirpart = fileparts(mfilename('fullpath')); +localScript = fullfile(dirpart, scriptName); +% Copy the local wrapper script to the job directory +copyfile(localScript, localJobDirectory); + +% The command that will be executed on the remote host to run the job. +remoteScriptName = sprintf('%s%s%s', remoteJobDirectory, fileSeparator, scriptName); +quotedScriptName = sprintf('%s%s%s', quote, remoteScriptName, quote); + +% Choose a file for the output. Please note that currently, JobStorageLocation refers +% to a directory on disk, but this may change in the future. +logFile = sprintf('%s%s%s', remoteJobDirectory, fileSeparator, sprintf('Job%d.log', job.ID)); +quotedLogFile = sprintf('%s%s%s', quote, logFile, quote); + +jobName = sprintf('Job%d', job.ID); +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% CUSTOMIZATION MAY BE REQUIRED %% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% You might want to customize this section to match your cluster, +% for example to limit the number of nodes for a single job. +additionalSubmitArgs = sprintf('--ntasks=%d', environmentProperties.NumberOfTasks); +commonSubmitArgs = getCommonSubmitArgs(cluster); +if ~isempty(commonSubmitArgs) && ischar(commonSubmitArgs) + additionalSubmitArgs = strtrim([additionalSubmitArgs, ' ', commonSubmitArgs]); +end +% Create a script to submit a Slurm job - this will be created in the job directory +dctSchedulerMessage(5, '%s: Generating script for job.', currFilename); +localScriptName = tempname(localJobDirectory); +[~, scriptName] = fileparts(localScriptName); +remoteScriptLocation = sprintf('%s%s%s', remoteJobDirectory, fileSeparator, scriptName); +createSubmitScript(localScriptName, jobName, quotedLogFile, quotedScriptName, ... + variables, additionalSubmitArgs); +% Create the command to run on the remote host. +commandToRun = sprintf('sh %s', remoteScriptLocation); + +% Now ask the cluster to run the submission command +dctSchedulerMessage(4, '%s: Submitting job using command:\n\t%s', currFilename, commandToRun); +% Execute the command on the remote host. +[cmdFailed, cmdOut] = remoteConnection.runCommand(commandToRun); +if cmdFailed + error('parallelexamples:GenericSLURM:FailedToSubmitJob', ... + 'Failed to submit job to Slurm using command:\n\t%s.\nReason: %s', ... + commandToRun, cmdOut); +end + +jobIDs = extractJobId(cmdOut); +% jobIDs must be a cell array +if isempty(jobIDs) + warning('parallelexamples:GenericSLURM:FailedToParseSubmissionOutput', ... + 'Failed to parse the job identifier from the submission output: "%s"', ... + cmdOut); +end +if ~iscell(jobIDs) + jobIDs = {jobIDs}; +end + +% set the cluster host, job ID on the job cluster data +jobData = struct('ClusterJobIDs', {jobIDs}, ... + 'RemoteHost', clusterHost); +cluster.setJobClusterData(job, jobData); diff --git a/+opencossan/+highperformancecomputing/@JobManagerSlurm/remote/createSubmitScript.m b/+opencossan/+highperformancecomputing/@JobManagerSlurm/remote/createSubmitScript.m new file mode 100644 index 00000000..ad339588 --- /dev/null +++ b/+opencossan/+highperformancecomputing/@JobManagerSlurm/remote/createSubmitScript.m @@ -0,0 +1,30 @@ +function createSubmitScript(outputFilename, jobName, quotedLogFile, quotedScriptName, ... + environmentVariables, additionalSubmitArgs) +% Create a script that sets the correct environment variables and then +% executes the Slurm sbatch command. + +% Copyright 2010-2016 The MathWorks, Inc. + +% Open file in binary mode to make it cross-platform. +fid = fopen(outputFilename, 'w'); +if fid < 0 + error('parallelexamples:GenericSLURM:FileError', ... + 'Failed to open file %s for writing', outputFilename); +end + +% Specify Shell to use +fprintf(fid, '#!/bin/sh\n'); + +% Write the commands to set and export environment variables +for ii = 1:size(environmentVariables, 1) + fprintf(fid, '%s=%s\n', environmentVariables{ii,1}, environmentVariables{ii,2}); + fprintf(fid, 'export %s\n', environmentVariables{ii,1}); +end + +% Generate the command to run and write it. +commandToRun = getSubmitString(jobName, quotedLogFile, quotedScriptName, ... + additionalSubmitArgs); +fprintf(fid, '%s\n', commandToRun); + +% Close the file +fclose(fid); diff --git a/+opencossan/+highperformancecomputing/@JobManagerSlurm/remote/deleteJobFcn.m b/+opencossan/+highperformancecomputing/@JobManagerSlurm/remote/deleteJobFcn.m new file mode 100644 index 00000000..29e94ca7 --- /dev/null +++ b/+opencossan/+highperformancecomputing/@JobManagerSlurm/remote/deleteJobFcn.m @@ -0,0 +1,8 @@ +function deleteJobFcn(cluster, job) +%DELETEJOBFCN Deletes a job on cluster +% +% Set your cluster's IntegrationScriptsLocation to the parent folder of this +% function to run it when you delete a job. + +% Copyright 2017 The MathWorks, Inc. +cancelJobFcn(cluster, job); diff --git a/+opencossan/+highperformancecomputing/@JobManagerSlurm/remote/getCommonSubmitArgs.m b/+opencossan/+highperformancecomputing/@JobManagerSlurm/remote/getCommonSubmitArgs.m new file mode 100644 index 00000000..8938cff7 --- /dev/null +++ b/+opencossan/+highperformancecomputing/@JobManagerSlurm/remote/getCommonSubmitArgs.m @@ -0,0 +1,18 @@ +function commonSubmitArgs = getCommonSubmitArgs(cluster) +% Get any additional submit arguments for the Slurm sbatch command +% that are common to both independent and communicating jobs. + +% Copyright 2016-2017 The MathWorks, Inc. + +commonSubmitArgs = ''; + +% Append any arguments provided by the AdditionalSubmitArgs field of cluster.AdditionalProperties. +if isprop(cluster.AdditionalProperties, 'AdditionalSubmitArgs') + extraArgs = cluster.AdditionalProperties.AdditionalSubmitArgs; + if ~isempty(extraArgs) && ischar(extraArgs) + commonSubmitArgs = strtrim([commonSubmitArgs, ' ', extraArgs]); + end +end + +% You may wish to support further cluster.AdditionalProperties fields here +% and modify the submission command arguments accordingly. diff --git a/+opencossan/+highperformancecomputing/@JobManagerSlurm/remote/getJobStateFcn.m b/+opencossan/+highperformancecomputing/@JobManagerSlurm/remote/getJobStateFcn.m new file mode 100644 index 00000000..acf55a66 --- /dev/null +++ b/+opencossan/+highperformancecomputing/@JobManagerSlurm/remote/getJobStateFcn.m @@ -0,0 +1,119 @@ +function state = getJobStateFcn(cluster, job, state) +%GETJOBSTATEFCN Gets the state of a job from Slurm +% +% Set your cluster's IntegrationScriptsLocation to the parent folder of this +% function to run it when you query the state of a job. + +% Copyright 2010-2017 The MathWorks, Inc. + +% Store the current filename for the errors, warnings and dctSchedulerMessages +currFilename = mfilename; +if ~isa(cluster, 'parallel.Cluster') + error('parallelexamples:GenericSLURM:SubmitFcnError', ... + 'The function %s is for use with clusters created using the parcluster command.', currFilename) +end +if ~cluster.HasSharedFilesystem + error('parallelexamples:GenericSLURM:SubmitFcnError', ... + 'The submit function %s is for use with shared filesystems.', currFilename) +end + +% Shortcut if the job state is already finished or failed +jobInTerminalState = strcmp(state, 'finished') || strcmp(state, 'failed'); +if jobInTerminalState + return +end +% Get the information about the actual cluster used +data = cluster.getJobClusterData(job); +if isempty(data) + % This indicates that the job has not been submitted, so just return + dctSchedulerMessage(1, '%s: Job cluster data was empty for job with ID %d.', currFilename, job.ID); + return +end +try + clusterHost = data.RemoteHost; +catch err + ex = MException('parallelexamples:GenericSLURM:FailedToRetrieveRemoteParameters', ... + 'Failed to retrieve remote parameters from the job cluster data.'); + ex = ex.addCause(err); + throw(ex); +end +remoteConnection = getRemoteConnection(cluster, clusterHost); +try + jobIDs = data.ClusterJobIDs; +catch err + ex = MException('parallelexamples:GenericSLURM:FailedToRetrieveJobID', ... + 'Failed to retrieve clusters''s job IDs from the job cluster data.'); + ex = ex.addCause(err); + throw(ex); +end + + +% Get the top level job state from sacct +% We are not using squeue because it omits information about completed/terminated jobs. +% We use the '--allocations' option to request cumulative statistics for each job. +% (Normal output of sacct includes intermediate Slurm job steps which we don't want.) +% If the Slurm JobID counter has been reset, there is a short period of time after a job +% has been submitted that sacct may return information on an old job with the same JobID. +% The option '--user=$USER' makes this less likely, as it will only happen if the old job +% was submitted by the same user. +jobList = sprintf('-j ''%s'' ', jobIDs{:}); +commandToRun = sprintf('sacct --allocations --user=$USER %s', jobList); +dctSchedulerMessage(4, '%s: Querying cluster for job state using command:\n\t%s', currFilename, commandToRun); + +try + % We will ignore the status returned from the state command because + % a non-zero status is returned if the job no longer exists + % Execute the command on the remote host. + [~, cmdOut] = remoteConnection.runCommand(commandToRun); +catch err + ex = MException('parallelexamples:GenericSLURM:FailedToGetJobState', ... + 'Failed to get job state from cluster.'); + ex = ex.addCause(err); + throw(ex); +end + +clusterState = iExtractJobState(cmdOut, numel(jobIDs)); +dctSchedulerMessage(6, '%s: State %s was extracted from cluster output.', currFilename, clusterState); + +% If we could determine the cluster's state, we'll use that, otherwise +% stick with MATLAB's job state. +if ~strcmp(clusterState, 'unknown') + state = clusterState; +end + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +function state = iExtractJobState(sacctOut, numJobs) +% Function to extract the job state from the output of sacct + +numPending = numel(regexp(sacctOut, 'PENDING')); +numRunning = numel(regexp(sacctOut, 'RUNNING|SUSPENDED|COMPLETING|CONFIGURING')); +numFinished = numel(regexp(sacctOut, 'COMPLETED')); +numFailed = numel(regexp(sacctOut, 'CANCELLED|FAILED|TIMEOUT|PREEMPTED|NODE_FAIL')); + +% If all of the jobs that we asked about have finished, then we know the job has finished. +if numFinished == numJobs + state = 'finished'; + return +end + +% Any running indicates that the job is running +if numRunning > 0 + state = 'running'; + return +end + +% We know numRunning == 0 so if there are some still pending then the +% job must be queued again, even if there are some finished +if numPending > 0 + state = 'queued'; + return +end + +% Deal with any tasks that have failed +if numFailed > 0 + % Set this job to be failed + state = 'failed'; + return +end + +state = 'unknown'; diff --git a/+opencossan/+highperformancecomputing/@JobManagerSlurm/remote/getRemoteConnection.m b/+opencossan/+highperformancecomputing/@JobManagerSlurm/remote/getRemoteConnection.m new file mode 100644 index 00000000..9526639a --- /dev/null +++ b/+opencossan/+highperformancecomputing/@JobManagerSlurm/remote/getRemoteConnection.m @@ -0,0 +1,185 @@ +function remoteConnection = getRemoteConnection(cluster, clusterHost) +%GETREMOTECONNECTION Get a connected RemoteClusterAccess +% +% getRemoteConnection will either retrieve a RemoteClusterAccess from the +% cluster's UserData or it will create a new RemoteClusterAccess. + +% Copyright 2010-2017 The MathWorks, Inc. + +% Store the current filename for the dctSchedulerMessages +currFilename = mfilename; + +if ~ischar(clusterHost) + error('parallelexamples:GenericSLURM:IncorrectArguments', ... + 'Hostname must be a string'); +end + +needToCreateNewConnection = false; +if isempty(cluster.UserData) + needToCreateNewConnection = true; +else + if ~isstruct(cluster.UserData) + error('parallelexamples:GenericSLURM:IncorrectUserData', ... + ['Failed to retrieve remote connection from cluster''s UserData.\n' ... + 'Expected cluster''s UserData to be a structure, but found %s'], ... + class(cluster.UserData)); + end + + if isfield(cluster.UserData, 'RemoteConnection') + % Get the remote connection out of the cluster user data + remoteConnection = cluster.UserData.RemoteConnection; + + % And check it is of the type that we expect + if isempty(remoteConnection) + needToCreateNewConnection = true; + else + clusterAccessClassname = 'parallel.cluster.RemoteClusterAccess'; + if ~isa(remoteConnection, clusterAccessClassname) + error('parallelexamples:GenericSLURM:IncorrectArguments', ... + ['Failed to retrieve remote connection from cluster''s UserData.\n' ... + 'Expected the RemoteConnection field of the UserData to contain an object of type %s, but found %s.'], ... + clusterAccessClassname, class(remoteConnection)); + end + + + if ~remoteConnection.IsConnected + needToCreateNewConnection = true; + elseif ~strcmpi(remoteConnection.Hostname, clusterHost) + % The connection stored in the user data does not match the cluster host requested + warning('parallelexamples:GenericSLURM:DifferentRemoteParameters', ... + ['The current cluster is already using cluster host.\n', ... + 'The existing connection to %s will be replaced.'], ... + remoteConnection.Hostname, remoteConnection.Hostname); + cluster.UserData.RemoteConnection = []; + needToCreateNewConnection = true; + end + end + else + needToCreateNewConnection = true; + end +end + +if ~needToCreateNewConnection + return +end + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% CUSTOMIZATION MAY BE REQUIRED %% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% Get the credential options from the user using simple +% MATLAB dialogs or command line input. You should change +% this section if you wish for users to provide their credential +% options in a different way. +% The pertinent options are: +% username - the username to use when running commands on the remote host +% useIdentityFile - whether or not to use an identity file (true/false). +% False means that a password is used +% identityFilename - the full path to the identity file +% fileHasPassphrase - whether or not the identity file requires a passphrase +% (true/false). +if isempty(javachk('awt')) + % MATLAB has been started with a desktop, so use dialogs to get credential data. + [username, useIdentityFile, identityFilename, fileHasPassphrase] = iGetCredentialsFromUI(clusterHost); +else + % MATLAB has been started in nodisplay mode, so use command line to get credential data + [username, useIdentityFile, identityFilename, fileHasPassphrase] = iGetCredentialsFromCommandLine(clusterHost); +end + +% Establish a new connection +if useIdentityFile + dctSchedulerMessage(1, '%s: Identity file %s will be used for remote connections', ... + currFilename, username, identityFilename); + userArgs = {username, ... + 'IdentityFilename', identityFilename, 'IdentityFileHasPassphrase', fileHasPassphrase}; +else + userArgs = {username}; +end + +% Now connect and store the connection +dctSchedulerMessage(1, '%s: Connecting to remote host %s', ... + currFilename, clusterHost); +remoteConnection = parallel.cluster.RemoteClusterAccess.getConnectedAccess(clusterHost, userArgs{:}); +dctSchedulerMessage(5, '%s: Storing remote connection in cluster''s user data.', currFilename); +cluster.UserData.RemoteConnection = remoteConnection; + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +function [username, useIdentityFile, identityFilename, fileHasPassphrase] = iGetCredentialsFromUI(clusterHost) +% Function to get the user credentials using dialogs +identityFilename = ''; +fileHasPassphrase = false; + +dlgTitle = 'User Credentials'; +numlines = 1; +dlgMessage = sprintf('Enter the username for %s', clusterHost); +usernameResponse = inputdlg(dlgMessage, dlgTitle, numlines); +% Hitting cancel gives an empty cell array, but a user providing an empty string gives +% a (non-empty) cell array containing an empty string +if isempty(usernameResponse) + % User hit cancel + error('parallelexamples:GenericSLURM:UserCancelledOperation', 'User cancelled operation.'); +end +username = char(usernameResponse); + +dlgMessage = sprintf('Use an identity file to login to %s?', clusterHost); +identityFileResponse = questdlg(dlgMessage, dlgTitle); +if strcmp(identityFileResponse, 'Cancel') + % User hit cancel + error('parallelexamples:GenericSLURM:UserCancelledOperation', 'User cancelled operation.'); +end + +useIdentityFile = strcmp(identityFileResponse, 'Yes'); +if ~useIdentityFile + return +end + +dlgMessage = 'Select Identity File to use'; +[filename, pathname] = uigetfile({'*.*', 'All Files (*.*)'}, dlgMessage); +% If the user hit cancel, then filename and pathname will both be 0. +if isequal(filename, 0) && isequal(pathname,0) + error('parallelexamples:GenericSLURM:UserCancelledOperation', 'User cancelled operation.'); +end + +identityFilename = fullfile(pathname, filename); +dlgMessage = 'Does the identity file require a password?'; +passphraseResponse = questdlg(dlgMessage, dlgTitle); +if strcmp(passphraseResponse, 'Cancel') + % User hit cancel + error('parallelexamples:GenericSLURM:UserCancelledOperation', 'User cancelled operation.'); +end +fileHasPassphrase = strcmp(passphraseResponse, 'Yes'); + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +function [username, useIdentityFile, identityFilename, fileHasPassphrase] = iGetCredentialsFromCommandLine(clusterHost) +% Function to get the user credentials from the command line +identityFilename = ''; +fileHasPassphrase = false; +validYesNoResponse = {'y', 'n'}; + +% Allow the username to be empty +username = input(sprintf('Enter the username for %s:\n', clusterHost), 's'); + +identityFileMessage = sprintf('Use an identity file to login to %s? (y or n)\n', clusterHost); +identityFileResponse = iLoopUntilValidStringInput(identityFileMessage, validYesNoResponse); +useIdentityFile = strcmpi(identityFileResponse, 'y'); +if ~useIdentityFile + return +end + +identityFilename = ''; +while isempty(identityFilename) + identityFilename = input(sprintf('Please enter the full path to the Identity File to use:\n'), 's'); +end + +passphraseMessage = 'Does the identity file require a password? (y or n)\n'; +passphraseResponse = iLoopUntilValidStringInput(passphraseMessage, validYesNoResponse); +fileHasPassphrase = strcmpi(passphraseResponse, 'y'); + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +function returnValue = iLoopUntilValidStringInput(message, validValues) +% Function to loop until a valid response is obtained user input +returnValue = ''; + +while isempty(returnValue) || ~any(strcmpi(returnValue, validValues)) + returnValue = input(message, 's'); +end + diff --git a/+opencossan/+highperformancecomputing/@JobManagerSlurm/remote/getSubmitString.m b/+opencossan/+highperformancecomputing/@JobManagerSlurm/remote/getSubmitString.m new file mode 100644 index 00000000..a1520e27 --- /dev/null +++ b/+opencossan/+highperformancecomputing/@JobManagerSlurm/remote/getSubmitString.m @@ -0,0 +1,7 @@ +function submitString = getSubmitString(jobName, quotedLogFile, quotedCommand, additionalSubmitArgs) +%GETSUBMITSTRING Gets the correct sbatch command for a Slurm cluster + +% Copyright 2010-2016 The MathWorks, Inc. + +submitString = sprintf('sbatch --job-name=%s --output=%s %s %s', ... + jobName, quotedLogFile, additionalSubmitArgs, quotedCommand); diff --git a/+opencossan/+highperformancecomputing/@JobManagerSlurm/remote/independentJobWrapper.sh b/+opencossan/+highperformancecomputing/@JobManagerSlurm/remote/independentJobWrapper.sh new file mode 100644 index 00000000..3e62e00d --- /dev/null +++ b/+opencossan/+highperformancecomputing/@JobManagerSlurm/remote/independentJobWrapper.sh @@ -0,0 +1,15 @@ +#!/bin/sh +# This wrapper script is intended to support independent execution. +# +# This script uses the following environment variables set by the submit MATLAB code: +# MDCE_MATLAB_EXE - the MATLAB executable to use +# MDCE_MATLAB_ARGS - the MATLAB args to use +# + +# Copyright 2010-2017 The MathWorks, Inc. + +echo "Executing: ${MDCE_MATLAB_EXE} ${MDCE_MATLAB_ARGS}" +eval "${MDCE_MATLAB_EXE}" ${MDCE_MATLAB_ARGS} +EXIT_CODE=${?} +echo "Exiting with code: ${EXIT_CODE}" +exit ${EXIT_CODE} \ No newline at end of file diff --git a/+opencossan/+highperformancecomputing/@JobManagerSlurm/remote/independentSubmitFcn.m b/+opencossan/+highperformancecomputing/@JobManagerSlurm/remote/independentSubmitFcn.m new file mode 100644 index 00000000..1c6b1bc2 --- /dev/null +++ b/+opencossan/+highperformancecomputing/@JobManagerSlurm/remote/independentSubmitFcn.m @@ -0,0 +1,147 @@ +function independentSubmitFcn(cluster, job, environmentProperties) +%INDEPENDENTSUBMITFCN Submit a MATLAB job to a Slurm cluster +% +% Set your cluster's IntegrationScriptsLocation to the parent folder of this +% function to run it when you submit an independent job. +% +% See also parallel.cluster.generic.independentDecodeFcn. + +% Copyright 2010-2017 The MathWorks, Inc. + +% Store the current filename for the errors, warnings and dctSchedulerMessages +currFilename = mfilename; +if ~isa(cluster, 'parallel.Cluster') + error('parallelexamples:GenericSLURM:SubmitFcnError', ... + 'The function %s is for use with clusters created using the parcluster command.', currFilename) +end + +decodeFunction = 'parallel.cluster.generic.independentDecodeFcn'; + +if ~cluster.HasSharedFilesystem + error('parallelexamples:GenericSLURM:SubmitFcnError', ... + 'The submit function %s is for use with shared filesystems.', currFilename) +end + +if ~isprop(cluster.AdditionalProperties, 'ClusterHost') + error('parallelexamples:GenericSLURM:MissingAdditionalProperties', ... + 'Required field %s is missing from AdditionalProperties.', 'ClusterHost'); +end +clusterHost = cluster.AdditionalProperties.ClusterHost; + +if ~strcmpi(cluster.OperatingSystem, 'unix') + error('parallelexamples:GenericSLURM:SubmitFcnError', ... + 'The submit function %s only supports clusters with unix OS.', currFilename) +end +if ~ischar(clusterHost) + error('parallelexamples:GenericSLURM:IncorrectArguments', ... + 'Hostname must be a character vector'); +end + +remoteConnection = getRemoteConnection(cluster, clusterHost); + +% The job specific environment variables +% Remove leading and trailing whitespace from the MATLAB arguments +matlabArguments = strtrim(environmentProperties.MatlabArguments); +variables = {'MDCE_DECODE_FUNCTION', decodeFunction; ... + 'MDCE_STORAGE_CONSTRUCTOR', environmentProperties.StorageConstructor; ... + 'MDCE_JOB_LOCATION', environmentProperties.JobLocation; ... + 'MDCE_MATLAB_EXE', environmentProperties.MatlabExecutable; ... + 'MDCE_MATLAB_ARGS', matlabArguments; ... + 'MDCE_DEBUG', 'true'; ... + 'MLM_WEB_LICENSE', environmentProperties.UseMathworksHostedLicensing; ... + 'MLM_WEB_USER_CRED', environmentProperties.UserToken; ... + 'MLM_WEB_ID', environmentProperties.LicenseWebID; ... + 'MDCE_LICENSE_NUMBER', environmentProperties.LicenseNumber; ... + 'MDCE_STORAGE_LOCATION', environmentProperties.StorageLocation}; +% Trim the environment variables of empty values. +nonEmptyValues = cellfun(@(x) ~isempty(strtrim(x)), variables(:,2)); +variables = variables(nonEmptyValues, :); + +% Get the correct quote and file separator for the Cluster OS. +% This check is unnecessary in this file because we explicitly +% checked that the ClusterOsType is unix. This code is an example +% of how your integration code should deal with clusters that +% can be unix or pc. +if strcmpi(cluster.OperatingSystem, 'unix') + quote = ''''; + fileSeparator = '/'; +else + quote = '"'; + fileSeparator = '\'; +end + +% The local job directory +localJobDirectory = cluster.getJobFolder(job); +% Find out how we should refer to the job storage location on the cluster. +remoteJobDirectory = cluster.getJobFolderOnCluster(job); + +% The script name is independentJobWrapper.sh +scriptName = 'independentJobWrapper.sh'; +% The wrapper script is in the same directory as this file +dirpart = fileparts(mfilename('fullpath')); +localScript = fullfile(dirpart, scriptName); +% Copy the local wrapper script to the job directory +copyfile(localScript, localJobDirectory); + +% The command that will be executed on the remote host to run the job. +remoteScriptName = sprintf('%s%s%s', remoteJobDirectory, fileSeparator, scriptName); +quotedScriptName = sprintf('%s%s%s', quote, remoteScriptName, quote); + +% Get the tasks for use in the loop +tasks = job.Tasks; +numberOfTasks = environmentProperties.NumberOfTasks; +jobIDs = cell(numberOfTasks, 1); +% Loop over every task we have been asked to submit +for ii = 1:numberOfTasks + taskLocation = environmentProperties.TaskLocations{ii}; + % Add the task location to the environment variables + environmentVariables = [variables; ... + {'MDCE_TASK_LOCATION', taskLocation}]; + + % Choose a file for the output. Please note that currently, JobStorageLocation refers + % to a directory on disk, but this may change in the future. + logFile = sprintf('%s%s%s', remoteJobDirectory, fileSeparator, sprintf('Task%d.log', tasks(ii).ID)); + quotedLogFile = sprintf('%s%s%s', quote, logFile, quote); + + % Submit one task at a time + jobName = sprintf('Job%d.%d', job.ID, tasks(ii).ID); + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + %% CUSTOMIZATION MAY BE REQUIRED %% + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + additionalSubmitArgs = '--ntasks=1'; + commonSubmitArgs = getCommonSubmitArgs(cluster); + if ~isempty(commonSubmitArgs) && ischar(commonSubmitArgs) + additionalSubmitArgs = strtrim([additionalSubmitArgs, ' ', commonSubmitArgs]); + end + % Create a script to submit a Slurm job - this will be created in the job directory + dctSchedulerMessage(5, '%s: Generating script for task %i', currFilename, ii); + localScriptName = tempname(localJobDirectory); + [~, scriptName] = fileparts(localScriptName); + remoteScriptLocation = sprintf('%s%s%s', remoteJobDirectory, fileSeparator, scriptName); + createSubmitScript(localScriptName, jobName, quotedLogFile, quotedScriptName, ... + environmentVariables, additionalSubmitArgs); + % Create the command to run on the remote host. + commandToRun = sprintf('sh %s', remoteScriptLocation); + + % Now ask the cluster to run the submission command + dctSchedulerMessage(4, '%s: Submitting job using command:\n\t%s', currFilename, commandToRun); + % Execute the command on the remote host. + [cmdFailed, cmdOut] = remoteConnection.runCommand(commandToRun); + if cmdFailed + error('parallelexamples:GenericSLURM:FailedToSubmitJob', ... + 'Failed to submit job to Slurm using command:\n\t%s.\nReason: %s', ... + commandToRun, cmdOut); + end + jobIDs{ii} = extractJobId(cmdOut); + + if isempty(jobIDs{ii}) + warning('parallelexamples:GenericSLURM:FailedToParseSubmissionOutput', ... + 'Failed to parse the job identifier from the submission output: "%s"', ... + cmdOut); + end +end + +% set the cluster host, job ID on the job cluster data +jobData = struct('ClusterJobIDs', {jobIDs}, ... + 'RemoteHost', clusterHost); +cluster.setJobClusterData(job, jobData); diff --git a/+opencossan/+highperformancecomputing/@JobManagerSlurm/shared/README b/+opencossan/+highperformancecomputing/@JobManagerSlurm/shared/README new file mode 100644 index 00000000..e2daea95 --- /dev/null +++ b/+opencossan/+highperformancecomputing/@JobManagerSlurm/shared/README @@ -0,0 +1,114 @@ +Copyright 2010-2018 The MathWorks, Inc. + +MATLAB and Simulink are registered trademarks of The MathWorks, Inc. +See www.mathworks.com/trademarks for a list of additional trademarks. +Other product or brand names may be trademarks or registered trademarks of their respective holders. + +This folder contains a number of files to allow Parallel Computing Toolbox +to be used with Slurm via the generic cluster interface. + +The files in this folder assume that the client and cluster share a file system +and that the client is able to submit directly to the cluster using the +sbatch command. + +Note that all the files in this directory will work only for clusters that are +running on UNIX. + +1) Instructions for Use +======================= +On the Slurm cluster +-------------------- +Enable job accounting on the Slurm cluster. +This allows the sacct command to run. +The scripts use the sacct output to track the state of Slurm jobs. + +Read the documentation for using the generic cluster interface with +the Parallel Computing Toolbox and familiarize yourself with the different +properties that can be set for a generic cluster. + +In the MATLAB Client +-------------------- +1. Create a generic cluster object for your cluster. Set the IntegrationScriptsLocation +to this folder. For independent jobs, independentSubmitFcn is used as your +submit function. For communicating jobs, communicatingSubmitFcn is used as +your submit function. + +Example: +% Use a folder that both the client and cluster can access +% as the JobStorageLocation. If your cluster and client use +% different operating systems, specify JobStorageLocation +% to be a structure. Refer to the documentation on +% generic cluster for more information. +cluster = parallel.cluster.Generic('JobStorageLocation', '/home/JOB_STORAGE_LOCATION'); +set(cluster, 'HasSharedFilesystem', true); +set(cluster, 'IntegrationScriptsLocation', '/parallel/slurm/shared'); +set(cluster, 'ClusterMatlabRoot', '/apps/matlab'); +set(cluster, 'OperatingSystem', 'unix'); + +2. Create a job and some tasks, submit the job, and wait for it to finish before +getting the results. Do the same for communicating jobs if required. + + +As an alternative to these steps, create a profile that defines the appropriate +properties and run profile validation to verify that the profile +works correctly. + + +2) Description of Files +======================= +For more detail about these files, refer to the help and comments contained in the files themselves. + +MATLAB Functions Required for generic cluster +---------------------------------------------- +independentSubmitFcn.m + Submit function for independent jobs. Used as the IndependentSubmitFcn for your generic cluster object. +communicatingSubmitFcn.m + Submit function for communicating jobs. Used as the CommunicatingSubmitFcn for your generic cluster object. +deleteJobFcn.m + Delete a job on the cluster. Used as the DeleteJobFcn for your generic cluster object. +getJobStateFcn.m + Get the job state from the cluster. Used as the GetJobStateFcn for your generic cluster object. + +Other MATLAB Functions +----------------------- +extractJobId.m + Get the cluster's job ID from the submission output. +getSubmitString.m + Get the submission string for the cluster. + +Executable Scripts +------------------- +independentJobWrapper.sh + Script used by the cluster to launch the MATLAB worker processes for independent jobs. +communicatingJobWrapper.sh + Script used by the cluster to launch the MATLAB worker processes for communicating jobs. + + +3) Optional Customizations +========================== +The code customizations listed in this section are clearly marked in the relevant files. + +independentSubmitFcn.m +---------------------- +independentSubmitFcn provides the ability to supply additional submit arguments to the +sbatch command. Modify the AdditionalProperties.AdditionalSubmitArgs +property to include additional submit arguments that are appropriate to your cluster. +For more information, refer to the sbatch documentation provided with your cluster. + +communicatingSubmitFcn.m +------------------------ +communicatingSubmitFcn calculates the number of nodes to request from the cluster from the +NumWorkersRange property of the communicating job. Customize the number of +nodes requested to suit your cluster's requirements. + +communicatingSubmitFcn provides the ability to supply additional submit arguments to the +sbatch command. Modify the AdditionalProperties.AdditionalSubmitArgs +property to include additional submit arguments that are appropriate to your cluster. +For more information, refer to the sbatch documentation provided with your cluster. + +communicatingJobWrapper.sh +-------------------------- +communicatingJobWrapper.sh uses the StrictHostKeyChecking=no and UserKnownHostsFile=/dev/null options +for ssh. Customize the ssh options to suit your cluster's requirements. For +more information, refer to your operating system's ssh documentation. + diff --git a/+opencossan/+highperformancecomputing/@JobManagerSlurm/shared/communicatingJobWrapper.sh b/+opencossan/+highperformancecomputing/@JobManagerSlurm/shared/communicatingJobWrapper.sh new file mode 100644 index 00000000..556513fa --- /dev/null +++ b/+opencossan/+highperformancecomputing/@JobManagerSlurm/shared/communicatingJobWrapper.sh @@ -0,0 +1,52 @@ +#!/bin/sh +# This wrapper script is intended to be submitted to Slurm to support +# communicating jobs. +# +# This script uses the following environment variables set by the submit MATLAB code: +# MDCE_CMR - the value of ClusterMatlabRoot (may be empty) +# MDCE_MATLAB_EXE - the MATLAB executable to use +# MDCE_MATLAB_ARGS - the MATLAB args to use +# PARALLEL_SERVER_DEBUG - used to debug problems on the cluster +# +# The following environment variables are forwarded through mpiexec: +# MDCE_DECODE_FUNCTION - the decode function to use +# MDCE_STORAGE_LOCATION - used by decode function +# MDCE_STORAGE_CONSTRUCTOR - used by decode function +# MDCE_JOB_LOCATION - used by decode function +# +# The following environment variables are set by Slurm: +# SLURM_NODELIST - list of hostnames allocated to this Slurm job + +# Copyright 2015-2018 The MathWorks, Inc. + +# Echo the nodes that the scheduler has allocated to this job: +echo The scheduler has allocated the following nodes to this job: ${SLURM_NODELIST:?"Node list undefined"} + +# Create full path to mw_mpiexec if needed. +FULL_MPIEXEC=${MDCE_CMR:+${MDCE_CMR}/bin/}mw_mpiexec + +# Label stdout/stderr with the rank of the process +MPI_VERBOSE=-l + +# Increase the verbosity of mpiexec if PARALLEL_SERVER_DEBUG or MDCE_DEBUG (for backwards compatibility) is true +if [ "X${PARALLEL_SERVER_DEBUG}X" = "XtrueX" ] || [ "X${MDCE_DEBUG}X" = "XtrueX" ]; then +MPI_VERBOSE="${MPI_VERBOSE} -v -print-all-exitcodes" +fi + +# Construct the command to run. +CMD="\"${FULL_MPIEXEC}\" ${MPI_VERBOSE} -n ${MDCE_TOTAL_TASKS} \"${MDCE_MATLAB_EXE}\" ${MDCE_MATLAB_ARGS}" + +# Echo the command so that it is shown in the output log. +echo $CMD + +# Execute the command. +eval $CMD + +MPIEXEC_EXIT_CODE=${?} +if [ ${MPIEXEC_EXIT_CODE} -eq 42 ] ; then + # Get here if user code errored out within MATLAB. Overwrite this to zero in + # this case. + echo "Overwriting MPIEXEC exit code from 42 to zero (42 indicates a user-code failure)" + MPIEXEC_EXIT_CODE=0 +fi +exit ${MPIEXEC_EXIT_CODE} diff --git a/+opencossan/+highperformancecomputing/@JobManagerSlurm/shared/communicatingJobWrapperSmpd.sh b/+opencossan/+highperformancecomputing/@JobManagerSlurm/shared/communicatingJobWrapperSmpd.sh new file mode 100644 index 00000000..117b8967 --- /dev/null +++ b/+opencossan/+highperformancecomputing/@JobManagerSlurm/shared/communicatingJobWrapperSmpd.sh @@ -0,0 +1,258 @@ +#!/bin/sh +# This wrapper script is intended to be submitted to Slurm to support +# communicating jobs. +# +# This script uses the following environment variables set by the submit MATLAB code: +# MDCE_CMR - the value of ClusterMatlabRoot (might be empty) +# MDCE_MATLAB_EXE - the MATLAB executable to use +# MDCE_MATLAB_ARGS - the MATLAB args to use +# +# The following environment variables are forwarded through mpiexec: +# MDCE_DECODE_FUNCTION - the decode function to use +# MDCE_STORAGE_LOCATION - used by decode function +# MDCE_STORAGE_CONSTRUCTOR - used by decode function +# MDCE_JOB_LOCATION - used by decode function + +# The following environment variables are set by Slurm +# SLURM_JOB_ID - number of nodes allocated to Slurm job +# SLURM_JOB_NUM_NODES - number of hosts allocated to Slurm job +# SLURM_JOB_NODELIST - list of hostnames allocated to Slurm job +# SLURM_TASKS_PER_NODE - list containing number of tasks allocated per host to Slurm job + +# Copyright 2015-2018 The MathWorks, Inc. + +# Users of Slurm older than v1.1.34 should uncomment the following code +# to enable mapping from old Slurm environment variables: + +# SLURM_JOB_ID=${SLURM_JOBID} +# SLURM_JOB_NUM_NODES=${SLURM_NNODES} +# SLURM_JOB_NODELIST=${SLURM_NODELIST} + +# Create full paths to mw_smpd/mw_mpiexec if needed +FULL_SMPD=${MDCE_CMR:+${MDCE_CMR}/bin/}mw_smpd +FULL_MPIEXEC=${MDCE_CMR:+${MDCE_CMR}/bin/}mw_mpiexec + +######################################################################################### +# Work out where we need to launch SMPDs given our hosts file - defines SMPD_HOSTS +chooseSmpdHosts() { + + # SLURM_JOB_NODELIST is required: the following line either echoes the value, or aborts. + echo Node file: ${SLURM_JOB_NODELIST:?"Node file undefined"} + + # SMPD_HOSTS is a single line comma separated list of hostnames: + # node136,node138,node140,node141,node142,node143,node157 + # + # Our source of information is SLURM_JOB_NODELIST in the form: + # cnode[136,138],cnode[140-43],cnode157 + # + # 'scontrol show hostname ${SLURM_JOB_NODELIST}' produces multi-line list of hostnames: + # node136 + # node138 + # node140 + # ... + # + # Pipe through "tr" to convert newlines to spaces. + + SMPD_HOSTS=`scontrol show hostname ${SLURM_JOB_NODELIST} | tr '\n', ' '` +} + +######################################################################################### +# Work out which port to use for SMPD +chooseSmpdPort() { + + # Extract the numeric part of SLURM_JOB_ID using sed to choose unique port for SMPD to run on. + # Assumes SLURM_JOB_ID starts with a number, such as: 15.slurm-server-host.domain.com + JOB_NUM=`echo ${SLURM_JOB_ID:?"SLURM_JOB_ID undefined"} | sed 's#^\([0-9][0-9]*\).*$#\1#'` + # Base smpd_port on the numeric part of the above + SMPD_PORT=`expr $JOB_NUM % 10000 + 20000` +} + +######################################################################################### +# Work out how many processes to launch - set MACHINE_ARG +# +# Inputs: +# SLURM_JOB_NUM_NODES Slurm environment variable: Number of nodes allocated to Slurm job +# +# SMPD_HOSTS Comma separated list of hostnames of nodes set by chooseSmpdHosts +# +# SLURM_TASKS_PER_NODE Slurm environment variable: Number of tasks allocated per node. +# If two or more consecutive nodes have the same task count, +# that count is followed by "(x#)" where "#" is the repetition count. +# Output: +# MACHINE_ARG Arguments to pass to mpiexec in the form: +# -hosts host1 tasks_on_host1 host2 tasks_on_host2 +# +# Example +# ------- +# Inputs: +# SLURM_JOB_NUM_NODES 7 +# SMPD_HOSTS node136,node138,node140,node141,node42,node143,node157 +# SLURM_TASKS_PER_NODE 12(x4),7,9(x2) +# Output: +# -hosts 7 node136 12 node138 12 node140 12 node141 12 node142 7 node143 9 node157 9 +# +chooseMachineArg() { + + # Transform SLURM_TASKS_PER_NODE into TASKS_PER_NODE_LIST + # + # Examples: SLURM_TASKS_PER_NODE -> TASKS_PER_NODE_LIST + # ------- -------------------- ------------------- + # Single node has 12 tasks 12 -> 12 + # Three nodes have 12 tasks 12(x3) -> 12,12,12 + # First two nodes have 7 tasks, the third has 8 tasks 7(x2),8 -> 7,7,8 + + TASKS_PER_NODE_LIST='' + # Replace commas with spaces to create space delimited list to use with for loop + LIST_FROM_SLURM=`echo ${SLURM_TASKS_PER_NODE} | sed 's/,/ /g'` + for ITEM in ${LIST_FROM_SLURM} + do + if [ `echo ${ITEM} | grep -e '^[0-9][0-9]*$' -c` -eq 1 ] ; then + # "NUM_TASKS" == "NUM_TASKS(x1)" + NUM_NODES=1 + NUM_TASKS=${ITEM} + else + # "NUM_TASKS(xNUM_NODES)" + NUM_NODES=`echo $ITEM | sed 's/^[0-9][0-9]*(x\([0-9][0-9]*\))$/\1/'` + NUM_TASKS=`echo $ITEM | sed 's/^\([0-9][0-9]*\)(x[0-9][0-9]*)$/\1/'` + fi + + # Repeat NUM_NODES iterations: append NUM_TASKS to TASKS_PER_NODE_LIST + COUNT=0 + while [ ${COUNT} -lt ${NUM_NODES} ] + do + if [ -z "${TASKS_PER_NODE_LIST}" ] ; then + # List empty, therefore adding first item to list - avoid adding comma + TASKS_PER_NODE_LIST=${NUM_TASKS} + else + # Appending to list - add a comma to delimit entries + TASKS_PER_NODE_LIST="${TASKS_PER_NODE_LIST},${NUM_TASKS}" + fi + COUNT=`expr ${COUNT} + 1` + done + done + + # Add -hosts argument at start of MACHINE_ARG + MACHINE_ARG="-hosts ${SLURM_JOB_NUM_NODES}" + + # For each hostname in SMPD_HOSTS, append ' ' to MACHINE_ARG + INDEX=0 + for HOSTNAME in ${SMPD_HOSTS} + do + INDEX=`expr ${INDEX} + 1` + # Use cut to index the '${INDEX}th' item in TASKS_PER_NODE_LIST + TASKS_PER_NODE=`echo ${TASKS_PER_NODE_LIST} | cut -f ${INDEX} -d,` + MACHINE_ARG="${MACHINE_ARG} ${HOSTNAME} ${TASKS_PER_NODE}" + done + echo "Machine args: $MACHINE_ARG" +} + +######################################################################################### +# Shut down SMPDs and exit with the exit code of the last command executed +cleanupAndExit() { + EXIT_CODE=${?} + + echo "Stopping SMPD ..." + + STOP_SMPD_CMD="srun --ntasks-per-node=1 --ntasks=${SLURM_JOB_NUM_NODES} ${FULL_SMPD} -shutdown -phrase MATLAB -port ${SMPD_PORT}" + echo $STOP_SMPD_CMD + eval $STOP_SMPD_CMD + + echo "Exiting with code: ${EXIT_CODE}" + exit ${EXIT_CODE} +} + +######################################################################################### +# Use srun to launch the SMPD daemons on each processor +launchSmpds() { + + # Launch the SMPD processes on all hosts using srun + echo "Starting SMPD on ${SMPD_HOSTS} ..." + + START_SMPD_CMD="srun --ntasks-per-node=1 --ntasks=${SLURM_JOB_NUM_NODES} ${FULL_SMPD} -phrase MATLAB -port ${SMPD_PORT} -debug 0 &" + echo $START_SMPD_CMD + eval $START_SMPD_CMD + + # Check that the SMPD processes are running on all hosts + SUCCESS=0 + NUM_ATTEMPTS=60 + ATTEMPT=1 + while [ ${ATTEMPT} -le ${NUM_ATTEMPTS} ] + do + echo "Checking that SMPD processes are running (Attempt ${ATTEMPT} of ${NUM_ATTEMPTS})" + SMPD_LAUNCHED_HOSTS="" + NUM_HOSTS_FOUND=0 + for HOST in ${SMPD_HOSTS} + do + CHECK_SMPD_CMD="${FULL_SMPD} -phrase MATLAB -port ${SMPD_PORT} -status ${HOST} > /dev/null 2>&1" + echo $CHECK_SMPD_CMD + eval $CHECK_SMPD_CMD + EXIT_CODE=${?} + if [ $EXIT_CODE -ne 0 ]; then + echo "No SMPD process running on ${HOST}" + else + echo "SMPD process found running on ${HOST}" + NUM_HOSTS_FOUND=$((NUM_HOSTS_FOUND+1)) + + # Append HOST to SMPD_LAUNCHED_HOSTS if it does not already contain it. + case "${SMPD_LAUNCHED_HOSTS}" in + *$HOST* ) ;; + * ) SMPD_LAUNCHED_HOSTS="${SMPD_LAUNCHED_HOSTS} ${HOST}" ;; + esac + fi + done + if [ ${SLURM_JOB_NUM_NODES} -eq ${NUM_HOSTS_FOUND} ] ; then + SUCCESS=1 + break + elif [ ${ATTEMPT} -ne ${NUM_ATTEMPTS} ] ; then + sleep 1 + fi + ATTEMPT=$((ATTEMPT+1)) + done + if [ $SUCCESS -ne 1 ] ; then + if [ $NUM_HOSTS_FOUND -eq 0 ] ; then + echo "No SMPD processes were found running. Aborting." + else + echo "Found SMPD processes running on only ${NUM_HOSTS_FOUND} of ${SLURM_JOB_NUM_NODES} nodes. Aborting." + echo "Hosts found: ${SMPD_LAUNCHED_HOSTS}" + fi + exit 1 + fi + echo "All SMPDs launched" +} + +######################################################################################### +runMpiexec() { + + CMD="\"${FULL_MPIEXEC}\" -smpd -phrase MATLAB -port ${SMPD_PORT} \ + -l ${MACHINE_ARG} -genvlist \ + MDCE_DECODE_FUNCTION,MDCE_STORAGE_LOCATION,MDCE_STORAGE_CONSTRUCTOR,MDCE_JOB_LOCATION,MDCE_DEBUG,MDCE_LICENSE_NUMBER,MLM_WEB_LICENSE,MLM_WEB_USER_CRED,MLM_WEB_ID \ + \"${MDCE_MATLAB_EXE}\" ${MDCE_MATLAB_ARGS}" + + # As a debug stage: echo the command ... + echo $CMD + + # ... and then execute it. + eval $CMD + + MPIEXEC_CODE=${?} + if [ ${MPIEXEC_CODE} -ne 0 ] ; then + exit ${MPIEXEC_CODE} + fi +} + +######################################################################################### +# Define the order in which we execute the stages defined above +MAIN() { + # Install a trap to ensure that SMPDs are closed if something errors or the + # job is cancelled. + trap "cleanupAndExit" 0 1 2 15 + chooseSmpdHosts + chooseSmpdPort + launchSmpds + chooseMachineArg + runMpiexec + exit 0 # Explicitly exit 0 to trigger cleanupAndExit +} + +# Call the MAIN loop +MAIN diff --git a/+opencossan/+highperformancecomputing/@JobManagerSlurm/shared/communicatingSubmitFcn.m b/+opencossan/+highperformancecomputing/@JobManagerSlurm/shared/communicatingSubmitFcn.m new file mode 100644 index 00000000..5fc8541b --- /dev/null +++ b/+opencossan/+highperformancecomputing/@JobManagerSlurm/shared/communicatingSubmitFcn.m @@ -0,0 +1,136 @@ +function communicatingSubmitFcn(cluster, job, environmentProperties) +%COMMUNICATINGSUBMITFCN Submit a communicating MATLAB job to a Slurm cluster +% +% Set your cluster's IntegrationScriptsLocation to the parent folder of this +% function to run it when you submit a communicating job. +% +% See also parallel.cluster.generic.communicatingDecodeFcn. + +% Copyright 2010-2018 The MathWorks, Inc. + +% Store the current filename for the errors, warnings and dctSchedulerMessages +currFilename = mfilename; +if ~isa(cluster, 'parallel.Cluster') + error('parallelexamples:GenericSLURM:NotClusterObject', ... + 'The function %s is for use with clusters created using the parcluster command.', currFilename) +end + +decodeFunction = 'parallel.cluster.generic.communicatingDecodeFcn'; + +if ~cluster.HasSharedFilesystem + error('parallelexamples:GenericSLURM:NotSharedFileSystem', ... + 'The function %s is for use with shared filesystems.', currFilename) +end + + +if ~strcmpi(cluster.OperatingSystem, 'unix') + error('parallelexamples:GenericSLURM:UnsupportedOS', ... + 'The function %s only supports clusters with unix OS.', currFilename) +end + +enableDebug = 'false'; +if isprop(cluster.AdditionalProperties, 'EnableDebug') ... + && islogical(cluster.AdditionalProperties.EnableDebug) ... + && cluster.AdditionalProperties.EnableDebug + enableDebug = 'true'; +end + +% The job specific environment variables +% Remove leading and trailing whitespace from the MATLAB arguments +matlabArguments = strtrim(environmentProperties.MatlabArguments); +variables = {'MDCE_DECODE_FUNCTION', decodeFunction; ... + 'MDCE_STORAGE_CONSTRUCTOR', environmentProperties.StorageConstructor; ... + 'MDCE_JOB_LOCATION', environmentProperties.JobLocation; ... + 'MDCE_MATLAB_EXE', environmentProperties.MatlabExecutable; ... + 'MDCE_MATLAB_ARGS', matlabArguments; ... + 'PARALLEL_SERVER_DEBUG', enableDebug; ... + 'MLM_WEB_LICENSE', environmentProperties.UseMathworksHostedLicensing; ... + 'MLM_WEB_USER_CRED', environmentProperties.UserToken; ... + 'MLM_WEB_ID', environmentProperties.LicenseWebID; ... + 'MDCE_LICENSE_NUMBER', environmentProperties.LicenseNumber; ... + 'MDCE_STORAGE_LOCATION', environmentProperties.StorageLocation; ... + 'MDCE_CMR', cluster.ClusterMatlabRoot; ... + 'MDCE_TOTAL_TASKS', num2str(environmentProperties.NumberOfTasks); ... + 'MDCE_NUM_THREADS', num2str(cluster.NumThreads)}; +% Set each environment variable to newValue if currentValue differs. +% We must do this particularly when newValue is an empty value, +% to be sure that we clear out old values from the environment. +for ii = 1:size(variables, 1) + variableName = variables{ii,1}; + currentValue = getenv(variableName); + newValue = variables{ii,2}; + if ~strcmp(currentValue, newValue) + setenv(variableName, newValue); + end +end + +% Deduce the correct quote to use based on the OS of the current machine +if ispc + quote = '"'; +else + quote = ''''; +end + +% Specify the job wrapper script to use. +if isprop(cluster.AdditionalProperties, 'UseSmpd') && cluster.AdditionalProperties.UseSmpd + scriptName = 'communicatingJobWrapperSmpd.sh'; +else + scriptName = 'communicatingJobWrapper.sh'; +end +% The wrapper script is in the same directory as this file +dirpart = fileparts(mfilename('fullpath')); +quotedScriptName = sprintf('%s%s%s', quote, fullfile(dirpart, scriptName), quote); + +% Choose a file for the output. Please note that currently, JobStorageLocation refers +% to a directory on disk, but this may change in the future. +logFile = cluster.getLogLocation(job); +quotedLogFile = sprintf('%s%s%s', quote, logFile, quote); + +jobName = sprintf('Job%d', job.ID); +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% CUSTOMIZATION MAY BE REQUIRED %% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% You might want to customize this section to match your cluster, +% for example to limit the number of nodes for a single job. +%additionalSubmitArgs = sprintf('--ntasks=%d --cpus-per-task=%d', environmentProperties.NumberOfTasks, cluster.NumThreads); +additionalSubmitArgs = sprintf('--nodes=%d --tasks-per-node=%d -t %s -p %s -A %s',... + cluster.AdditionalProperties.Nodes, cluster.AdditionalProperties.Ppn,... + cluster.AdditionalProperties.Time, cluster.AdditionalProperties.Queue,... + cluster.AdditionalProperties.Aname); +commonSubmitArgs = getCommonSubmitArgs(cluster); +if ~isempty(commonSubmitArgs) && ischar(commonSubmitArgs) + additionalSubmitArgs = strtrim([additionalSubmitArgs, ' ', commonSubmitArgs]); +end +dctSchedulerMessage(5, '%s: Generating command for task %i', currFilename, ii); +commandToRun = getSubmitString(jobName, quotedLogFile, quotedScriptName, ... + additionalSubmitArgs); + +% Now ask the cluster to run the submission command +dctSchedulerMessage(4, '%s: Submitting job using command:\n\t%s', currFilename, commandToRun); +try + % Make the shelled out call to run the command. + [cmdFailed, cmdOut] = system(commandToRun); +catch err + cmdFailed = true; + cmdOut = err.message; +end +if cmdFailed + error('parallelexamples:GenericSLURM:SubmissionFailed', ... + 'Submit failed with the following message:\n%s', cmdOut); +end + +dctSchedulerMessage(1, '%s: Job output will be written to: %s\nSubmission output: %s\n', currFilename, logFile, cmdOut); + +jobIDs = extractJobId(cmdOut); +% jobIDs must be a cell array +if isempty(jobIDs) + warning('parallelexamples:GenericSLURM:FailedToParseSubmissionOutput', ... + 'Failed to parse the job identifier from the submission output: "%s"', ... + cmdOut); +end +if ~iscell(jobIDs) + jobIDs = {jobIDs}; +end + +% set the job ID on the job cluster data +cluster.setJobClusterData(job, struct('ClusterJobIDs', {jobIDs})); diff --git a/+opencossan/+highperformancecomputing/@JobManagerSlurm/shared/extractJobId.m b/+opencossan/+highperformancecomputing/@JobManagerSlurm/shared/extractJobId.m new file mode 100644 index 00000000..4c42cf9f --- /dev/null +++ b/+opencossan/+highperformancecomputing/@JobManagerSlurm/shared/extractJobId.m @@ -0,0 +1,33 @@ +function jobID = extractJobId(sbatchCommandOutput) +% Extracts the job ID from the sbatch command output for Slurm + +% Copyright 2015-2017 The MathWorks, Inc. + +% Output from sbatch expected to be in the following format: +% Submitted batch job 12345 +% +% sbatch could also attach a warning to the output, such as: +% +% sbatch: Warning: can't run 1 processes on 3 nodes, setting nnodes to 1 +% Submitted batch job 12346 + +% Trim sbatch command output for use in debug message +trimmedCommandOutput = strtrim(sbatchCommandOutput); + +% Ignore anything before or after 'Submitted batch job ###', and extract the numeric value. +searchPattern = '.*Submitted batch job ([0-9]+).*'; + +% When we match searchPattern, matchedTokens is a single entry cell array containing the jobID. +% Otherwise we failed to match searchPattern, so matchedTokens is an empty cell array. +matchedTokens = regexp(sbatchCommandOutput, searchPattern, 'tokens', 'once'); + +if isempty(matchedTokens) + % Callers check for error in extracting Job ID using isempty() on return value. + jobID = ''; + opencossan.OpenCossan.cossanDisp(sprintf('%s: Failed to extract Job ID from sbatch output: \n\t%s',... + mfilename, trimmedCommandOutput),1); +else + jobID = matchedTokens{1}; + opencossan.OpenCossan.cossanDisp(sprintf('%s: Job ID %s was extracted from sbatch output: \n\t%s', ... + mfilename, jobID, trimmedCommandOutput),1); +end diff --git a/+opencossan/+highperformancecomputing/@JobManagerSlurm/shared/getCommonSubmitArgs.m b/+opencossan/+highperformancecomputing/@JobManagerSlurm/shared/getCommonSubmitArgs.m new file mode 100644 index 00000000..8938cff7 --- /dev/null +++ b/+opencossan/+highperformancecomputing/@JobManagerSlurm/shared/getCommonSubmitArgs.m @@ -0,0 +1,18 @@ +function commonSubmitArgs = getCommonSubmitArgs(cluster) +% Get any additional submit arguments for the Slurm sbatch command +% that are common to both independent and communicating jobs. + +% Copyright 2016-2017 The MathWorks, Inc. + +commonSubmitArgs = ''; + +% Append any arguments provided by the AdditionalSubmitArgs field of cluster.AdditionalProperties. +if isprop(cluster.AdditionalProperties, 'AdditionalSubmitArgs') + extraArgs = cluster.AdditionalProperties.AdditionalSubmitArgs; + if ~isempty(extraArgs) && ischar(extraArgs) + commonSubmitArgs = strtrim([commonSubmitArgs, ' ', extraArgs]); + end +end + +% You may wish to support further cluster.AdditionalProperties fields here +% and modify the submission command arguments accordingly. diff --git a/+opencossan/+highperformancecomputing/@JobManagerSlurm/shared/getJobStateFcn.m b/+opencossan/+highperformancecomputing/@JobManagerSlurm/shared/getJobStateFcn.m new file mode 100644 index 00000000..25bc966e --- /dev/null +++ b/+opencossan/+highperformancecomputing/@JobManagerSlurm/shared/getJobStateFcn.m @@ -0,0 +1,110 @@ +function state = getJobStateFcn(cluster, job, state) +%GETJOBSTATEFCN Gets the state of a job from Slurm +% +% Set your cluster's IntegrationScriptsLocation to the parent folder of this +% function to run it when you query the state of a job. + +% Copyright 2010-2018 The MathWorks, Inc. + +% Store the current filename for the errors, warnings and dctSchedulerMessages +currFilename = mfilename; +if ~isa(cluster, 'parallel.Cluster') + error('parallelexamples:GenericSLURM:SubmitFcnError', ... + 'The function %s is for use with clusters created using the parcluster command.', currFilename) +end +if ~cluster.HasSharedFilesystem + error('parallelexamples:GenericSLURM:NotSharedFileSystem', ... + 'The function %s is for use with shared filesystems.', currFilename) +end + +% Shortcut if the job state is already finished or failed +jobInTerminalState = strcmp(state, 'finished') || strcmp(state, 'failed'); +if jobInTerminalState + return +end +% Get the information about the actual cluster used +data = cluster.getJobClusterData(job); +if isempty(data) + % This indicates that the job has not been submitted, so just return + dctSchedulerMessage(1, '%s: Job cluster data was empty for job with ID %d.', currFilename, job.ID); + return +end +try + jobIDs = data.ClusterJobIDs; +catch err + ex = MException('parallelexamples:GenericSLURM:FailedToRetrieveJobID', ... + 'Failed to retrieve clusters''s job IDs from the job cluster data.'); + ex = ex.addCause(err); + throw(ex); +end + + +% Get the top level job state from sacct +% We are not using squeue because it omits information about completed/terminated jobs. +% We use the '--allocations' option to request cumulative statistics for each job. +% (Normal output of sacct includes intermediate Slurm job steps which we don't want.) +% If the Slurm JobID counter has been reset, there is a short period of time after a job +% has been submitted that sacct may return information on an old job with the same JobID. +% The option '--user=$USER' makes this less likely, as it will only happen if the old job +% was submitted by the same user. +jobList = sprintf('-j ''%s'' ', jobIDs{:}); +commandToRun = sprintf('sacct --allocations --user=$USER %s', jobList); +dctSchedulerMessage(4, '%s: Querying cluster for job state using command:\n\t%s', currFilename, commandToRun); + +try + % We will ignore the status returned from the state command because + % a non-zero status is returned if the job no longer exists + % Make the shelled out call to run the command. + [~, cmdOut] = system(commandToRun); +catch err + ex = MException('parallelexamples:GenericSLURM:FailedToGetJobState', ... + 'Failed to get job state from cluster.'); + ex = ex.addCause(err); + throw(ex); +end + +clusterState = iExtractJobState(cmdOut, numel(jobIDs)); +dctSchedulerMessage(6, '%s: State %s was extracted from cluster output.', currFilename, clusterState); + +% If we could determine the cluster's state, we'll use that, otherwise +% stick with MATLAB's job state. +if ~strcmp(clusterState, 'unknown') + state = clusterState; +end + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +function state = iExtractJobState(sacctOut, numJobs) +% Function to extract the job state from the output of sacct + +numPending = numel(regexp(sacctOut, 'PENDING')); +numRunning = numel(regexp(sacctOut, 'RUNNING|SUSPENDED|COMPLETING|CONFIGURING')); +numFinished = numel(regexp(sacctOut, 'COMPLETED')); +numFailed = numel(regexp(sacctOut, 'CANCELLED|FAILED|TIMEOUT|PREEMPTED|NODE_FAIL')); + +% If all of the jobs that we asked about have finished, then we know the job has finished. +if numFinished == numJobs + state = 'finished'; + return +end + +% Any running indicates that the job is running +if numRunning > 0 + state = 'running'; + return +end + +% We know numRunning == 0 so if there are some still pending then the +% job must be queued again, even if there are some finished +if numPending > 0 + state = 'queued'; + return +end + +% Deal with any tasks that have failed +if numFailed > 0 + % Set this job to be failed + state = 'failed'; + return +end + +state = 'unknown'; diff --git a/+opencossan/+highperformancecomputing/@JobManagerSlurm/shared/independentJobWrapper.sh b/+opencossan/+highperformancecomputing/@JobManagerSlurm/shared/independentJobWrapper.sh new file mode 100644 index 00000000..e487e6a5 --- /dev/null +++ b/+opencossan/+highperformancecomputing/@JobManagerSlurm/shared/independentJobWrapper.sh @@ -0,0 +1,15 @@ +#!/bin/sh +# This wrapper script is intended to support independent execution. +# +# This script uses the following environment variables set by the submit MATLAB code: +# MDCE_MATLAB_EXE - the MATLAB executable to use +# MDCE_MATLAB_ARGS - the MATLAB args to use +# + +# Copyright 2010-2018 The MathWorks, Inc. + +echo "Executing: ${MDCE_MATLAB_EXE} ${MDCE_MATLAB_ARGS}" +eval "${MDCE_MATLAB_EXE}" ${MDCE_MATLAB_ARGS} +EXIT_CODE=${?} +echo "Exiting with code: ${EXIT_CODE}" +exit ${EXIT_CODE} diff --git a/+opencossan/+highperformancecomputing/@JobManagerSlurm/submitJob.m b/+opencossan/+highperformancecomputing/@JobManagerSlurm/submitJob.m new file mode 100644 index 00000000..287af46e --- /dev/null +++ b/+opencossan/+highperformancecomputing/@JobManagerSlurm/submitJob.m @@ -0,0 +1,122 @@ +function JobObject=submitJob(obj,varargin) +%INDEPENDENTSUBMITFCN Submit a job to a Slurm cluster +% +% Set your cluster's IntegrationScriptsLocation to the parent folder of this +% function to run it when you submit an independent job. +% +% See also: JobManagerSlurm + +% Store the current filename for the errors and warnings +currFilename = mfilename; + +%% check resources available +[areAvailable, errorMsg]=obj.checkResources(varargin); + +if ~areAvailable + error('JobManagerSlurm:SubmissionFailed', ... + 'Resources not available on the defined cluster:\n%s', errorMsg); +end + +[requiredArgs, varargin] = opencossan.common.utilities.parseRequiredNameValuePairs(... + ["Hostname","Queue"], varargin{:}); + +Sscriptname = Xobj.prepareScript(Number); + +%TODO: I don't know the meanining of quotedCommand. Integrate to the object +%or remove it. +additionalSubmitArgs=''; +quotedCommand=''; +commandToRun = getSubmitString(obj, quotedCommand, additionalSubmitArgs); + +if obj.isRemoteCluster + % copy the script file to the remote host + obj.SSHconnection.putFile('SlocalFileName',fullfile(obj.JobStorageLocation,Sscriptname),... + 'DestinationFolder',obj.WorkingPath); + opencossan.OpenCossan.cossanDisp(sprintf('%s: Copyfile file: %s\n\tto remote cluster:\%s.',... + currFilename, fullfile(obj.JobStorageLocation,Sscriptname),obj.WorkingPath),3); + + [cmdFailed, cmdOut] = obj.SSHconnection.runCommand(commandToRun); + +else + try + % Make the shelled out call to run the command. + [cmdFailed, cmdOut] = system(commandToRun); + catch err + cmdFailed = true; + cmdOut = err.message; + end +end + +if cmdFailed + error('JobManagerSlurm:SubmissionFailed', ... + 'Submit failed with the following message:\n%s', cmdOut); +end + +% Extract jobID and store into a Job object +jobID=obj.extractJobId(cmdOut); +jobState=obj.getJobState(sbatchCommandOutput); + +JobObject=Job('SscriptName',Sscriptname,'ID',jobID,'Name','to be fixed','State',jobState); + + +% % The wrapper script is in the same directory as this file +% dirpart = fileparts(mfilename('fullpath')); +% quotedScriptName = sprintf('%s%s%s', quote, fullfile(dirpart, scriptName), quote); +% +% % Get the tasks for use in the loop +% tasks = JobObject.Tasks; +% numberOfTasks = environmentProperties.NumberOfTasks; +% jobIDs = cell(numberOfTasks, 1); +% % Loop over every task we have been asked to submit +% for ii = 1:numberOfTasks +% taskLocation = environmentProperties.TaskLocations{ii}; +% % Set the environment variable that defines the location of this task +% setenv('MDCE_TASK_LOCATION', taskLocation); +% +% % Choose a file for the output. Please note that currently, JobStorageLocation refers +% % to a directory on disk, but this may change in the future. +% logFile = obj.getLogLocation(tasks(ii)); +% quotedLogFile = sprintf('%s%s%s', quote, logFile, quote); +% +% % Submit one task at a time +% jobName = sprintf('Job%d.%d', JobObject.ID, JobObject.tasks(ii).ID); +% +% additionalSubmitArgs = sprintf('--ntasks=1 --tasks-per-node=1 -A %s -t %s -p %s',... +% obj.AdditionalProperties.Aname,obj.AdditionalProperties.Time,obj.AdditionalProperties.Queue); +% +% commonSubmitArgs = getCommonSubmitArgs(obj); +% if ~isempty(commonSubmitArgs) && ischar(commonSubmitArgs) +% additionalSubmitArgs = strtrim([additionalSubmitArgs, ' ', commonSubmitArgs]); +% end +% dctSchedulerMessage(5, '%s: Generating command for task %i', currFilename, ii); +% commandToRun = getSubmitString(jobName, quotedLogFile, quotedScriptName, ... +% additionalSubmitArgs); +% +% % Now ask the cluster to run the submission command +% dctSchedulerMessage(4, '%s: Submitting job using command:\n\t%s', currFilename, commandToRun); +% try +% % Make the shelled out call to run the command. +% [cmdFailed, cmdOut] = system(commandToRun); +% catch err +% cmdFailed = true; +% cmdOut = err.message; +% end +% if cmdFailed +% error('parallelexamples:GenericSLURM:SubmissionFailed', ... +% 'Submit failed with the following message:\n%s', cmdOut); +% end +% +% dctSchedulerMessage(1, '%s: Job output will be written to: %s\nSubmission output: %s\n', currFilename, logFile, cmdOut); +% jobIDs{ii} = extractJobId(cmdOut); +% +% if isempty(jobIDs{ii}) +% warning('parallelexamples:GenericSLURM:FailedToParseSubmissionOutput', ... +% 'Failed to parse the job identifier from the submission output: "%s"', ... +% cmdOut); +% end +% end + + + + + From 4170d0061ca53defa2121654a4864789df20574d Mon Sep 17 00:00:00 2001 From: Edoardo Patelli Date: Thu, 21 May 2020 14:50:22 +0100 Subject: [PATCH 6/9] add slurm support (initial implementation) --- .../+highperformancecomputing/@Job/Job.m | 3 +- .../@JobManager/JobManager.m | 30 ++-- .../@JobManagerSlurm/JobManagerSlurm.m | 22 ++- .../@JobManagerSlurm/extractJobId.m | 2 +- .../@JobManagerSlurm/getSubmitString.m | 4 +- .../@JobManagerSlurm/shared/extractJobId.m | 33 ---- .../@JobManagerSlurm/shared/getJobStateFcn.m | 15 +- .../@ProbabilisticModel/ProbabilisticModel.m | 19 ++- .../computeFailureProbability.m | 2 +- +opencossan/+workers/@Evaluator/Evaluator.m | 22 +-- +opencossan/+workers/@Evaluator/add.m | 2 +- .../JobManagerSlurmTest.m | 160 ++++++++++++++++++ .../+highperformancecomputing/JobTest.m | 61 +++++++ .../unit/+opencossan/+workers/EvaluatorTest.m | 20 +-- 14 files changed, 287 insertions(+), 108 deletions(-) delete mode 100644 +opencossan/+highperformancecomputing/@JobManagerSlurm/shared/extractJobId.m create mode 100644 test/unit/+opencossan/+highperformancecomputing/JobManagerSlurmTest.m create mode 100644 test/unit/+opencossan/+highperformancecomputing/JobTest.m diff --git a/+opencossan/+highperformancecomputing/@Job/Job.m b/+opencossan/+highperformancecomputing/@Job/Job.m index 41920e12..f056e01a 100644 --- a/+opencossan/+highperformancecomputing/@Job/Job.m +++ b/+opencossan/+highperformancecomputing/@Job/Job.m @@ -41,7 +41,7 @@ ["ID","Status"], varargin{:}); [optionalArgs, superArg] = opencossan.common.utilities.parseOptionalNameValuePairs(... - ["Name","Dependences"],{[],[]}, varargin{:}); + ["Name","Dependences" "ScriptName"],{[],[],[]}, varargin{:}); end obj@opencossan.common.CossanObject(superArg{:}); @@ -51,6 +51,7 @@ obj.State=requiredArgs.state; obj.Name=optionalArgs.name; obj.Dependences=optionalArgs.dependences; + obj.ScriptName=optionalArgs.scriptname; end end diff --git a/+opencossan/+highperformancecomputing/@JobManager/JobManager.m b/+opencossan/+highperformancecomputing/@JobManager/JobManager.m index dbf6687b..0a752829 100644 --- a/+opencossan/+highperformancecomputing/@JobManager/JobManager.m +++ b/+opencossan/+highperformancecomputing/@JobManager/JobManager.m @@ -29,10 +29,10 @@ properties PreExeCmd % prepocessor executeble cmd TODO: check with connector PostExeCmd % postprocessor executeble cmd TODO: check with connector - WorkingPath % directory when the job will be executed (better if not on NFS!) - MainInputPath + WorkingPath % directory when the job will be executed (better if not on NFS!) + MainInputPath % SubFolder % Subfolder name for the batch execution - HasSharedFilesystem true % + HasSharedFilesystem(1,1) logical % ClusterMatlabRoot OperatingSystem JobStorageLocation @@ -73,8 +73,9 @@ "PostExeCmd",[];... "WorkingPath",opencossan.OpenCossan.getWorkingPath;... "MainInputPath",[];... - "SubFolder",datestr(now,30),... - "HasSharedFilesystem",true,... + "OperatingSystem",'unix';... + "SubFolder",datestr(now,30);... + "HasSharedFilesystem",true;... "LogFile","OpenCossanJob.log"}; [optionalArgs, superArg] = opencossan.common.utilities.parseOptionalNameValuePairs(... @@ -108,21 +109,16 @@ end - jobID = submitJob(obj,varargin) % submit single job - jobID = submitAnalysis(obj,Samples) % submit analysis (multiple jobs) + jobObj = submitJob(obj,varargin) % submit single job + jobObj = submitAnalysis(obj,Samples) % submit analysis (multiple jobs) % Define function requirements for subclasses - isOK=cancelJob(obj,JobObject) - State = getJobState(Xobj,JobObject) - Lsuccessfull = checkSuccessfulJobs(obj,ScheckFileName) - Status=checkCluster(obj,varargin - - - end - - methods (static) - jobId=extractJobId(sbatchCommandOutput) + isOK=cancelJob(obj,JobObject) % delete job from the job manager + state = getJobState(Xobj,JobObject) % Get state of the Job + clusterStatus=checkCluster(obj,varargin)% Check status of the cluster + % Extract job ID + jobID=extractJobId(obj,sbatchCommandOutput) end end diff --git a/+opencossan/+highperformancecomputing/@JobManagerSlurm/JobManagerSlurm.m b/+opencossan/+highperformancecomputing/@JobManagerSlurm/JobManagerSlurm.m index 5b4c6203..ff893298 100644 --- a/+opencossan/+highperformancecomputing/@JobManagerSlurm/JobManagerSlurm.m +++ b/+opencossan/+highperformancecomputing/@JobManagerSlurm/JobManagerSlurm.m @@ -1,4 +1,4 @@ -classdef(Abstract) JobManagerSlurm < opencossan.highperformancecomputing.JobManager +classdef JobManagerSlurm < opencossan.highperformancecomputing.JobManager % This class defines the interface with the cluster/cloud computing. % The requested computation are automatically converted to jobs and % submitted to the Job management software on the cluster. The results are then retrieved and @@ -27,9 +27,13 @@ % ===================================================================== properties - + ModuleList % list of module to load + AdditionalSubmitArgs % Custumisation arguments for submitting jobs + end + + properties (Dependent) + quotedCommand end - methods @@ -42,20 +46,20 @@ % Process optional paramets OptionalsArguments={... - "Undefined", [];}; + ["moduleList","AdditionalSubmitArgs"], {[] []};}; - [~, superArg] = opencossan.common.utilities.parseOptionalNameValuePairs(... + [optionalArgs, superArg] = opencossan.common.utilities.parseOptionalNameValuePairs(... [OptionalsArguments{:,1}],OptionalsArguments(:,2), varargin{:}); end obj@opencossan.highperformancecomputing.JobManager(superArg{:}); + if optionalArgs > 0 + obj.ModuleList=optionalArgs.modulelist; + obj.AdditionalSubmitArgs=optionalArgs.additionalsubmitargs; + end end end - - methods (private) - jobID = extractJobId(obj,sbatchCommandOutput) - end end diff --git a/+opencossan/+highperformancecomputing/@JobManagerSlurm/extractJobId.m b/+opencossan/+highperformancecomputing/@JobManagerSlurm/extractJobId.m index fe8baf40..3f51f981 100644 --- a/+opencossan/+highperformancecomputing/@JobManagerSlurm/extractJobId.m +++ b/+opencossan/+highperformancecomputing/@JobManagerSlurm/extractJobId.m @@ -1,4 +1,4 @@ -function jobID = extractJobId(sbatchCommandOutput) +function jobID = extractJobId(~,sbatchCommandOutput) % Extracts the job ID from the sbatch command output for Slurm % Adapted from matlab-roll % https://github.com/cossan-working-group/matlab-roll diff --git a/+opencossan/+highperformancecomputing/@JobManagerSlurm/getSubmitString.m b/+opencossan/+highperformancecomputing/@JobManagerSlurm/getSubmitString.m index ec51b7a1..de97e4dd 100644 --- a/+opencossan/+highperformancecomputing/@JobManagerSlurm/getSubmitString.m +++ b/+opencossan/+highperformancecomputing/@JobManagerSlurm/getSubmitString.m @@ -1,7 +1,7 @@ -function submitString = getSubmitString(obj, quotedCommand, additionalSubmitArgs) +function submitString = getSubmitString(obj) %GETSUBMITSTRING Gets the correct sbatch command for a Slurm cluster % Copyright 2010-2016 The MathWorks, Inc. submitString = sprintf('sbatch --job-name=%s --output=%s %s %s', ... - obj.jobName, obj.LogFile, additionalSubmitArgs, quotedCommand); + obj.jobName, obj.LogFile, obj.additionalSubmitArgs, obj.quotedCommand); diff --git a/+opencossan/+highperformancecomputing/@JobManagerSlurm/shared/extractJobId.m b/+opencossan/+highperformancecomputing/@JobManagerSlurm/shared/extractJobId.m deleted file mode 100644 index 4c42cf9f..00000000 --- a/+opencossan/+highperformancecomputing/@JobManagerSlurm/shared/extractJobId.m +++ /dev/null @@ -1,33 +0,0 @@ -function jobID = extractJobId(sbatchCommandOutput) -% Extracts the job ID from the sbatch command output for Slurm - -% Copyright 2015-2017 The MathWorks, Inc. - -% Output from sbatch expected to be in the following format: -% Submitted batch job 12345 -% -% sbatch could also attach a warning to the output, such as: -% -% sbatch: Warning: can't run 1 processes on 3 nodes, setting nnodes to 1 -% Submitted batch job 12346 - -% Trim sbatch command output for use in debug message -trimmedCommandOutput = strtrim(sbatchCommandOutput); - -% Ignore anything before or after 'Submitted batch job ###', and extract the numeric value. -searchPattern = '.*Submitted batch job ([0-9]+).*'; - -% When we match searchPattern, matchedTokens is a single entry cell array containing the jobID. -% Otherwise we failed to match searchPattern, so matchedTokens is an empty cell array. -matchedTokens = regexp(sbatchCommandOutput, searchPattern, 'tokens', 'once'); - -if isempty(matchedTokens) - % Callers check for error in extracting Job ID using isempty() on return value. - jobID = ''; - opencossan.OpenCossan.cossanDisp(sprintf('%s: Failed to extract Job ID from sbatch output: \n\t%s',... - mfilename, trimmedCommandOutput),1); -else - jobID = matchedTokens{1}; - opencossan.OpenCossan.cossanDisp(sprintf('%s: Job ID %s was extracted from sbatch output: \n\t%s', ... - mfilename, jobID, trimmedCommandOutput),1); -end diff --git a/+opencossan/+highperformancecomputing/@JobManagerSlurm/shared/getJobStateFcn.m b/+opencossan/+highperformancecomputing/@JobManagerSlurm/shared/getJobStateFcn.m index 25bc966e..583d00c8 100644 --- a/+opencossan/+highperformancecomputing/@JobManagerSlurm/shared/getJobStateFcn.m +++ b/+opencossan/+highperformancecomputing/@JobManagerSlurm/shared/getJobStateFcn.m @@ -1,4 +1,4 @@ -function state = getJobStateFcn(cluster, job, state) +function state = getJobStateFcn(obj, job) %GETJOBSTATEFCN Gets the state of a job from Slurm % % Set your cluster's IntegrationScriptsLocation to the parent folder of this @@ -8,25 +8,22 @@ % Store the current filename for the errors, warnings and dctSchedulerMessages currFilename = mfilename; -if ~isa(cluster, 'parallel.Cluster') +if ~isa(obj, 'parallel.Cluster') error('parallelexamples:GenericSLURM:SubmitFcnError', ... 'The function %s is for use with clusters created using the parcluster command.', currFilename) end -if ~cluster.HasSharedFilesystem - error('parallelexamples:GenericSLURM:NotSharedFileSystem', ... - 'The function %s is for use with shared filesystems.', currFilename) -end + % Shortcut if the job state is already finished or failed -jobInTerminalState = strcmp(state, 'finished') || strcmp(state, 'failed'); +jobInTerminalState = strcmp(job.state, 'finished') || strcmp(job.state, 'failed'); if jobInTerminalState return end % Get the information about the actual cluster used -data = cluster.getJobClusterData(job); +data = obj.getJobClusterData(job); if isempty(data) % This indicates that the job has not been submitted, so just return - dctSchedulerMessage(1, '%s: Job cluster data was empty for job with ID %d.', currFilename, job.ID); + pencossan.OpenCossan.cossanDisp(sprintf('%s: Job cluster data was empty for job with ID %d.', currFilename, job.ID,1)); return end try diff --git a/+opencossan/+reliability/@ProbabilisticModel/ProbabilisticModel.m b/+opencossan/+reliability/@ProbabilisticModel/ProbabilisticModel.m index 0025af1f..c36104ec 100644 --- a/+opencossan/+reliability/@ProbabilisticModel/ProbabilisticModel.m +++ b/+opencossan/+reliability/@ProbabilisticModel/ProbabilisticModel.m @@ -38,17 +38,18 @@ if nargin == 0 super_args = {}; else - [required, super_args] = ... + [required, varargin] = ... opencossan.common.utilities.parseRequiredNameValuePairs(... - ["model", "performancefunction"], varargin{:}); - % TODO: Why Input arguments???? - super_args = [super_args; {'input', required.model.Input, ... - 'evaluator', required.model.Evaluator}]; + "performancefunction", varargin{:}); + + [optionalArg, superArg] = opencossan.common.utilities.parseOptionalNameValuePairs(... + ["model", []], varargin{:}); + end + + obj@opencossan.common.Model(superArg{:}); - obj@opencossan.common.Model(super_args{:}); - - if nargin > 0 + obj.PerformanceFunctionVariable % Add PerformanceFunction to Evaluator if isempty(obj.Evaluator.Solver) obj.Evaluator = obj.Evaluator.add('Solver',required.performancefunction); @@ -57,7 +58,7 @@ obj.Evaluator = obj.Evaluator.add('Solver',required.performancefunction,'SolverName','N/A','Slots',Inf,... 'MaxCuncurrentJobs',Inf,'Hostname','localhost','Queue','','ParallelEnvironment',''); end - end + end function variable = get.PerformanceFunctionVariable(obj) diff --git a/+opencossan/+simulations/@SubsetInfinite/computeFailureProbability.m b/+opencossan/+simulations/@SubsetInfinite/computeFailureProbability.m index d7714fd0..639102e4 100644 --- a/+opencossan/+simulations/@SubsetInfinite/computeFailureProbability.m +++ b/+opencossan/+simulations/@SubsetInfinite/computeFailureProbability.m @@ -39,7 +39,7 @@ % along with openCOSSAN. If not, see . % ===================================================================== -import opencossan.common.inputs.random.RandomVariableSet +import opencossan.common.inputs.random.RandomVariableSet.* import opencossan.common.Samples import opencossan.simulations.SubsetOutput import opencossan.reliability.FailureProbability diff --git a/+opencossan/+workers/@Evaluator/Evaluator.m b/+opencossan/+workers/@Evaluator/Evaluator.m index 1047326d..a904652f 100644 --- a/+opencossan/+workers/@Evaluator/Evaluator.m +++ b/+opencossan/+workers/@Evaluator/Evaluator.m @@ -25,19 +25,19 @@ properties JobManager opencossan.highperformancecomputing.JobManagerInterface % Define JobManager to submit job to grid/cluster computer - Solver(1,:) %List of OpenCossan Workers opencossan.workers.Worker - SolverName(1,:) string % Names of the workers (optional) - Queues(1,:) string % Where to submit solvers - Hostnames(1,:) string % Names of hostnames where to evaluate workers - ParallelEnvironments(:,1) string % Name of the parallel environment of each solver - Slots(1,:) double {mustBePositive} % Number of slots used in each job - IsCompiled(1,:) logical % Number of slots used in each job + Solver(1,:) % List of OpenCossan Workers opencossan.workers.Worker + SolverName(1,:) string % Names of the workers (optional) + Queues(1,:) string % Where to submit workers (one per solver) + Hostnames(1,:) string % Names of hostnames where to evaluate workers + ParallelEnvironments(:,1) string % Name of the parallel environment of each solver + Slots(1,:) double {mustBePositive} % Number of slots used in each job + IsCompiled(1,:) logical % Number of slots used in each job MaxCuncurrentJobs(1,:) double {mustBePositive} = 1 % Number of concurrent execution of each solver - RemoteInjectExtract = false %TODO: make it true by default - VerticalSplit = false % if true split the analysis in vertical components (see wiki for more details) + RemoteInjectExtract = false %TODO: make it true by default + VerticalSplit = false % if true split the analysis in vertical components (see wiki for more details) MaxNumberofJobs double {mustBePositive} = 1 % max number of jobs submitted for each analysis - WrapperMatlabInputName(1,1) string % Name of the input Matlab file loaded by the job - WrapperMatlabOutputName(1,1) string % Name of the output Matlab file create by the job + WrapperMatlabInputName(1,1) string % Name of the input Matlab file loaded by the job + WrapperMatlabOutputName(1,1) string % Name of the output Matlab file create by the job end properties (Dependent=true) diff --git a/+opencossan/+workers/@Evaluator/add.m b/+opencossan/+workers/@Evaluator/add.m index 30c460f8..600ffd7e 100644 --- a/+opencossan/+workers/@Evaluator/add.m +++ b/+opencossan/+workers/@Evaluator/add.m @@ -32,7 +32,7 @@ OptionalsArguments={... "Queues","";... "Hostnames","";... - "ParallelEnvironments","";... + "ParallelEnvironments",'';... "Slots",[];... "MaxCuncurrentJobs",[];... "RemoteInjectExtract", false;... diff --git a/test/unit/+opencossan/+highperformancecomputing/JobManagerSlurmTest.m b/test/unit/+opencossan/+highperformancecomputing/JobManagerSlurmTest.m new file mode 100644 index 00000000..ad53bf27 --- /dev/null +++ b/test/unit/+opencossan/+highperformancecomputing/JobManagerSlurmTest.m @@ -0,0 +1,160 @@ +classdef JobManagerSlurmTest < matlab.unittest.TestCase + % JOBMANAGERSLURMTEST Unit tests for the class + % common.highperformancecompiting.JobMagagerSlurm + % + % See also: Job, JobManager + % + % @author Edoardo Patelli + % @date 23.02.2020 + + properties + Evaluator + end + methods (TestClassSetup) + + + function defineWorker(testCase) + MatWorker = opencossan.workers.MatlabWorker('Script',"Toutput.out2=Tinput.out1+5;", ... + 'Format',"structure", ... + 'OutputNames',{'out3'},... + 'InputNames',{'X1' 'X2' 'X4' 'out1'}); + + testCase.Evaluator = opencossan.workers.Evaluator('Solver',MatWorker,... + 'SolverName',{'Xmio'}); + end + + end + + methods (Test) + %% Constructor + function emptyConstructor(testCase) + Xjm = opencossan.highperformancecomputing.JobManagerSlurm; + testCase.assertClass(Xjm,'opencossan.workers.highperformancecomputing'); + end + + function constructorShouldSetDescription(testCase) + Xjm = opencossan.highperformancecomputing.JobManagerSlurm(... + 'description','TestJobManager','moduleList',["matlabR2019a" "secondModule"], ... + 'ClusterMatlabRoot','/Apps/Matlab/R2019b'); + testCase.assertEqual(Xe.Description,"Evaluator"); + end + + function constructorShouldSetMatlabWorker(testCase) + Xe = opencossan.workers.Evaluator('Solver',testCase.MatWorker,... + 'SolverName',{'Xmio'}); + testCase.assertEqual(Xe.Solver(1),testCase.MatWorker); + end + + function constructorShouldFail(testCase) + + Xe = opencossan.workers.Evaluator('Solver',[testCase.MatWorker testCase.MatWorker2],... + 'SolverName',["MatlabWorker" "ExtraName"]); + testCase.assertEqual(Xe.Solver(2),testCase.MatWorker2); + end + + + + function constructorShouldSetJobManagerInterface(testCase) + Xjmi = opencossan.highperformancecomputing.JobManagerInterface(); + Xe = opencossan.workers.Evaluator('Solver',testCase.MatWorker,'JobManager',Xjmi); + testCase.assertEqual(Xe.JobManager,Xjmi); + end + + function constructorShouldSetLremoteInjectExtract(testCase) + Xe = opencossan.workers.Evaluator('Solver',testCase.MatWorker,... + 'RemoteInjectExtract',true); + testCase.assertTrue(Xe.RemoteInjectExtract); + end + + function constructorShouldSetHostNames(testCase) + + Squeues = ["Queue1" "Queue2"]; + Shostnames = ["Host1","Host2"]; + Xe = opencossan.workers.Evaluator('Solver',[testCase.ConWorker testCase.MatWorker],... + 'Hostnames',Shostnames,... + 'Queues',Squeues); + testCase.assertEqual(Xe.Hostnames,Shostnames); + end + + function constructorShouldSetQueues(testCase) + Squeues = ["Queue1" "Queue2"]; + Xe = opencossan.workers.Evaluator('Solver',... + [testCase.ConWorker testCase.MatWorker],'Queues',Squeues); + testCase.assertEqual(Xe.Queues,Squeues); + end + + function constructorShouldSetVconcurrent(testCase) + Xe = opencossan.workers.Evaluator('Solver',[testCase.ConWorker testCase.MatWorker],... + 'MaxCuncurrentJobs',[Inf 4]); + testCase.assertEqual(Xe.MaxCuncurrentJobs,[Inf 4]); + end + + function constructorShouldSetMembers(testCase) + Xe = opencossan.workers.Evaluator('Solver',[testCase.ConWorker testCase.MatWorker]); + testCase.assertEqual(Xe.Solver,[testCase.ConWorker testCase.MatWorker]); + end + + function constructorShouldSetNames(testCase) + CSnames = ["Connector Name", "Mio name"]; + Xe = opencossan.workers.Evaluator('Solver',[testCase.ConWorker testCase.MatWorker],'SolverName',CSnames); + testCase.assertEqual(Xe.Solver,[testCase.ConWorker testCase.MatWorker]); + testCase.assertEqual(Xe.SolverName,CSnames); + end + + function constructorShouldSetSolutionSequence(testCase) + Xss = opencossan.workers.SolutionSequence(); + Xe = opencossan.workers.Evaluator('Solver',Xss); + testCase.assertEqual(Xe.Solver(1),Xss); + end + + function constructorShouldSetMetaModel(testCase) + Xrs = opencossan.metamodels.ResponseSurface(); + Xe = opencossan.workers.Evaluator('Solver',Xrs); + testCase.assertEqual(Xe.Solver(1),Xrs); + end + + function constructorShouldSetParallelEnvironment(testCase) + Xe = opencossan.workers.Evaluator('Solver',[testCase.ConWorker testCase.MatWorker],... + 'ParallelEnvironments',["parallel" "parallel"]); + testCase.assertEqual(Xe.ParallelEnvironments,["parallel";"parallel"]); + end + + function constructorShouldSetVSlots(testCase) + Xe = opencossan.workers.Evaluator('Solver',[testCase.ConWorker testCase.MatWorker],... + 'Slots',[1 1]); + testCase.assertEqual(Xe.Slots,[1 1]); + end + + function constructorTestInputNames(testCase) + + Xe = opencossan.workers.Evaluator('Solver',[testCase.MatWorker testCase.MatWorker2]); + + Cinput=[testCase.MatWorker.InputNames testCase.MatWorker2.InputNames]; + testCase.assertEqual(Xe.InputNames,Cinput); + end + + function constructorTestOutputNames(testCase) + + + Xe = opencossan.workers.Evaluator('Solver',[testCase.MatWorker testCase.MatWorker2]); + + Coutput=[testCase.MatWorker.OutputNames testCase.MatWorker2.OutputNames]; + testCase.assertEqual(Xe.OutputNames,Coutput); + end + + %% Deterministic Analysis + function deterministicAnalyis(testCase) + Xmio = opencossan.workers.MatlabWorker('Script',"Toutput.out1=Tinput.Xpar",... + 'OutputNames',{'out1'},'InputNames',{'Xpar'},'Format',"structure"); + Xpar = opencossan.common.inputs.Parameter('value',10.2); + Xinput = opencossan.common.inputs.Input('Parameter',Xpar); + Xe = opencossan.workers.Evaluator('Solver',Xmio,'SolverName',"mio Name"); + Xout=Xe.deterministicAnalysis(Xinput); + testCase.assertClass(Xout,'opencossan.common.outputs.SimulationData'); + testCase.assertEqual(Xout.getValues('Sname','out1'),10.2); + end + + end + +end + diff --git a/test/unit/+opencossan/+highperformancecomputing/JobTest.m b/test/unit/+opencossan/+highperformancecomputing/JobTest.m new file mode 100644 index 00000000..3e0fa0b5 --- /dev/null +++ b/test/unit/+opencossan/+highperformancecomputing/JobTest.m @@ -0,0 +1,61 @@ +classdef JobTest < matlab.unittest.TestCase + % JOB Unit tests for the class common.highperformancecompiting.Job + % See also: Job, JobManager + % + % @author Edoardo Patelli + % @date 23.02.2020 + + properties + JobManagerSlurm + end + + methods (TestClassSetup) + function defineWorker(testCase) + + stringSimulated='Simulating submission Job *Submitted batch job 2020.'; + + testCase.JobManagerSLurm = opencossan.highperformancecomputing.JobManagerSlurm(... + 'isRemoteCluster','false','AdditionalSubmitArgs',stringSimulated); + end + end + + methods (Test) + %% Constructor + function emptyConstructor(testCase) + obj = opencossan.highperformancecomputing.Job; + testCase.assertClass(obj,'opencossan.workers.highperformancecomputing.Job'); + end + + function constructorWithMandatoryArguments(testCase) + obj = opencossan.highperformancecomputing.Job(... + 'ID','2020','state','unknown'); + testCase.assertEqual(obj.ID,"2020"); + end + + function constructorWithOptionalArguments(testCase) + obj = opencossan.highperformancecomputing.Job(... + 'ID','2020','State','unknown','Sescription','TestJob',... + 'ScriptName','myScript','Dependences',[]); + testCase.assertEqual(obj.description,'TestJob'); + end + + function constructorShouldFail(testCase) + % Missing mandatory inputs + testCase.verifyError(@() opencossan.highperformancecomputing.Job(... + 'State','unknown','Sescription','TestJob'),... + 'OpenCossan:MissingRequiredInput') + end + + % TODO: Add additional tests + function testJobContructionFromJobManagerSlurm(testCase) + + obj=testCase.JobManagerSlurm.submitJob('Hostname','localhost',... + 'Queue','none'); + testCase.assertEqual(obj.ID,'2020'); + + end + + end + +end + diff --git a/test/unit/+opencossan/+workers/EvaluatorTest.m b/test/unit/+opencossan/+workers/EvaluatorTest.m index cfa49a95..1eaaf0ab 100644 --- a/test/unit/+opencossan/+workers/EvaluatorTest.m +++ b/test/unit/+opencossan/+workers/EvaluatorTest.m @@ -27,9 +27,7 @@ MatWorker2 ConWorker end - methods (TestClassSetup) - - + methods (TestClassSetup) function defineWorker(testCase) testCase.MatWorker = opencossan.workers.MatlabWorker( ... 'Description','covariance function', ... @@ -37,15 +35,14 @@ function defineWorker(testCase) 'InputNames',{'TestInput'},... % Define the inputs 'Script', "%Do Nothing",'OutputNames',{'TestOutput'}); % Define the outputs - testCase.MatWorker2 = opencossan.workers.MatlabWorker('Script',"Toutput.out2=Tinput.out1+5;", ... + testCase.MatWorker2 = opencossan.workers.MatlabWorker('Script',"Toutput.out2=Tinput.out1+5;", ... 'Format',"structure", ... 'OutputNames',{'out3'},... 'InputNames',{'X1' 'X2' 'X4' 'out1'}); testCase.ConWorker=opencossan.workers.Connector; end - - end + end methods (Test) %% Constructor @@ -73,8 +70,6 @@ function constructorShouldFail(testCase) testCase.assertEqual(Xe.Solver(2),testCase.MatWorker2); end - - function constructorShouldSetJobManagerInterface(testCase) Xjmi = opencossan.highperformancecomputing.JobManagerInterface(); Xe = opencossan.workers.Evaluator('Solver',testCase.MatWorker,'JobManager',Xjmi); @@ -94,7 +89,7 @@ function constructorShouldSetHostNames(testCase) Xe = opencossan.workers.Evaluator('Solver',[testCase.ConWorker testCase.MatWorker],... 'Hostnames',Shostnames,... 'Queues',Squeues); - testCase.assertEqual(Xe.Hostnames,Shostnames); + testCase.assertEqual(Xe.Hostnames,Shostnames); end function constructorShouldSetQueues(testCase) @@ -147,7 +142,7 @@ function constructorShouldSetVSlots(testCase) end function constructorTestInputNames(testCase) - + Xe = opencossan.workers.Evaluator('Solver',[testCase.MatWorker testCase.MatWorker2]); Cinput=[testCase.MatWorker.InputNames testCase.MatWorker2.InputNames]; @@ -155,10 +150,7 @@ function constructorTestInputNames(testCase) end function constructorTestOutputNames(testCase) - - - Xe = opencossan.workers.Evaluator('Solver',[testCase.MatWorker testCase.MatWorker2]); - + Xe = opencossan.workers.Evaluator('Solver',[testCase.MatWorker testCase.MatWorker2]); Coutput=[testCase.MatWorker.OutputNames testCase.MatWorker2.OutputNames]; testCase.assertEqual(Xe.OutputNames,Coutput); end From b797b003462e45cddf2aef8fc67772dede56c539 Mon Sep 17 00:00:00 2001 From: Etheban Date: Thu, 21 May 2020 18:23:57 +0100 Subject: [PATCH 7/9] Fix TutorialCantileverBeam --- .../+optimization/@Constraint/Constraint.m | 4 +- .../+optimization/@Constraint/evaluate.m | 2 +- .../@ObjectiveFunction/ObjectiveFunction.m | 4 +- .../@ObjectiveFunction/evaluate.m | 2 +- .../@ProbabilisticModel/ProbabilisticModel.m | 4 +- +opencossan/+workers/@Evaluator/Evaluator.m | 2 +- +opencossan/+workers/@Evaluator/add.m | 2 +- .../TutorialCantileverBeamMatlab.m | 39 ++++++++------ ...TutorialCantileverBeamMatlabOptimization.m | 54 +++++++------------ .../TutorialCantileverBeamMatlabRBO.m | 43 +++++++-------- ...lCantileverBeamMatlabReliabilityAnalysis.m | 27 ++++++---- 11 files changed, 95 insertions(+), 88 deletions(-) diff --git a/+opencossan/+optimization/@Constraint/Constraint.m b/+opencossan/+optimization/@Constraint/Constraint.m index a43e1b13..89ce8f5e 100644 --- a/+opencossan/+optimization/@Constraint/Constraint.m +++ b/+opencossan/+optimization/@Constraint/Constraint.m @@ -1,4 +1,4 @@ -classdef Constraint < opencossan.workers.Mio +classdef Constraint < opencossan.workers.MatlabWorker %CONSTRAINT The class Constraint defines the constraints for the % optimization problem. @@ -40,7 +40,7 @@ "inequality", {true}, varargin{:}); end - obj@opencossan.workers.Mio(super_args{:}); + obj@opencossan.workers.MatlabWorker(super_args{:}); if nargin > 0 obj.IsInequality = optional.inequality; diff --git a/+opencossan/+optimization/@Constraint/evaluate.m b/+opencossan/+optimization/@Constraint/evaluate.m index 0c383086..36555e33 100644 --- a/+opencossan/+optimization/@Constraint/evaluate.m +++ b/+opencossan/+optimization/@Constraint/evaluate.m @@ -61,7 +61,7 @@ % loop over all constraints for j = 1:numel(obj) - TableOutConstrains = evaluate@opencossan.workers.Mio(obj(j), ... + TableOutConstrains = evaluate@opencossan.workers.MatlabWorker(obj(j), ... output(:,obj(j).InputNames)); constraints(:,j) = TableOutConstrains.(obj(j).OutputNames{1}); diff --git a/+opencossan/+optimization/@ObjectiveFunction/ObjectiveFunction.m b/+opencossan/+optimization/@ObjectiveFunction/ObjectiveFunction.m index 52262d82..832a2e30 100644 --- a/+opencossan/+optimization/@ObjectiveFunction/ObjectiveFunction.m +++ b/+opencossan/+optimization/@ObjectiveFunction/ObjectiveFunction.m @@ -1,4 +1,4 @@ -classdef ObjectiveFunction < opencossan.workers.Mio +classdef ObjectiveFunction < opencossan.workers.MatlabWorker %OBJECTIVEFUNCTION Defines an objective function for use in %optimization problems. @@ -26,7 +26,7 @@ % % see also - obj@opencossan.workers.Mio(varargin{:}); + obj@opencossan.workers.MatlabWorker(varargin{:}); if nargin > 0 % The objective function must have a single output diff --git a/+opencossan/+optimization/@ObjectiveFunction/evaluate.m b/+opencossan/+optimization/@ObjectiveFunction/evaluate.m index df01577a..73e34a75 100644 --- a/+opencossan/+optimization/@ObjectiveFunction/evaluate.m +++ b/+opencossan/+optimization/@ObjectiveFunction/evaluate.m @@ -59,7 +59,7 @@ % loop over all objective functions for j = 1:length(obj) - XoutObjective = evaluate@opencossan.workers.Mio(obj(j), ... + XoutObjective = evaluate@opencossan.workers.MatlabWorker(obj(j), ... output(:,obj(j).InputNames)); objective(:,j) = XoutObjective.(obj(j).OutputNames{1}); diff --git a/+opencossan/+reliability/@ProbabilisticModel/ProbabilisticModel.m b/+opencossan/+reliability/@ProbabilisticModel/ProbabilisticModel.m index c36104ec..a022dcd5 100644 --- a/+opencossan/+reliability/@ProbabilisticModel/ProbabilisticModel.m +++ b/+opencossan/+reliability/@ProbabilisticModel/ProbabilisticModel.m @@ -41,9 +41,11 @@ [required, varargin] = ... opencossan.common.utilities.parseRequiredNameValuePairs(... "performancefunction", varargin{:}); + + OptionalsArguments={"Model", []}; [optionalArg, superArg] = opencossan.common.utilities.parseOptionalNameValuePairs(... - ["model", []], varargin{:}); + [OptionalsArguments{:,1}],{OptionalsArguments{:,2}}, varargin{:}); end diff --git a/+opencossan/+workers/@Evaluator/Evaluator.m b/+opencossan/+workers/@Evaluator/Evaluator.m index a904652f..23cf8140 100644 --- a/+opencossan/+workers/@Evaluator/Evaluator.m +++ b/+opencossan/+workers/@Evaluator/Evaluator.m @@ -29,7 +29,7 @@ SolverName(1,:) string % Names of the workers (optional) Queues(1,:) string % Where to submit workers (one per solver) Hostnames(1,:) string % Names of hostnames where to evaluate workers - ParallelEnvironments(:,1) string % Name of the parallel environment of each solver + ParallelEnvironments(1,:) string % Name of the parallel environment of each solver Slots(1,:) double {mustBePositive} % Number of slots used in each job IsCompiled(1,:) logical % Number of slots used in each job MaxCuncurrentJobs(1,:) double {mustBePositive} = 1 % Number of concurrent execution of each solver diff --git a/+opencossan/+workers/@Evaluator/add.m b/+opencossan/+workers/@Evaluator/add.m index 600ffd7e..30c460f8 100644 --- a/+opencossan/+workers/@Evaluator/add.m +++ b/+opencossan/+workers/@Evaluator/add.m @@ -32,7 +32,7 @@ OptionalsArguments={... "Queues","";... "Hostnames","";... - "ParallelEnvironments",'';... + "ParallelEnvironments","";... "Slots",[];... "MaxCuncurrentJobs",[];... "RemoteInjectExtract", false;... diff --git a/docs/tutorials/CantileverBeam/TutorialCantileverBeamMatlab.m b/docs/tutorials/CantileverBeam/TutorialCantileverBeamMatlab.m index 68085d6a..4adeea37 100644 --- a/docs/tutorials/CantileverBeam/TutorialCantileverBeamMatlab.m +++ b/docs/tutorials/CantileverBeam/TutorialCantileverBeamMatlab.m @@ -28,52 +28,61 @@ % Reset the random number generator in order to obtain always the same results. % DO NOT CHANGE THE VALUES OF THE SEED +%% Import packages +import opencossan.* +import opencossan.common.* +import opencossan.common.inputs.* +import opencossan.common.inputs.random.* +import opencossan.workers.* +import opencossan.simulations.* + + %% Define an Analysis % This allows to store the results of the simulation in a database (if % initialised) or/and to define the random number generator -opencossan.OpenCossan.setAnalysis('ProjectName', 'TutorialCantileverBeam', ... +OpenCossan.setAnalysis('ProjectName', 'TutorialCantileverBeam', ... 'AnalysisName', 'Tutorial', ... 'Seed', 213985) %% Preparation of the Input % Definition of the Parameters -L = opencossan.common.inputs.Parameter('value', 1.8, 'description', 'Beam Length'); -b = opencossan.common.inputs.Parameter('value', 0.12, 'description', 'Beam width'); -maxDisplacement = opencossan.common.inputs.Parameter('value', 0.010, 'description', 'Maximum allowed displacement'); +L = Parameter('value', 1.8, 'description', 'Beam Length'); +b = Parameter('value', 0.12, 'description', 'Beam width'); +maxDisplacement = Parameter('value', 0.010, 'description', 'Maximum allowed displacement'); % Definition of the Random Varibles -P = opencossan.common.inputs.random.LognormalRandomVariable.fromMeanAndStd('mean', 5000, 'std', 400, 'Description', 'Load'); -h = opencossan.common.inputs.random.NormalRandomVariable('mean', 0.24, 'std', 0.01, 'description', 'Beam Heigth'); -rho = opencossan.common.inputs.random.LognormalRandomVariable.fromMeanAndStd('mean', 600, 'std', 140, 'description', 'density'); -E = opencossan.common.inputs.random.LognormalRandomVariable.fromMeanAndStd('mean', 10e9, 'std', 1.6e9, 'description', 'Young''s modulus'); +P = LognormalRandomVariable.fromMeanAndStd('mean', 5000, 'std', 400, 'Description', 'Load'); +h = NormalRandomVariable('mean', 0.24, 'std', 0.01, 'description', 'Beam Heigth'); +rho = LognormalRandomVariable.fromMeanAndStd('mean', 600, 'std', 140, 'description', 'density'); +E = LognormalRandomVariable.fromMeanAndStd('mean', 10e9, 'std', 1.6e9, 'description', 'Young''s modulus'); % Definition of the Function -I = opencossan.common.inputs.Function('Description', 'Moment of Inertia', 'Expression', '<&b&>.*<&h&>.^3/12'); +I = Function('Description', 'Moment of Inertia', 'Expression', '<&b&>.*<&h&>.^3/12'); % Set of Random Variable Set Mcorrelation = eye(4); Mcorrelation(3, 4) = 0.8; % Add correlation between rho and E Mcorrelation(4, 3) = 0.8; -Xrvset = opencossan.common.inputs.random.RandomVariableSet('members', [P; h; rho; E], ... +Xrvset = RandomVariableSet('members', [P; h; rho; E], ... 'names', ["P", "h", "rho", "E"], 'Correlation', Mcorrelation); %% Prepare Input Object % The above prepared object can be added to an Input Object -Xinput = opencossan.common.inputs.Input('Members', {L b Xrvset I maxDisplacement}, ... +Xinput = Input('Members', {L b Xrvset I maxDisplacement}, ... 'MembersNames', {'L', 'b', 'Xrvset', 'I', 'maxDisplacement'}); % Show summary of the Input Object display(Xinput) %% Preparation of the Evaluator % Use of a matlab script to compute the Beam displacement currentFolder = fileparts(mfilename('fullpath')); -Xmio = opencossan.workers.MatlabWorker('FullFileName', fullfile(currentFolder, 'model', 'tipDisplacement.m'), ... +Xmio = MatlabWorker('FullFileName', fullfile(currentFolder, 'model', 'tipDisplacement.m'), ... 'InputNames', {'I', 'b', 'L', 'h', 'rho', 'P', 'E'}, ... 'OutputNames', {'w'}, 'Format', 'structure'); % Add the MIO object to an Evaluator object -Xevaluator = opencossan.workers.Evaluator('Solver', Xmio, 'SolverName',"Xmio"); +Xevaluator = Evaluator('Solver', Xmio, 'SolverName',"Xmio"); %% Preparation of the Physical Model % Define the Physical Model -XmodelBeamMatlab = opencossan.common.Model('Input', Xinput, 'Evaluator', Xevaluator); +XmodelBeamMatlab = Model('Input', Xinput, 'Evaluator', Xevaluator); % Perform deterministic analysis Xout = XmodelBeamMatlab.deterministicAnalysis; @@ -86,7 +95,7 @@ %% Uncertainty Quantification % Define simulation method -Xmc = opencossan.simulations.MonteCarlo('Nsamples', 1000); +Xmc = MonteCarlo('Nsamples', 1000); % preform Analysis XsimOutMC = Xmc.apply(XmodelBeamMatlab); diff --git a/docs/tutorials/CantileverBeam/TutorialCantileverBeamMatlabOptimization.m b/docs/tutorials/CantileverBeam/TutorialCantileverBeamMatlabOptimization.m index ef0bed0a..3837128b 100644 --- a/docs/tutorials/CantileverBeam/TutorialCantileverBeamMatlabOptimization.m +++ b/docs/tutorials/CantileverBeam/TutorialCantileverBeamMatlabOptimization.m @@ -3,27 +3,7 @@ % % % See Also http://cossan.co.uk/wiki/index.php/Cantilever_Beam -% -% -%

Copyright 2006-2020: COSSAN working group

-% Author: Edoardo-Patelli
-% Institute for Risk and Uncertainty, University of Liverpool, UK -%
COSSAN web site: http://www.cossan.co.uk -%

-% This file is part of openCOSSAN. The open source general purpose matlab toolbox -% for numerical analysis, risk and uncertainty quantification (http://www.cossan.co.uk). -%
-% openCOSSAN is free software: you can redistribute it and/or modify -% it under the terms of the GNU General Public License as published by -% the Free Software Foundation, either version 3 of the License. -% openCOSSAN is distributed in the hope that it will be useful, -% but WITHOUT ANY WARRANTY; without even the implied warranty of -% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -% GNU General Public License for more details. -% You should have received a copy of the GNU General Public License -% along with openCOSSAN. If not, see http://www.gnu.org/licenses/". -% + %{ This file is part of OpenCossan . @@ -43,38 +23,44 @@ with OpenCossan. If not, see . %} +%% Import packages +import opencossan.* +import opencossan.common.* +import opencossan.common.inputs.* +import opencossan.common.inputs.random.* +import opencossan.workers.* +import opencossan.optimization.* + %% Preparation of the Input % In this tutorial the random variable are replaced by two design variables % % The optimization analysis requires the definition of Design Variables % (i.e. the variables that define new configurations) -b = opencossan.optimization.ContinuousDesignVariable(... - 'value', 0.12, 'lowerBound', 0.01, 'upperBound', 0.50,... +b = ContinuousDesignVariable('value', 0.12, 'lowerBound', 0.01, 'upperBound', 0.50,... 'Description','Beam width'); -h = opencossan.optimization.ContinuousDesignVariable(... - 'value', 0.54, 'lowerBound', 0.02, 'upperBound', 1,... +h = ContinuousDesignVariable('value', 0.54, 'lowerBound', 0.02, 'upperBound', 1,... 'Description', 'Beam Heigth'); % In this example we do not use random variables and we only use Parameters -L = opencossan.common.inputs.Parameter('value', 1.8, 'Description', 'Beam Length'); -maxDisplacement = opencossan.common.inputs.Parameter('value', 0.001, 'Description', 'Maximum allowed displacement'); -P = opencossan.common.inputs.Parameter('value', 10000, 'Description', 'Load'); -rho = opencossan.common.inputs.Parameter('value', 600, 'Description', 'density'); -E = opencossan.common.inputs.Parameter('value', 10e9, 'Description','Young''s modulus'); +L = Parameter('value', 1.8, 'Description', 'Beam Length'); +maxDisplacement = Parameter('value', 0.001, 'Description', 'Maximum allowed displacement'); +P = Parameter('value', 10000, 'Description', 'Load'); +rho = Parameter('value', 600, 'Description', 'density'); +E = Parameter('value', 10e9, 'Description','Young''s modulus'); % Definition of the Function -I = opencossan.common.inputs.Function('Description','Moment of Inertia','Expression','<&b&>.*<&h&>.^3/12'); +I = Function('Description','Moment of Inertia','Expression','<&b&>.*<&h&>.^3/12'); %% Prepare Input Object % The above prepared objects can be added to an Input Object -XinputOptimization = opencossan.common.inputs.Input(... +XinputOptimization = Input(... 'Members', {L b P h rho E I maxDisplacement},... 'MembersNames', {'L' 'b' 'P' 'h' 'rho' 'E' 'I' 'MaxW'}); %% Preparation of the Evaluator % Use of a matlab script to compute the Beam displacement folder = fileparts(mfilename('fullpath'));% returns the current folder -Xmio = opencossan.workers.Mio(... +Xmio = MatlabWorker(... 'FunctionHandle', @tipDisplacement, ... 'IsFunction', true, ... 'Format', 'table', ... @@ -82,7 +68,7 @@ 'OutputNames',{'w'}); % Add the MIO object to an Evaluator object -Xevaluator=Evaluator('Solvers',Xmio,'SolversName',"Xmio"); +Xevaluator=Evaluator('Solver',Xmio,'SolverName',"Xmio"); %% Preparation of the Physical Model % Define the Physical Model diff --git a/docs/tutorials/CantileverBeam/TutorialCantileverBeamMatlabRBO.m b/docs/tutorials/CantileverBeam/TutorialCantileverBeamMatlabRBO.m index 2aaaf634..4c263241 100644 --- a/docs/tutorials/CantileverBeam/TutorialCantileverBeamMatlabRBO.m +++ b/docs/tutorials/CantileverBeam/TutorialCantileverBeamMatlabRBO.m @@ -8,31 +8,32 @@ % The performance function is defined by the maximum allowable % stress level minus the actual stress in a clamped beam. % -% | +% | % //| v % //|--------------------------------------- % //| % -% -%

Copyright 2006-2014: COSSAN working group

-% Author: Edoardo-Patelli
-% Institute for Risk and Uncertainty, University of Liverpool, UK -%
COSSAN web site: http://www.cossan.co.uk -%

-% This file is part of openCOSSAN. The open source general purpose matlab toolbox -% for numerical analysis, risk and uncertainty quantification (http://www.cossan.co.uk). -%
-% openCOSSAN is free software: you can redistribute it and/or modify -% it under the terms of the GNU General Public License as published by -% the Free Software Foundation, either version 3 of the License. -% openCOSSAN is distributed in the hope that it will be useful, -% but WITHOUT ANY WARRANTY; without even the implied warranty of -% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -% GNU General Public License for more details. -% You should have received a copy of the GNU General Public License -% along with openCOSSAN. If not, see http://www.gnu.org/licenses/". -% +% +% See Also http://cossan.co.uk/wiki/index.php/Cantilever_Beam + + +%{ +This file is part of OpenCossan . +Copyright (C) 2006-2020 COSSAN WORKING GROUP + +OpenCossan is free software: you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free +Software Foundation, either version 3 of the License or, (at your option) +any later version. + +OpenCossan is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +more details. + +You should have received a copy of the GNU General Public License along +with OpenCossan. If not, see . +%} %% Check avilability of the probabilistic model assert(logical(exist('XprobModelBeamMatlab','var')),'openCOSSAN:Tutorial', ... diff --git a/docs/tutorials/CantileverBeam/TutorialCantileverBeamMatlabReliabilityAnalysis.m b/docs/tutorials/CantileverBeam/TutorialCantileverBeamMatlabReliabilityAnalysis.m index ff9d7ce4..18e0df18 100644 --- a/docs/tutorials/CantileverBeam/TutorialCantileverBeamMatlabReliabilityAnalysis.m +++ b/docs/tutorials/CantileverBeam/TutorialCantileverBeamMatlabReliabilityAnalysis.m @@ -20,6 +20,15 @@ %} +%% Import packages +import opencossan.* +import opencossan.reliability.* +import opencossan.common.inputs.* +import opencossan.common.inputs.random.* +import opencossan.sensitivity.* +import opencossan.optimization.* +import opencossan.simulations.* + %% Create model if necessary % This tutorial requires the model created by the tutorial % TutorialCantileverBeamMatlab @@ -30,18 +39,18 @@ %% Define a Probabilistic Model % Performance Function -Xperfun = opencossan.reliability.PerformanceFunction('OutputName', 'Vg',... +Xperfun = PerformanceFunction('OutputName', 'Vg',... 'Demand', 'w', 'Capacity', 'maxDisplacement'); % Define a Probabilistic Model -XprobModelBeamMatlab = opencossan.reliability.ProbabilisticModel(... +XprobModelBeamMatlab = ProbabilisticModel(... 'Model', XmodelBeamMatlab, 'PerformanceFunction', Xperfun); %% Reliability Analysis via Monte Carlo Sampling % Reset the random number generator to always produce the same results -opencossan.OpenCossan.resetRandomNumberGenerator(51125); +OpenCossan.resetRandomNumberGenerator(51125); % Create MonteCarlo simulation object to run 1e5 samples in 1 batch -Xmc = opencossan.simulations.MonteCarlo('Nsamples',1e5,'Nbatches',1); +Xmc = MonteCarlo('Nsamples',1e5,'Nbatches',1); % Run reliability analysis XfailureProbMC = Xmc.computeFailureProbability(XprobModelBeamMatlab); @@ -54,11 +63,11 @@ %% Reliability Analysis via Latin Hypercube Sampling % Reset the random number generator to always produce the same results -opencossan.OpenCossan.resetRandomNumberGenerator(49564); +OpenCossan.resetRandomNumberGenerator(49564); % Create LatinHypercubeSampling simulation object to run 1e5 samples in 1 % batch -Xlhs=opencossan.simulations.LatinHypercubeSampling('Nsamples',1e4); +Xlhs=LatinHypercubeSampling('Nsamples',1e4); % Run reliability analysis XfailureProbLHS = Xlhs.computeFailureProbability(XprobModelBeamMatlab); @@ -71,18 +80,18 @@ %% Reliability Analysis via LineSampling % Reset the random number generator to always produce the same results -opencossan.OpenCossan.resetRandomNumberGenerator(49564); +OpenCossan.resetRandomNumberGenerator(49564); % Line Sampling requires the definition of the so-called important % direction. It can be compute using sensitivity methods. For instance, % here the gradient in standard normal space is computed. -XlsFD = opencossan.sensitivity.LocalSensitivityFiniteDifference(... +XlsFD = LocalSensitivityFiniteDifference(... 'Xmodel', XprobModelBeamMatlab, 'Coutputname', {'Vg'}); XlocalSensitivity = XlsFD.computeGradientStandardNormalSpace(); % Use sensitivity information to define the important direction for LineSampling -XLS=opencossan.simulations.LineSampling(... +XLS=LineSampling(... 'XlocalSensitivityMeasures', XlocalSensitivity,'Nlines', 50,... 'Vset', 0.5:0.5:3.5); From 4b34ceb4e2172e557f77ae4e8e2af95d25149cdd Mon Sep 17 00:00:00 2001 From: Edoardo Patelli Date: Thu, 21 May 2020 23:56:13 +0100 Subject: [PATCH 8/9] Fixing Evaluator --- +opencossan/+common/@Model/apply.m | 2 +- .../@ProbabilisticModel/ProbabilisticModel.m | 30 ++++---- +opencossan/+workers/@Evaluator/Evaluator.m | 40 ++-------- +opencossan/+workers/@Evaluator/add.m | 12 +-- .../@ExecutionStrategy/ExecutionStrategy.m | 75 +++++++++++++++++++ +opencossan/+workers/@Worker/Worker.m | 9 ++- 6 files changed, 109 insertions(+), 59 deletions(-) create mode 100644 +opencossan/+workers/@ExecutionStrategy/ExecutionStrategy.m diff --git a/+opencossan/+common/@Model/apply.m b/+opencossan/+common/@Model/apply.m index da061d1a..c3c82a57 100644 --- a/+opencossan/+common/@Model/apply.m +++ b/+opencossan/+common/@Model/apply.m @@ -3,7 +3,7 @@ %{ This file is part of OpenCossan . -Copyright (C) 2006-2018 COSSAN WORKING GROUP +Copyright (C) 2006-2020 COSSAN WORKING GROUP OpenCossan is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/+opencossan/+reliability/@ProbabilisticModel/ProbabilisticModel.m b/+opencossan/+reliability/@ProbabilisticModel/ProbabilisticModel.m index a022dcd5..663ebd31 100644 --- a/+opencossan/+reliability/@ProbabilisticModel/ProbabilisticModel.m +++ b/+opencossan/+reliability/@ProbabilisticModel/ProbabilisticModel.m @@ -25,7 +25,7 @@ properties (Dependent=true) PerformanceFunctionVariable % Name of the output of the performance function - StdDeviationIndicatorFunction % StdDeviationIndicatorFunction of the performance Function + StdDeviationIndicatorFunction end methods @@ -36,31 +36,29 @@ % This class allows to perform reliability analysis and uncertainty quantification if nargin == 0 - super_args = {}; + superArg = {}; else [required, varargin] = ... opencossan.common.utilities.parseRequiredNameValuePairs(... "performancefunction", varargin{:}); - OptionalsArguments={"Model", []}; - - [optionalArg, superArg] = opencossan.common.utilities.parseOptionalNameValuePairs(... + OptionalsArguments={"Model", []}; + + [modelArg, superArg] = opencossan.common.utilities.parseOptionalNameValuePairs(... [OptionalsArguments{:,1}],{OptionalsArguments{:,2}}, varargin{:}); + end - + obj@opencossan.common.Model(superArg{:}); - obj.PerformanceFunctionVariable - % Add PerformanceFunction to Evaluator - if isempty(obj.Evaluator.Solver) - obj.Evaluator = obj.Evaluator.add('Solver',required.performancefunction); - else - % TODO: What does this do? - obj.Evaluator = obj.Evaluator.add('Solver',required.performancefunction,'SolverName','N/A','Slots',Inf,... - 'MaxCuncurrentJobs',Inf,'Hostname','localhost','Queue','','ParallelEnvironment',''); - end - + if ~isempty(modelArg.model) + obj=modelArg.model; + end + + % Add PerformanceFunction to Evaluator + obj.Evaluator = obj.Evaluator.add('Solver',required.performancefunction); + end function variable = get.PerformanceFunctionVariable(obj) diff --git a/+opencossan/+workers/@Evaluator/Evaluator.m b/+opencossan/+workers/@Evaluator/Evaluator.m index 23cf8140..87d8ca78 100644 --- a/+opencossan/+workers/@Evaluator/Evaluator.m +++ b/+opencossan/+workers/@Evaluator/Evaluator.m @@ -27,17 +27,7 @@ JobManager opencossan.highperformancecomputing.JobManagerInterface % Define JobManager to submit job to grid/cluster computer Solver(1,:) % List of OpenCossan Workers opencossan.workers.Worker SolverName(1,:) string % Names of the workers (optional) - Queues(1,:) string % Where to submit workers (one per solver) - Hostnames(1,:) string % Names of hostnames where to evaluate workers - ParallelEnvironments(1,:) string % Name of the parallel environment of each solver - Slots(1,:) double {mustBePositive} % Number of slots used in each job - IsCompiled(1,:) logical % Number of slots used in each job - MaxCuncurrentJobs(1,:) double {mustBePositive} = 1 % Number of concurrent execution of each solver - RemoteInjectExtract = false %TODO: make it true by default - VerticalSplit = false % if true split the analysis in vertical components (see wiki for more details) - MaxNumberofJobs double {mustBePositive} = 1 % max number of jobs submitted for each analysis - WrapperMatlabInputName(1,1) string % Name of the input Matlab file loaded by the job - WrapperMatlabOutputName(1,1) string % Name of the output Matlab file create by the job + VerticalSplit = false % if true split the analysis in vertical components end properties (Dependent=true) @@ -52,7 +42,7 @@ % % Please see the reference manual for the complete documentation % - % Copyright 1983-2015 COSSAN Working Group + % Copyright 1983-2020 COSSAN Working Group % Author: Edoardo Patelli @@ -67,18 +57,9 @@ OptionalsArguments={... "JobManager", opencossan.highperformancecomputing.JobManagerInterface.empty(1,0);... - "Queues",[];... - "Hostnames",[];... - "ParallelEnvironments",[];... - "Slots",[];... - "IsCompiled",false;... - "MaxCuncurrentJobs",[];... - "RemoteInjectExtract", false;... - "VerticalSplit",[];... - "MaxNumberofJobs",[];... - "WrapperMatlabInputName","";... - "WrapperMatlabOutputName","";... - "SolverName",[]}; + "Queues","";... + "SolverName",[];... + "VerticalSplit",true}; [optionalArg, superArg] = opencossan.common.utilities.parseOptionalNameValuePairs(... [OptionalsArguments{:,1}],{OptionalsArguments{:,2}}, varargin{:}); @@ -101,18 +82,7 @@ numel(obj.Solver),numel(obj.SolverName)); end - obj.JobManager = optionalArg.jobmanager; - obj.Queues = optionalArg.queues; - obj.Hostnames = optionalArg.hostnames; - obj.ParallelEnvironments = optionalArg.parallelenvironments; - obj.Slots = optionalArg.slots; - obj.IsCompiled = optionalArg.iscompiled; - obj.MaxCuncurrentJobs = optionalArg.maxcuncurrentjobs; - obj.RemoteInjectExtract=optionalArg.remoteinjectextract; obj.VerticalSplit = optionalArg.verticalsplit; - obj.MaxNumberofJobs = optionalArg.maxnumberofjobs; - obj.WrapperMatlabInputName = optionalArg.wrappermatlabinputname; - obj.WrapperMatlabOutputName = optionalArg.wrappermatlaboutputname; obj=validateObject(obj); end diff --git a/+opencossan/+workers/@Evaluator/add.m b/+opencossan/+workers/@Evaluator/add.m index 30c460f8..ab0b1b7d 100644 --- a/+opencossan/+workers/@Evaluator/add.m +++ b/+opencossan/+workers/@Evaluator/add.m @@ -34,7 +34,7 @@ "Hostnames","";... "ParallelEnvironments","";... "Slots",[];... - "MaxCuncurrentJobs",[];... + "MaxCuncurrentJobs",inf;... "RemoteInjectExtract", false;... "MaxNumberofJobs",[];... "SolverName",""}; @@ -46,11 +46,11 @@ obj.SolverName = [obj.SolverName optionalArg.solvername]; -obj.Queues = [obj.Queues optionalArg.queues]; -obj.Hostnames = [obj.Hostnames optionalArg.hostnames]; -obj.ParallelEnvironments = [obj.ParallelEnvironments optionalArg.parallelenvironments]; -obj.Slots = [obj.Slots optionalArg.slots]; -obj.MaxCuncurrentJobs = [obj.MaxCuncurrentJobs optionalArg.maxcuncurrentjobs]; +obj.Queues = [obj.Queues, optionalArg.queues]; +obj.Hostnames = [obj.Hostnames, optionalArg.hostnames]; +obj.ParallelEnvironments = [obj.ParallelEnvironments, optionalArg.parallelenvironments]; +obj.Slots = [obj.Slots, optionalArg.slots]; +obj.MaxCuncurrentJobs = [obj.MaxCuncurrentJobs, optionalArg.maxcuncurrentjobs]; obj.MaxNumberofJobs = optionalArg.maxnumberofjobs; % Validate object diff --git a/+opencossan/+workers/@ExecutionStrategy/ExecutionStrategy.m b/+opencossan/+workers/@ExecutionStrategy/ExecutionStrategy.m new file mode 100644 index 00000000..ea1cc3dc --- /dev/null +++ b/+opencossan/+workers/@ExecutionStrategy/ExecutionStrategy.m @@ -0,0 +1,75 @@ +classdef ExecutionStrategy < opencossan.common.CossanObject + %EXECUTIONSTRATEGY Summary of this class goes here + % Detailed explanation goes here + + properties + Queues(1,1) string % Where to submit workers (one per solver) + Hostnames(1,1) string % Names of hostnames where to evaluate workers + ParallelEnvironments(1,1) string % Name of the parallel environment of each solver + Slots(1,1) double {mustBePositive} % Number of slots used in each job + IsCompiled(1,1) logical % Compiled MatlabWorker + MaxCuncurrentJobs(1,1) double {mustBePositive} = Inf % Number of concurrent execution of each solver + RemoteInjectExtract = false %TODO: make it true by default + MaxNumberofJobs double {mustBePositive} = 1 % max number of jobs submitted for each analysis + WrapperMatlabInputName(1,1) string % Name of the input Matlab file loaded by the job + WrapperMatlabOutputName(1,1) string % Name of the output Matlab file create by the job + end + + methods + function obj = ExecutionStrategy(varargin) + %EXECUTIONSTRATEGY Construct an instance of this class + % Detailed explanation goes here + if nargin==0 + % Crate empty object + superArg={}; + else + [requiredArgs, varargin] = opencossan.common.utilities.parseRequiredNameValuePairs(... + "Solver", varargin{:}); + + % Define optional arguments and default values + + OptionalsArguments={... + "JobManager", opencossan.highperformancecomputing.JobManagerInterface.empty(1,0);... + "Queues","";... + "Hostnames","";... + "ParallelEnvironments","";... + "Slots",[];... + "IsCompiled",false;... + "MaxCuncurrentJobs",[];... + "RemoteInjectExtract", false;... + "MaxNumberofJobs",[];... + "WrapperMatlabInputName","";... + "WrapperMatlabOutputName","";... + "SolverName",[]}; + + [optionalArg, superArg] = opencossan.common.utilities.parseOptionalNameValuePairs(... + [OptionalsArguments{:,1}],{OptionalsArguments{:,2}}, varargin{:}); + + end + + % Now we define all the inputs not filtered out by the parsers + obj@opencossan.common.CossanObject(superArg{:}); + + obj.JobManager = optionalArg.jobmanager; + obj.Queues = optionalArg.queues; + obj.Hostnames = optionalArg.hostnames; + obj.ParallelEnvironments = optionalArg.parallelenvironments; + obj.Slots = optionalArg.slots; + obj.IsCompiled = optionalArg.iscompiled; + obj.MaxCuncurrentJobs = optionalArg.maxcuncurrentjobs; + obj.RemoteInjectExtract=optionalArg.remoteinjectextract; + obj.MaxNumberofJobs = optionalArg.maxnumberofjobs; + obj.WrapperMatlabInputName = optionalArg.wrappermatlabinputname; + obj.WrapperMatlabOutputName = optionalArg.wrappermatlaboutputname; + + + end + + function outputArg = method1(obj,inputArg) + %METHOD1 Summary of this method goes here + % Detailed explanation goes here + outputArg = obj.Property1 + inputArg; + end + end +end + diff --git a/+opencossan/+workers/@Worker/Worker.m b/+opencossan/+workers/@Worker/Worker.m index 2f93feec..93f8cd49 100644 --- a/+opencossan/+workers/@Worker/Worker.m +++ b/+opencossan/+workers/@Worker/Worker.m @@ -32,6 +32,7 @@ OutputNames cell {opencossan.common.utilities.isunique(OutputNames)} InputNames cell {opencossan.common.utilities.isunique(InputNames)} KeepSimulationFiles(1,1) logical = false % Keep simulation files + ExecutionStrategy(1,1) opencossan.common.worker.ExecutionStrategy end methods (Abstract) @@ -50,8 +51,13 @@ [requiredArgs, varargin] = opencossan.common.utilities.parseRequiredNameValuePairs(... ["InputNames","OutputNames"], varargin{:}); + OptionalsArguments={... + "KeepSimulationFiles", false;... + "ExecutionStrategy",[]}; + [optionalArgs, superArg] = opencossan.common.utilities.parseOptionalNameValuePairs(... - "KeepSimulationFiles",{false}, varargin{:}); + [OptionalsArguments{:,1}],{OptionalsArguments{:,2}}, varargin{:}); + end obj@opencossan.common.CossanObject(superArg{:}); @@ -59,6 +65,7 @@ obj.InputNames=requiredArgs.inputnames; obj.OutputNames=requiredArgs.outputnames; obj.KeepSimulationFiles=optionalArgs.keepsimulationfiles; + obj.ExecutionStrategy=optionalArgs.executionstrategy; end end end From 14d2ee9320ff080c540e25e2a62230ae73bbd04d Mon Sep 17 00:00:00 2001 From: Etheban Date: Sat, 23 May 2020 11:08:21 +0100 Subject: [PATCH 9/9] Reworking of Workers and subclasses Reworking compled, TutorialCossanObject fixed. Known issues: * Optimization does not comply with Workers * Some tutorials do not work due to problem with Input refactoring --- +opencossan/+common/@Timer/Timer.m | 9 +- .../@ExecutionStrategy/ExecutionStrategy.m | 4 +- .../+optimization/@Constraint/Constraint.m | 4 +- .../@ObjectiveFunction/ObjectiveFunction.m | 6 +- .../@ObjectiveFunction/evaluate.m | 2 +- .../@ProbabilisticModel/ProbabilisticModel.m | 19 ++- .../@LatinHypercubeSampling/apply.m | 10 +- +opencossan/+workers/@Evaluator/Evaluator.m | 23 ++-- +opencossan/+workers/@Evaluator/add.m | 35 ++--- .../@Evaluator/executeWorkersHorizontal.m | 39 +++--- .../@Evaluator/executeWorkersVertical.m | 61 ++++----- .../+workers/@Evaluator/validateObject.m | 59 +-------- .../+workers/@MatlabWorker/MatlabWorker.m | 29 ++-- .../@MatlabWorker/createSimulationData.m | 16 +-- +opencossan/+workers/@MatlabWorker/evaluate.m | 11 +- .../@MatlabWorker/validateConstructor.m | 125 +++++++++--------- +opencossan/+workers/@Worker/Worker.m | 4 +- .../@OpenCossan/validateCossanInputs.m | 2 +- .../TutorialCantileverBeamMatlab.m | 11 +- .../TutorialCantileverBeamMatlabRBO.m | 3 + .../tutorials/CantileverBeam/importpackages.m | 8 ++ .../ExampleMatlabWorkerFunction.m | 2 +- .../ExampleMatlabWorkerMatrix.m | 2 +- .../ExampleMatlabWorkerStructure.m | 2 +- .../ExampleMatlabWorkerTable.m | 2 +- .../TutorialAdaptiveLineSampling.m | 10 +- .../TutorialAdvancedLineSampling.m | 20 +-- docs/tutorials/CossanObjects/TutorialBobyqa.m | 6 +- docs/tutorials/CossanObjects/TutorialCobyla.m | 18 ++- .../CossanObjects/TutorialConstraint.m | 50 +++++-- .../TutorialCovarianceFunction.m | 22 +-- .../TutorialDesignOfExperiment.m | 6 +- .../CossanObjects/TutorialEvaluator.m | 58 ++++---- .../CossanObjects/TutorialExtractor.m | 22 ++- .../TutorialFailureProbability.m | 4 +- .../CossanObjects/TutorialFunction.m | 6 +- ...TutorialGaussianMixtureRandomVariableSet.m | 4 +- .../CossanObjects/TutorialGeneticAlgorithms.m | 4 +- ...rialGlobalSensitivityRandomBalanceDesign.m | 13 +- .../TutorialGlobalSensitivitySobol.m | 14 +- .../TutorialGlobalSensitivityUpperBound.m | 9 +- .../CossanObjects/TutorialGradient.m | 11 +- .../CossanObjects/TutorialHaltonSampling.m | 11 +- .../TutorialImportanceSampling.m | 15 +-- .../CossanObjects/TutorialJobManager.m | 53 -------- .../TutorialJobManagerInterface.m | 18 --- .../CossanObjects/TutorialJobManagerSlurm.m | 2 +- .../CossanObjects/TutorialKarhunenLoeve.m | 10 +- .../tutorials/CossanObjects/TutorialKriging.m | 5 +- .../TutorialLatinHypercubeSampling.m | 15 +-- .../CossanObjects/TutorialLineSampling.m | 16 +-- ...TutorialLocalSensitivityFiniteDifference.m | 38 +++--- .../TutorialLocalSensitivityMonteCarlo.m | 11 +- .../CossanObjects/TutorialMarkovChain.m | 4 +- .../CossanObjects/TutorialMatlabWorker.m | 12 +- .../CossanObjects/TutorialMetaModel.m | 3 +- .../tutorials/CossanObjects/TutorialMiniMax.m | 2 +- docs/tutorials/CossanObjects/TutorialModel.m | 16 +-- docs/tutorials/CossanObjects/TutorialModes.m | 4 +- .../CossanObjects/TutorialMonteCarlo.m | 10 +- .../CossanObjects/TutorialObjectiveFunction.m | 54 +++++--- docs/tutorials/CossanObjects/TutorialTimer.m | 49 ++++--- .../unit/+opencossan/+workers/EvaluatorTest.m | 77 ++--------- 63 files changed, 529 insertions(+), 661 deletions(-) rename +opencossan/{+workers => +highperformancecomputing}/@ExecutionStrategy/ExecutionStrategy.m (97%) create mode 100644 docs/tutorials/CantileverBeam/importpackages.m delete mode 100644 docs/tutorials/CossanObjects/TutorialJobManager.m delete mode 100644 docs/tutorials/CossanObjects/TutorialJobManagerInterface.m diff --git a/+opencossan/+common/@Timer/Timer.m b/+opencossan/+common/@Timer/Timer.m index a6109889..eae51c25 100644 --- a/+opencossan/+common/@Timer/Timer.m +++ b/+opencossan/+common/@Timer/Timer.m @@ -140,10 +140,11 @@ function reset(obj) % f = plot(obj,varargin) % % PLOT supports the following name-value parameters: - % 'FigureName' - Export the figure as FigureName.pdf - % (char) - % 'ExportFormat' - Specifiy the format to export as - % (char) + % Title: Title of the figure (char) + % FigureName: Export the figure as FigureName.pdf (char) + % ExportFormat: Specifiy the format to export as (char) + % + % See also: Timer %% Parse Inputs p = inputParser; diff --git a/+opencossan/+workers/@ExecutionStrategy/ExecutionStrategy.m b/+opencossan/+highperformancecomputing/@ExecutionStrategy/ExecutionStrategy.m similarity index 97% rename from +opencossan/+workers/@ExecutionStrategy/ExecutionStrategy.m rename to +opencossan/+highperformancecomputing/@ExecutionStrategy/ExecutionStrategy.m index ea1cc3dc..a53c4df7 100644 --- a/+opencossan/+workers/@ExecutionStrategy/ExecutionStrategy.m +++ b/+opencossan/+highperformancecomputing/@ExecutionStrategy/ExecutionStrategy.m @@ -33,7 +33,7 @@ "Queues","";... "Hostnames","";... "ParallelEnvironments","";... - "Slots",[];... + "Slots",Inf;... "IsCompiled",false;... "MaxCuncurrentJobs",[];... "RemoteInjectExtract", false;... @@ -50,7 +50,7 @@ % Now we define all the inputs not filtered out by the parsers obj@opencossan.common.CossanObject(superArg{:}); - obj.JobManager = optionalArg.jobmanager; + obj.JobManager = optionalArg.jobmanager; obj.Queues = optionalArg.queues; obj.Hostnames = optionalArg.hostnames; obj.ParallelEnvironments = optionalArg.parallelenvironments; diff --git a/+opencossan/+optimization/@Constraint/Constraint.m b/+opencossan/+optimization/@Constraint/Constraint.m index 89ce8f5e..5bb24833 100644 --- a/+opencossan/+optimization/@Constraint/Constraint.m +++ b/+opencossan/+optimization/@Constraint/Constraint.m @@ -37,13 +37,13 @@ super_args = {}; else [optional, super_args] = opencossan.common.utilities.parseOptionalNameValuePairs(... - "inequality", {true}, varargin{:}); + "IsInequality", {true}, varargin{:}); end obj@opencossan.workers.MatlabWorker(super_args{:}); if nargin > 0 - obj.IsInequality = optional.inequality; + obj.IsInequality = optional.isinequality; end % Assert that the constraint has a single output diff --git a/+opencossan/+optimization/@ObjectiveFunction/ObjectiveFunction.m b/+opencossan/+optimization/@ObjectiveFunction/ObjectiveFunction.m index 832a2e30..8b89e4ce 100644 --- a/+opencossan/+optimization/@ObjectiveFunction/ObjectiveFunction.m +++ b/+opencossan/+optimization/@ObjectiveFunction/ObjectiveFunction.m @@ -1,10 +1,12 @@ classdef ObjectiveFunction < opencossan.workers.MatlabWorker %OBJECTIVEFUNCTION Defines an objective function for use in %optimization problems. + % + % See also: Worker Optimization %{ This file is part of OpenCossan . - Copyright (C) 2006-2018 COSSAN WORKING GROUP + Copyright (C) 2006-2030 COSSAN WORKING GROUP OpenCossan is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -31,7 +33,7 @@ if nargin > 0 % The objective function must have a single output assert(length(obj.OutputNames) == 1,... - 'openCOSSAN:optimization:ObjectiveFunction',... + 'OpenCossan:optimization:ObjectiveFunction',... 'A single output (OutputNames) must be defined'); end end diff --git a/+opencossan/+optimization/@ObjectiveFunction/evaluate.m b/+opencossan/+optimization/@ObjectiveFunction/evaluate.m index 73e34a75..31c6780a 100644 --- a/+opencossan/+optimization/@ObjectiveFunction/evaluate.m +++ b/+opencossan/+optimization/@ObjectiveFunction/evaluate.m @@ -2,7 +2,7 @@ % EVALUATE Evaluate the objective function %{ - This file is part of OpenCossan . Copyright (C) 2006-2018 COSSAN WORKING + This file is part of OpenCossan . Copyright (C) 2006-2020 COSSAN WORKING GROUP OpenCossan is free software: you can redistribute it and/or modify it under the terms of the GNU diff --git a/+opencossan/+reliability/@ProbabilisticModel/ProbabilisticModel.m b/+opencossan/+reliability/@ProbabilisticModel/ProbabilisticModel.m index 663ebd31..cfeb7ca8 100644 --- a/+opencossan/+reliability/@ProbabilisticModel/ProbabilisticModel.m +++ b/+opencossan/+reliability/@ProbabilisticModel/ProbabilisticModel.m @@ -35,9 +35,12 @@ % physical model (Model) and a performance function (PerformanceFunction). % This class allows to perform reliability analysis and uncertainty quantification + if nargin == 0 superArg = {}; else + idxPerFun=find(strcmpi(varargin,'performancefunction')); + [required, varargin] = ... opencossan.common.utilities.parseRequiredNameValuePairs(... "performancefunction", varargin{:}); @@ -47,17 +50,23 @@ [modelArg, superArg] = opencossan.common.utilities.parseOptionalNameValuePairs(... [OptionalsArguments{:,1}],{OptionalsArguments{:,2}}, varargin{:}); + if ~isempty(modelArg.model) + superArg{end+1}="Input"; + superArg{end+1}=modelArg.model.Input; + superArg{end+1}="Evaluator"; + superArg{end+1}=modelArg.model.Evaluator; + end end obj@opencossan.common.Model(superArg{:}); + - if ~isempty(modelArg.model) - obj=modelArg.model; - end - + if nargin>0 % Add PerformanceFunction to Evaluator - obj.Evaluator = obj.Evaluator.add('Solver',required.performancefunction); + obj.Evaluator = obj.Evaluator.add("Solver",required.performancefunction,... + "SolverName",inputname(idxPerFun+1)); + end end diff --git a/+opencossan/+simulations/@LatinHypercubeSampling/apply.m b/+opencossan/+simulations/@LatinHypercubeSampling/apply.m index 1abc75db..93945df4 100644 --- a/+opencossan/+simulations/@LatinHypercubeSampling/apply.m +++ b/+opencossan/+simulations/@LatinHypercubeSampling/apply.m @@ -46,7 +46,7 @@ Xobj.ibatch = Xobj.ibatch + 1; % Lap time for each batch - opencossan.OpenCossan.setLaptime('description',[' Batch #' num2str(Xobj.ibatch)]); + opencossan.OpenCossan.getTimer().lap('description',[' Batch #' num2str(Xobj.ibatch)]); % Number of samples current batch if Xobj.ibatch==Xobj.Nbatches || Xobj.Nsimxbatch==0 @@ -63,7 +63,7 @@ Xinput=set(Xinput,'Xsamples',Xs); - OpenCossan.cossanDisp(['Latin Hypercube Sampling simulation Batch ' num2str(Xobj.ibatch) ... + opencossan.OpenCossan.cossanDisp(['Latin Hypercube Sampling simulation Batch ' num2str(Xobj.ibatch) ... ' ( ' num2str(Ns) ' samples)' ],4) %% evaluate performance function @@ -84,13 +84,13 @@ % Add termination criteria to the FailureProbability XsimOut(end).SexitFlag=SexitFlag; -XsimOut(end).SbatchFolder=[OpenCossan.getCossanWorkingPath filesep Xobj.SbatchFolder]; +XsimOut(end).SbatchFolder=[opencossan.OpenCossan.getWorkingPath filesep Xobj.SbatchFolder]; %% Set random number generator to state prior to running simulation -if exist('XRandomNumberGenerator','var'), +if exist('XRandomNumberGenerator','var') Simulations.restoreRandomNumberGenerator(XRandomNumberGenerator) end -OpenCossan.setLaptime('description','End apply@LatinHypercubeSampling'); +opencossan.OpenCossan.getTimer().lap('Description','End apply@LatinHypercubeSampling'); % Restore Global Random Stream restoreRandomStream(Xobj); diff --git a/+opencossan/+workers/@Evaluator/Evaluator.m b/+opencossan/+workers/@Evaluator/Evaluator.m index 87d8ca78..b3483715 100644 --- a/+opencossan/+workers/@Evaluator/Evaluator.m +++ b/+opencossan/+workers/@Evaluator/Evaluator.m @@ -25,9 +25,9 @@ properties JobManager opencossan.highperformancecomputing.JobManagerInterface % Define JobManager to submit job to grid/cluster computer - Solver(1,:) % List of OpenCossan Workers opencossan.workers.Worker + Solver(1,:) opencossan.workers.Worker % List of OpenCossan Workers opencossan.workers.Worker SolverName(1,:) string % Names of the workers (optional) - VerticalSplit = false % if true split the analysis in vertical components + VerticalSplit logical % if true split the analysis in vertical compartments end properties (Dependent=true) @@ -50,6 +50,7 @@ % Crate empty object superArg={}; else + idxSolver=find(strcmpi(varargin,'solver')); [requiredArgs, varargin] = opencossan.common.utilities.parseRequiredNameValuePairs(... "Solver", varargin{:}); @@ -57,9 +58,8 @@ OptionalsArguments={... "JobManager", opencossan.highperformancecomputing.JobManagerInterface.empty(1,0);... - "Queues","";... - "SolverName",[];... - "VerticalSplit",true}; + "VerticalSplit",false;... + "SolverName",[]}; [optionalArg, superArg] = opencossan.common.utilities.parseOptionalNameValuePairs(... [OptionalsArguments{:,1}],{OptionalsArguments{:,2}}, varargin{:}); @@ -73,18 +73,15 @@ obj.Solver = requiredArgs.solver; - obj.SolverName = optionalArg.solvername; - - if ~isempty(obj.SolverName) - assert(numel(obj.Solver) == numel(obj.SolverName),... - 'Evaluator:IllegalArguments',... - 'Number of solvers (%i) specified must be the same size of SolverName (%i)',... - numel(obj.Solver),numel(obj.SolverName)); + if isempty(optionalArg.solvername) + obj.SolverName = inputname(idxSolver+1); + else + obj.SolverName = optionalArg.solvername; end obj.VerticalSplit = optionalArg.verticalsplit; - obj=validateObject(obj); + obj=validateObject(obj); end diff --git a/+opencossan/+workers/@Evaluator/add.m b/+opencossan/+workers/@Evaluator/add.m index ab0b1b7d..4be456c3 100644 --- a/+opencossan/+workers/@Evaluator/add.m +++ b/+opencossan/+workers/@Evaluator/add.m @@ -1,5 +1,9 @@ function obj=add(obj,varargin) -%ADD This method adds a worker object to the current Evaluator +%ADD This method adds a worker object to the current Evaluator. +% +% Mandatory arguments: +% * Solver +% * SolverName % % See also: Evaluator, Worker % @@ -24,34 +28,11 @@ You should have received a copy of the GNU General Public License along with OpenCossan. If not, see . %} -[requiredArgs, varargin] = opencossan.common.utilities.parseRequiredNameValuePairs(... - "Solver", varargin{:}); - -% Define optional arguments and default values - -OptionalsArguments={... - "Queues","";... - "Hostnames","";... - "ParallelEnvironments","";... - "Slots",[];... - "MaxCuncurrentJobs",inf;... - "RemoteInjectExtract", false;... - "MaxNumberofJobs",[];... - "SolverName",""}; - -[optionalArg, ~] = opencossan.common.utilities.parseOptionalNameValuePairs(... - [OptionalsArguments{:,1}],{OptionalsArguments{:,2}}, varargin{:}); +[requiredArgs, ~] = opencossan.common.utilities.parseRequiredNameValuePairs(... + ["Solver" "SolverName"], varargin{:}); obj.Solver = [obj.Solver requiredArgs.solver]; - -obj.SolverName = [obj.SolverName optionalArg.solvername]; - -obj.Queues = [obj.Queues, optionalArg.queues]; -obj.Hostnames = [obj.Hostnames, optionalArg.hostnames]; -obj.ParallelEnvironments = [obj.ParallelEnvironments, optionalArg.parallelenvironments]; -obj.Slots = [obj.Slots, optionalArg.slots]; -obj.MaxCuncurrentJobs = [obj.MaxCuncurrentJobs, optionalArg.maxcuncurrentjobs]; -obj.MaxNumberofJobs = optionalArg.maxnumberofjobs; +obj.SolverName = [obj.SolverName requiredArgs.solvername]; % Validate object obj=validateObject(obj); diff --git a/+opencossan/+workers/@Evaluator/executeWorkersHorizontal.m b/+opencossan/+workers/@Evaluator/executeWorkersHorizontal.m index 55e9177f..3a876849 100644 --- a/+opencossan/+workers/@Evaluator/executeWorkersHorizontal.m +++ b/+opencossan/+workers/@Evaluator/executeWorkersHorizontal.m @@ -6,30 +6,31 @@ % % Usage: TableOutput = executeWorkersVertical(Xobj,TableInput) % -% See Also: http://cossan.co.uk/wiki/index.php/executeWorkersHorizontal@Evaluator +% See Also: Evaluator % % % Author: Edoardo Patelli -% Institute for Risk and Uncertainty, University of Liverpool, UK +% Cossan Working Group % email address: openengine@cossan.co.uk % Website: http://www.cossan.co.uk -% ===================================================================== -% This file is part of openCOSSAN. The open general purpose matlab -% toolbox for numerical analysis, risk and uncertainty quantification. -% -% openCOSSAN is free software: you can redistribute it and/or modify -% it under the terms of the GNU General Public License as published by -% the Free Software Foundation, either version 3 of the License. -% -% openCOSSAN is distributed in the hope that it will be useful, -% but WITHOUT ANY WARRANTY; without even the implied warranty of -% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -% GNU General Public License for more details. -% -% You should have received a copy of the GNU General Public License -% along with openCOSSAN. If not, see . -% ===================================================================== +%{ + This file is part of OpenCossan . + Copyright (C) 2006-2020 COSSAN WORKING GROUP + + OpenCossan is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License or, + (at your option) any later version. + + OpenCossan is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with OpenCossan. If not, see . +%} import opencossan.* %% Initialization @@ -79,7 +80,7 @@ % Construct a tableOutputSolverTmp with NaN TableOutputSolverTmp=array2table(NaN(height(TableSolver),length(Xobj.Solver(n).OutputNames)),... - 'VariableNames',Xobj.Solver{n}.OutputNames); + 'VariableNames',Xobj.Solver(n).OutputNames); end % Merge workers output for the currenct analysis TableOutputSolver=[TableOutputSolver, TableOutputSolverTmp]; %#ok diff --git a/+opencossan/+workers/@Evaluator/executeWorkersVertical.m b/+opencossan/+workers/@Evaluator/executeWorkersVertical.m index cfb40962..ebcc6acd 100644 --- a/+opencossan/+workers/@Evaluator/executeWorkersVertical.m +++ b/+opencossan/+workers/@Evaluator/executeWorkersVertical.m @@ -6,55 +6,56 @@ % % Usage: tableOutput = executeWorkersVertical(Xobj,tableInput) % -% See Also: http://cossan.co.uk/wiki/index.php/executeWorkersVertical@Evaluator +% See Also: Evaluator JobManager % % % Author: Edoardo Patelli -% Institute for Risk and Uncertainty, University of Liverpool, UK +% Cossan Working Group % email address: openengine@cossan.co.uk % Website: http://www.cossan.co.uk -% ===================================================================== -% This file is part of openCOSSAN. The open general purpose matlab -% toolbox for numerical analysis, risk and uncertainty quantification. -% -% openCOSSAN is free software: you can redistribute it and/or modify -% it under the terms of the GNU General Public License as published by -% the Free Software Foundation, either version 3 of the License. -% -% openCOSSAN is distributed in the hope that it will be useful, -% but WITHOUT ANY WARRANTY; without even the implied warranty of -% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -% GNU General Public License for more details. -% -% You should have received a copy of the GNU General Public License -% along with openCOSSAN. If not, see . -% ===================================================================== +%{ + This file is part of OpenCossan . + Copyright (C) 2006-2020 COSSAN WORKING GROUP + OpenCossan is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License or, + (at your option) any later version. + + OpenCossan is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with OpenCossan. If not, see . +%} %% Initialization assert(istable(TableInput),'OpenCossan:Evaluator:executeWorkersVertical:wrongInput',... 'A Tables object is required as input.\nProvided class %s is not valid',class(TableInput)) % Retrieve Analysis object; -Xanalysis=OpenCossan.getAnalysis; +Xanalysis=opencossan.OpenCossan.getAnalysis; % Preallocate table -TableOutput=array2table(zeros(height(TableInput),length(Xobj.Coutputnames)),... - 'VariableNames',Xobj.Coutputnames); +TableOutput=array2table(zeros(height(TableInput),length(Xobj.OutputNames)),... + 'VariableNames',Xobj.OutputNames); %% Analysis % The analysis is split over the number of samples and the workers are -% executed sequentially. The n-worker starts after only after the +% executed sequentially. The n-worker starts after only after completition of the % execution of the n-1 worker for ns=1:height(TableInput) % Predefine and reset tableOutput TableOutputSolver=[]; - + opencossan.OpenCossan.cossanDisp(['[Status:workers ] * Sample ' ... + num2str(ns) '/' num2str(height(TableInput))],4) % Evaluator execution - for n=1:length(Xobj.CXsolvers) - OpenCossan.cossanDisp(['[Status:workers ] * Processing solver ' ... - num2str(n) '/' num2str(length(Xobj.CXsolvers))],4) + for n=1:length(Xobj.Solver) + opencossan.OpenCossan.cossanDisp(['[Status:workers ] * Processing solver ' ... + num2str(n) '/' num2str(length(Xobj.Solver))],4) % Merge tableInput with output produced by the workers and then % pass only the inputs required by the specific worker. @@ -68,10 +69,10 @@ % Execute worker. % Only the input required by the worker are passes. No recheck % is needed in the evaluate method. - TableOutputSolverTmp=Xobj.CXsolvers{n}.evaluate(TableSolver(:,Xobj.CXsolvers{n}.Cinputnames)); + TableOutputSolverTmp=Xobj.Solver(n).evaluate(TableSolver(:,Xobj.Solver(n).InputNames)); catch Exception warning('OpenCossan:Evaluator:executeWorkersVertical:workerFaild',... - 'Unable to execute worker %i of type %s',n,class(Xobj.CXsolvers{n})) + 'Unable to execute worker %i of type %s',n,class(Xobj.Solver{n})) % Add meaningful error message msgID = 'OpenCossan:Evaluator:executeWorkersVertical:workerFaild'; msg = sprintf('Unable to execute worker %i for realization #%i.',n,ns); @@ -81,8 +82,8 @@ Xanalysis.Cerrors{end+1} = addCause(Exception,causeException); % Construct a tableOutputSolverTmp with NaN - TableOutputSolverTmp=array2table(NaN(1,length(Xobj.CXsolvers{n}.Coutputnames)),... - 'VariableNames',Xobj.CXsolvers{n}.Coutputnames); + TableOutputSolverTmp=array2table(NaN(1,length(Xobj.Solver(n).OutputNames)),... + 'VariableNames',Xobj.Solver(n).OutputNames); end % Merge workers output for the currenct analysis TableOutputSolver=[TableOutputSolver, TableOutputSolverTmp]; %#ok diff --git a/+opencossan/+workers/@Evaluator/validateObject.m b/+opencossan/+workers/@Evaluator/validateObject.m index 32b630c7..a5764607 100644 --- a/+opencossan/+workers/@Evaluator/validateObject.m +++ b/+opencossan/+workers/@Evaluator/validateObject.m @@ -25,64 +25,15 @@ along with OpenCossan. If not, see . %} -% Validate objects -if obj.VerticalSplit - Nworkers=1; -else - Nworkers=length(obj.Solver); -end -if ~isempty(obj.SolverName) - assert(length(obj.SolverName)==Nworkers,... - 'Evaluator:SolverName:wrongSize',... - ['Length of SolverNames (' num2str(length(obj.SolverName)) ... - ') must be equal to the length of Solver (' ... - num2str(Nworkers) ')' ]) -else - obj.SolverName=repmat("N/A",Nworkers,1); -end +assert(numel(obj.Solver) == numel(obj.SolverName),... + 'Evaluator:IllegalArguments',... + 'Number of solvers (%i) specified must be the same size of SolverName (%i)',... + numel(obj.Solver),numel(obj.SolverName)); + -if ~isempty(obj.MaxCuncurrentJobs) - assert(length(obj.MaxCuncurrentJobs)==Nworkers,... - 'Evaluator:MaxCuncurrentJob:wrongSize',... - ['Length of MaxCuncurrentJobs (' num2str(length(obj.MaxCuncurrentJobs)) ... - ') must be equal to the length of Solver (' ... - num2str(Nworkers) ')' ]) -else - obj.MaxCuncurrentJobs=inf(size(obj.Solver)); -end -if ~isempty(obj.Queues) - assert(length(obj.Queues)==Nworkers,... - 'Evaluator:Queues:wrongSize',... - 'Length of Queues (%i) must be equal to the length of Solver (%i)',... - length(obj.Queues),Nworkers) -end -if ~isempty(obj.Hostnames) - assert(length(obj.Hostnames)==Nworkers,... - 'Evaluator:Hostnames:wrongSize',... - 'Length of Hostnames (%i) must be equal to the length of Solver (%i)',... - length(obj.Hostnames),Nworkers) -end - -if ~isempty(obj.ParallelEnvironments) - assert(length(obj.ParallelEnvironments)==Nworkers,... - 'openCOSSAN:Evaluator',... - ['Length of ParallelEnvironments (' num2str(length(obj.ParallelEnvironments)) ... - ') must be equal to the length of Solver (' ... - num2str(Nworkers) ')' ]) -end - -if ~isempty(obj.Slots) - assert(length(obj.Slots)==Nworkers,... - 'openCOSSAN:Evaluator',... - ['Length of Slots (' num2str(length(obj.Slots)) ... - ') must be equal to the length of Solver (' ... - num2str(Nworkers) ')' ]) -else - obj.Slots=ones(size(obj.Solver)); -end diff --git a/+opencossan/+workers/@MatlabWorker/MatlabWorker.m b/+opencossan/+workers/@MatlabWorker/MatlabWorker.m index 0af4ea74..bf0d571d 100644 --- a/+opencossan/+workers/@MatlabWorker/MatlabWorker.m +++ b/+opencossan/+workers/@MatlabWorker/MatlabWorker.m @@ -1,13 +1,13 @@ classdef MatlabWorker < opencossan.workers.Worker % Matlab Input/Output interface - % See also: Worker, Evaluator, Connector % + % See also: Worker, Evaluator, Connector + % Author: Edoardo Patelli, Matteo Broggi, Marco De Angelis - % Institute for Risk and Uncertainty, University of Liverpool, UK % email address: openengine@cossan.co.uk % Website: http://www.cossan.co.uk - %{ +%{ This file is part of OpenCossan . Copyright (C) 2006-2020 COSSAN WORKING GROUP @@ -21,22 +21,22 @@ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. -You should have received a copy of the GNU General Public License -along with OpenCossan. If not, see . - %} + You should have received a copy of the GNU General Public License + along with OpenCossan. If not, see . +%} %% Properties of the object properties % Public access - FullFileName(1,:) char % Default Full path of the matlab file + FullFileName(1,:) char % Full path of the matlab file Format(1,1) string = 'table' % Define interface to the matlab file IsFunction(1,1) logical % Define if the matlab script is a function AdditionalPath(1,1) % Folder that contains additional files required for compiling MatlabWorker Script(1,:) char % field that contains the script end - properties (SetAccess=private,Hidden=true)% Define private propeties - IsCompiled logical = false % compiled status - FunctionHandle % function handle for external file + properties (SetAccess=private) % Define private propeties + IsCompiled logical = false % compiled status + FunctionHandle % function handle for external file end properties (Constant,Hidden) @@ -63,6 +63,13 @@ % input to the corresponding output, using a Matlab % function defined in an m-file % + % PropertyName: + % * FullFileName + % * Format + % * AdditionalPath + % * Script + % * FunctionHandle + % % See also: Worker, @@ -74,7 +81,6 @@ % Define optional arguments with default values OptionalsArguments={"FullFileName", [];... "Format","table";... - "IsFunction",false;... "AdditionalPath","";... "Script",[];... "FunctionHandle",[]}; @@ -91,7 +97,6 @@ obj.FullFileName = optionalArg.fullfilename; obj.Format = optionalArg.format; - obj.IsFunction = optionalArg.isfunction; obj.AdditionalPath = optionalArg.additionalpath; obj.Script = optionalArg.script; obj.FunctionHandle = optionalArg.functionhandle; diff --git a/+opencossan/+workers/@MatlabWorker/createSimulationData.m b/+opencossan/+workers/@MatlabWorker/createSimulationData.m index 5b21bac9..85b81f3f 100644 --- a/+opencossan/+workers/@MatlabWorker/createSimulationData.m +++ b/+opencossan/+workers/@MatlabWorker/createSimulationData.m @@ -6,7 +6,7 @@ case 'double' % check that the number of returned quantities is correct assert(size(Poutput,2)==length(Xmio.OutputNames),... - 'openCOSSAN:mio:createSimulationOutput',... + 'OpenCossanMatlabWorker:WrongNumberOfOutputDouble',... ['Wrong number of outputs returned by script/function.\n'... 'Nr. of required outputs: %d\n'... 'Nr. of columns of output matrix: %d'],... @@ -17,18 +17,18 @@ % check that all the required outputs are available as field of the % output structure Cnames = fieldnames(Poutput); - assert(all(ismember(Xmio.Coutputnames,Cnames)),... - 'openCOSSAN:mio:createSimulationOutput',... + assert(all(ismember(Xmio.OutputNames,Cnames)),... + 'OpenCossan:MatlabWorker:WrongNumberOfOutputStructure',... ['Not all the necessary outputs are contained in the output structure.\n'... 'Missing outputs: %s'], ... - sprintf('%s\n',Xmio.Coutputnames{~ismember(Xmio.Coutputnames,Cnames)})) + sprintf('%s\n',Xmio.OutputNames{~ismember(Xmio.OutputNames,Cnames)})) % check that no empty values are returned into the structure for ifield=1:length(Cnames) Ctemp = {Poutput.(Cnames{ifield})}; LemptyCell = cellfun('isempty',Ctemp); assert(~any(LemptyCell),... - 'openCOSSAN:mio:createSimulationOutput',... + 'OpenCossan:MatlabWorker:createSimulationOutput',... ['Output of name %s returned an empty quantity for samples %s.\n',... 'Please ensure that your script or function alway return non-empty quantities.'],... Cnames{ifield},num2str(find(LemptyCell),'%d,')) @@ -38,8 +38,8 @@ % create the simulation data XsimOut = SimulationData('Table',Poutput); otherwise - error('openCOSSAN:mio:createSimulationOutput',... - 'Unsupported output from Mio of type %s', class(Poutput)) + error('OpenCossan:MatlabWorker:unsupportedType',... + 'Unsupported output from MatlabWorker of type %s', class(Poutput)) end %% Check Outputs @@ -51,6 +51,6 @@ end %% Export results -XsimOut.Sdescription = 'generated by: run(@mio)'; %Sets description of Output object +XsimOut.Sdescription = 'generated by: run(@MatlabWorker)'; %Sets description of Output object end diff --git a/+opencossan/+workers/@MatlabWorker/evaluate.m b/+opencossan/+workers/@MatlabWorker/evaluate.m index 49abb448..deda9bf7 100644 --- a/+opencossan/+workers/@MatlabWorker/evaluate.m +++ b/+opencossan/+workers/@MatlabWorker/evaluate.m @@ -28,10 +28,13 @@ % along with openCOSSAN. If not, see . % ===================================================================== -% Use only required inputs. If some names are not present in the TableInput -% the following line is failing. -TableInput=TableInput(:,Xmio.InputNames); +assert(istable(TableInput),... + 'Provided input type %s is not valid in this method. A table is required',class(TableInput)) + % Use only required inputs. If some names are not present in the TableInput + % the following line is failing. + TableInput=TableInput(:,Xmio.InputNames); + %% Prepare inputs switch lower(Xmio.Format) case 'structure' @@ -87,6 +90,8 @@ TableOutput=array2table(Poutput,'VariableNames',Xmio.OutputNames); case 'table' + + if Xmio.IsFunction TableOutput = feval(Xmio.FunctionHandle,TableInput); else diff --git a/+opencossan/+workers/@MatlabWorker/validateConstructor.m b/+opencossan/+workers/@MatlabWorker/validateConstructor.m index 3224f6e2..d6963dc0 100644 --- a/+opencossan/+workers/@MatlabWorker/validateConstructor.m +++ b/+opencossan/+workers/@MatlabWorker/validateConstructor.m @@ -1,10 +1,9 @@ -function Xobj=validateConstructor(Xobj) +function obj=validateConstructor(obj) % VALIDATECONSTRUCTOR this private method check if the object is % constructed correctly. -% See also: https://cossan.co.uk/wiki/index.php/@Mio +% See also: Mio % % Author: Edoardo Patelli and Matteo Broggi -% Institute for Risk and Uncertainty, University of Liverpool, UK % email address: openengine@cossan.co.uk % Website: http://www.cossan.co.uk @@ -26,99 +25,93 @@ % ===================================================================== import opencossan.OpenCossan; -OpenCossan.cossanDisp('[Mio:validateConstructor] Validate Constructor',4) +OpenCossan.cossanDisp('[MatlabWorker:validateConstructor] Validate Constructor',4) -assert(~isempty(Xobj.OutputNames),'openCOSSAN:Mio',... +assert(~isempty(obj.OutputNames),'OpenCossan:MatlabWorker:EmptyOutputNames',... 'The names of the Output variables must be defined in the field OutputNames'); -if size(Xobj.OutputNames,1)>1 - Xobj.OutputNames=Xobj.OutputNames'; +if size(obj.OutputNames,1)>1 + obj.OutputNames=obj.OutputNames'; end -assert(~isempty(Xobj.InputNames),'openCOSSAN:Mio',... +assert(~isempty(obj.InputNames),'OpenCossan:MatlabWorker:EmptyInputNames',... 'The names of the Input variables should be defined in the field InputNames'); -if size(Xobj.InputNames,1)>1 - Xobj.InputNames=Xobj.InputNames'; +if size(obj.InputNames,1)>1 + obj.InputNames=obj.InputNames'; end -% Existance of m-file in directory Spath is checked -if isempty(Xobj.Script) && isempty(Xobj.FunctionHandle) +% Existance of m-file in directory Path is checked +if isempty(obj.Script) && isempty(obj.FunctionHandle) % check that an absolute path is given - assert(~isempty(Xobj.FullFileName),'openCOSSAN:Mio',... - 'A file name must be supplied.') + assert(~isempty(obj.FullFileName),'OpenCossan:MatlabWorker:NoFileName',... + 'A filename must be supplied if script of a function handle are not provided.') if ~isdeployed % if not deployed - %% If is a Mio function, check if the file in already in the path - if Xobj.IsFunction - % Check if the provided files is really a function - - [Nfid,Smessage]=fopen(Xobj.FullFileName); - - assert(isempty(Smessage),'openCOSSAN:Mio',... - 'The Script or Function specified in the field ''Spath/Sname'' (%s) can not be open\nError message: %s',... - Xobj.FullFileName,Smessage); - - + % Check if the file provided is a function or a script + + [Nfid,Smessage]=fopen(obj.FullFileName); + + assert(isempty(Smessage),'OpenCossan:MatlabWorker:ErrorOpeningFile',... + ['The file specified in the field ''FullFileName'' (%s) ',... + 'can not be open\nError message: %s'],... + obj.FullFileName,Smessage); + + % Read the file and check if it is a function or a script + Sline=fgetl(Nfid); + while isempty(regexp(Sline, '^%','once')) && ~isempty(Sline) Sline=fgetl(Nfid); - while 1 - if isempty(regexp(Sline, '^%','once')) && ~isempty(Sline) - % No comment identified, check for function - - assert(~isempty(regexp(Sline, '^function','once')),'openCOSSAN:Mio',... - 'The provided file (%s) does not seem to be a function!!\nPlease check the Lfunction flag and your file',Xobj.FullFileName); - break - else - % Process next line - Sline=fgetl(Nfid); - end - end - fclose(Nfid); - - opencossan.OpenCossan.cossanDisp(['[Mio:validateConstructor] convert : ' ... - Xobj.FullFileName ' to Handle function'] ,4) - [Spath,Sfile,Sext] = fileparts(Xobj.FullFileName); - if isempty(which([Sfile,Sext])) - addpath(Spath); - Xobj.FunctionHandle=str2func(Sfile); - rmpath(Spath); - else - Xobj.FunctionHandle=str2func(Sfile); - end - - end % nothing to do if is a script + end - if Xobj.IsFunction - % TODO: Include check of the file. It should at least contain - % the word function. - + % No comment identified, check for function + if isempty(regexp(Sline, '^function','once')) + obj.IsFunction=false; else - % TODO: Include check of the file. + obj.IsFunction=true; end - + + fclose(Nfid); + + opencossan.OpenCossan.cossanDisp(['[MatlabWorker:validateConstructor] convert : ' ... + obj.FullFileName ' to Handle function'] ,4) + [Spath,Sfile,Sext] = fileparts(obj.FullFileName); + if isempty(which([Sfile,Sext])) + addpath(Spath); + obj.FunctionHandle=str2func(Sfile); + rmpath(Spath); + else + obj.FunctionHandle=str2func(Sfile); + end + + + % nothing to do if is a script + + else %if is deployed version %% If is a Script, put the content of the script file in Sscript - if ~Xobj.IsFunction + if ~obj.IsFunction OpenCossan.cossanDisp(['[Mio:validateConstructor] convert : ' ... - Xobj.FullFileName ' script to a single string'] ,4) - Nfid = fopen(Xobj.FullFileName); + obj.FullFileName ' script to a single string'] ,4) + Nfid = fopen(obj.FullFileName); Vbytes = fread(Nfid); fclose(Nfid); % remove carriage return Vbytes(Vbytes==13)=[]; % substitute line feeds (newlines) with commas Vbytes(Vbytes==10)=44; - Xobj.Script=char(Vbytes'); + obj.Script=char(Vbytes'); end % nothing to do if is a function end end if ~isdeployed - assert((~isempty(Xobj.Script) || ~isempty(Xobj.FunctionHandle) || exist(Xobj.FullFileName,'file')), ... - 'openCOSSAN:Mio',strcat('A function or a script is required by the Mio object\n',... - 'Please use the propertyName Sfile, Sscript or AfunctionHandle or check that the files exists. \nFile: %s'),Xobj.FullFileName); + assert((~isempty(obj.Script) || ~isempty(obj.FunctionHandle) || exist(obj.FullFileName,'file')), ... + 'OpenCossan:MatlabWorker:MissingMatlabFile',... + strcat('A function or a script is required by the MatlabWorker object\n',... + 'Please use the PropertyName File, Script or FunctionHandle or check that the file exists. \nFile: %s'),obj.FullFileName); else - assert((~isempty(Xobj.Script) || exist(Xobj.FullFileName,'file')), ... - 'openCOSSAN:Mio',strcat('A function or a script is required by the Mio object\n',... + assert((~isempty(obj.Script) || exist(obj.FullFileName,'file')), ... + 'OpenCossan:MatlabWorker:NoScript',... + strcat('A function or a script is required by the MatlabWorker object\n',... 'Please use the propertyName Sfile or Sscript')); end diff --git a/+opencossan/+workers/@Worker/Worker.m b/+opencossan/+workers/@Worker/Worker.m index 93f8cd49..94b2c112 100644 --- a/+opencossan/+workers/@Worker/Worker.m +++ b/+opencossan/+workers/@Worker/Worker.m @@ -32,7 +32,7 @@ OutputNames cell {opencossan.common.utilities.isunique(OutputNames)} InputNames cell {opencossan.common.utilities.isunique(InputNames)} KeepSimulationFiles(1,1) logical = false % Keep simulation files - ExecutionStrategy(1,1) opencossan.common.worker.ExecutionStrategy + % ExecutionStrategy opencossan.highperformancecomputing.ExecutionStrategy end methods (Abstract) @@ -65,7 +65,7 @@ obj.InputNames=requiredArgs.inputnames; obj.OutputNames=requiredArgs.outputnames; obj.KeepSimulationFiles=optionalArgs.keepsimulationfiles; - obj.ExecutionStrategy=optionalArgs.executionstrategy; +% obj.ExecutionStrategy=optionalArgs.executionstrategy; end end end diff --git a/+opencossan/@OpenCossan/validateCossanInputs.m b/+opencossan/@OpenCossan/validateCossanInputs.m index 9341f8ed..f2989042 100644 --- a/+opencossan/@OpenCossan/validateCossanInputs.m +++ b/+opencossan/@OpenCossan/validateCossanInputs.m @@ -22,7 +22,7 @@ function validateCossanInputs(varargin) along with OpenCossan. If not, see . %} -warning('This feature is obsolete. Use Validate Property Values. \n Use https://uk.mathworks.com/help/matlab/matlab_oop/validate-property-values.html') +warning('This feature is obsolete. Use Validate Property Values. /n Use https://uk.mathworks.com/help/matlab/matlab_oop/validate-property-values.html') %% Check if the arguments has been passed as pairs of PropertyName/Value if mod(length(varargin),2) diff --git a/docs/tutorials/CantileverBeam/TutorialCantileverBeamMatlab.m b/docs/tutorials/CantileverBeam/TutorialCantileverBeamMatlab.m index 4adeea37..d9aa86a8 100644 --- a/docs/tutorials/CantileverBeam/TutorialCantileverBeamMatlab.m +++ b/docs/tutorials/CantileverBeam/TutorialCantileverBeamMatlab.m @@ -77,9 +77,16 @@ Xmio = MatlabWorker('FullFileName', fullfile(currentFolder, 'model', 'tipDisplacement.m'), ... 'InputNames', {'I', 'b', 'L', 'h', 'rho', 'P', 'E'}, ... 'OutputNames', {'w'}, 'Format', 'structure'); -% Add the MIO object to an Evaluator object +% Add the MatlabWorker object to an Evaluator object +% By default all the samples are processed by the workers using an +% horizontal splitting strategy (i.e. all the samples processed by the +% first workers, then the results of the first worker are passed to the +% second worker and so on). Xevaluator = Evaluator('Solver', Xmio, 'SolverName',"Xmio"); +% To split the execution in vertical use set "VerticalSplit",true); +%Xevaluator.VerticalSplit=true; + %% Preparation of the Physical Model % Define the Physical Model XmodelBeamMatlab = Model('Input', Xinput, 'Evaluator', Xevaluator); @@ -95,7 +102,7 @@ %% Uncertainty Quantification % Define simulation method -Xmc = MonteCarlo('Nsamples', 1000); +Xmc = MonteCarlo('Nsamples', 100); % preform Analysis XsimOutMC = Xmc.apply(XmodelBeamMatlab); diff --git a/docs/tutorials/CantileverBeam/TutorialCantileverBeamMatlabRBO.m b/docs/tutorials/CantileverBeam/TutorialCantileverBeamMatlabRBO.m index 4c263241..53ee81a8 100644 --- a/docs/tutorials/CantileverBeam/TutorialCantileverBeamMatlabRBO.m +++ b/docs/tutorials/CantileverBeam/TutorialCantileverBeamMatlabRBO.m @@ -35,6 +35,9 @@ with OpenCossan. If not, see . %} +%% Import packages +import opencossan.optimization.* + %% Check avilability of the probabilistic model assert(logical(exist('XprobModelBeamMatlab','var')),'openCOSSAN:Tutorial', ... 'Please run first the tutorial TutorialCantileverBeamMatlabReliabilityAnalysis') diff --git a/docs/tutorials/CantileverBeam/importpackages.m b/docs/tutorials/CantileverBeam/importpackages.m new file mode 100644 index 00000000..69792870 --- /dev/null +++ b/docs/tutorials/CantileverBeam/importpackages.m @@ -0,0 +1,8 @@ +%% Import packages +import opencossan.* +import opencossan.reliability.* +import opencossan.common.inputs.* +import opencossan.common.inputs.random.* +import opencossan.sensitivity.* +import opencossan.optimization.* +import opencossan.simulations.* diff --git a/docs/tutorials/CossanObjects/Files4MatlabWorker/ExampleMatlabWorkerFunction.m b/docs/tutorials/CossanObjects/Files4MatlabWorker/ExampleMatlabWorkerFunction.m index 76721f60..19c58333 100644 --- a/docs/tutorials/CossanObjects/Files4MatlabWorker/ExampleMatlabWorkerFunction.m +++ b/docs/tutorials/CossanObjects/Files4MatlabWorker/ExampleMatlabWorkerFunction.m @@ -1,4 +1,4 @@ -function [Voutput1, Voutput2] = ExampleMioFunction(Vinput1,Vinput2,Par1) +function [Voutput1, Voutput2] = ExampleMatlabWorkerFunction(Vinput1,Vinput2,Par1) % Example of Mio Function % The mio function receives 3 inputs and returns 2 outputs. Voutput1=Vinput2; diff --git a/docs/tutorials/CossanObjects/Files4MatlabWorker/ExampleMatlabWorkerMatrix.m b/docs/tutorials/CossanObjects/Files4MatlabWorker/ExampleMatlabWorkerMatrix.m index 161cdf8c..8e266629 100644 --- a/docs/tutorials/CossanObjects/Files4MatlabWorker/ExampleMatlabWorkerMatrix.m +++ b/docs/tutorials/CossanObjects/Files4MatlabWorker/ExampleMatlabWorkerMatrix.m @@ -1,4 +1,4 @@ -function Moutput = ExampleMioMatrix(Minput) +function Moutput = ExampleMatlabWorkerMatrix(Minput) % Example of Mio Function % Input and output are passed as an array of numbers diff --git a/docs/tutorials/CossanObjects/Files4MatlabWorker/ExampleMatlabWorkerStructure.m b/docs/tutorials/CossanObjects/Files4MatlabWorker/ExampleMatlabWorkerStructure.m index 6798460a..12a62823 100644 --- a/docs/tutorials/CossanObjects/Files4MatlabWorker/ExampleMatlabWorkerStructure.m +++ b/docs/tutorials/CossanObjects/Files4MatlabWorker/ExampleMatlabWorkerStructure.m @@ -1,4 +1,4 @@ -function Toutput = ExampleMioStructure(Tinput) +function Toutput = ExampleMatlabWorkerStructure(Tinput) % Example of Mio Function % Input and output passed as Structure % Required Inputs RV_1, RV_2 diff --git a/docs/tutorials/CossanObjects/Files4MatlabWorker/ExampleMatlabWorkerTable.m b/docs/tutorials/CossanObjects/Files4MatlabWorker/ExampleMatlabWorkerTable.m index 8671d5ee..105efa8e 100755 --- a/docs/tutorials/CossanObjects/Files4MatlabWorker/ExampleMatlabWorkerTable.m +++ b/docs/tutorials/CossanObjects/Files4MatlabWorker/ExampleMatlabWorkerTable.m @@ -1,4 +1,4 @@ -function TableOutput = ExampleMioTable(TableInput) +function TableOutput = ExampleMatlabWorkerTable(TableInput) % Example of Mio Function. % % Input and output passed as Matlab Table diff --git a/docs/tutorials/CossanObjects/TutorialAdaptiveLineSampling.m b/docs/tutorials/CossanObjects/TutorialAdaptiveLineSampling.m index ba425294..b46b3ccf 100644 --- a/docs/tutorials/CossanObjects/TutorialAdaptiveLineSampling.m +++ b/docs/tutorials/CossanObjects/TutorialAdaptiveLineSampling.m @@ -22,19 +22,17 @@ %% Define the Evaluator (i.e. how our model is evaluated) % Construct a Mio object -Xmio_model=opencossan.workers.Mio('Description', 'Matlab I-O for the performance model',... +Xmio_model=opencossan.workers.MatlabWorker('Description', 'Matlab I-O for the performance model',... 'Script',... 'Moutput=2-Minput(:,2)-0.1*Minput(:,1).^2+0.06*Minput(:,1).^3;',... 'Format','matrix',... -... 'Liostructure',false,... -... 'Liomatrix',true,... 'IsFunction',false,... 'OutputNames',{'performance_function'},... 'InputNames',{'X1' 'X2'}); % Construct the evaluator object -Xevaluator = opencossan.workers.Evaluator('Xmio',Xmio_model); +Xevaluator = opencossan.workers.Evaluator('Solver',Xmio_model); %% Define the physical model -Xmodel=opencossan.common.Model('Xevaluator',Xevaluator,'Xinput',Xinput); +Xmodel=opencossan.common.Model('Evaluator',Xevaluator,'Input',Xinput); % Test the Model % Generate 10 random realization of the input Xinput = sample(Xinput,'Nsamples',10); @@ -55,7 +53,7 @@ % Xperformance=PerformanceFunction('Sdescription','My Performance Function', ... % 'Xmio',Xmio_performance); % Construct the Probabilisti Model object -XprobModel=opencossan.reliability.ProbabilisticModel('Xmodel',Xmodel,'XperformanceFunction',Xmio_performance); +XprobModel=opencossan.reliability.ProbabilisticModel('Model',Xmodel,'PerformanceFunction',Xmio_performance); %% Define an Important Direction % Construct the Local Sensitivity by Finite Difference Xlsfd=opencossan.sensitivity.LocalSensitivityFiniteDifference('Xtarget',XprobModel, ... diff --git a/docs/tutorials/CossanObjects/TutorialAdvancedLineSampling.m b/docs/tutorials/CossanObjects/TutorialAdvancedLineSampling.m index cf3488d7..9937887d 100644 --- a/docs/tutorials/CossanObjects/TutorialAdvancedLineSampling.m +++ b/docs/tutorials/CossanObjects/TutorialAdvancedLineSampling.m @@ -21,7 +21,7 @@ Xinput = opencossan.common.inputs.Input('Members',{Xrvs},'MembersNames',{'Xrvs'}); %% Define the Evaluator (i.e. how our model is evaluated) % Construct a Mio object -Xmio_model=opencossan.workers.Mio('Description', 'Matlab I-O for the performance model',... +Xmio_model=opencossan.workers.MatlabWorker('Description', 'Matlab I-O for the performance model',... 'Script',... 'Moutput=2-Minput(:,2)-0.1*Minput(:,1).^2+0.06*Minput(:,1).^3;',... ... 'Liostructure',false,... @@ -30,9 +30,9 @@ 'OutputNames',{'performance_function'},... 'InputNames',{'X1' 'X2'}); % Construct the evaluator object -Xevaluator = opencossan.workers.Evaluator('Xmio',Xmio_model); +Xevaluator = opencossan.workers.Evaluator('Solver',Xmio_model); %% Define the physical model -Xmodel=opencossan.common.Model('Xevaluator',Xevaluator,'Xinput',Xinput); +Xmodel=opencossan.common.Model('Evaluator',Xevaluator,'Input',Xinput); % Test the Model % Generate 10 random realization of the input Xinput = sample(Xinput,'Nsamples',10); @@ -41,19 +41,13 @@ % Show Results display(Xo); %% Define the Probabilistic Model -Xmio_performance=opencossan.workers.Mio('Description', 'Matlab I-O for the performance function',... - 'Script','Moutput=Minput;',... -... 'Liostructure',false,... -... 'Liomatrix',true,... - 'IsFunction',false,... - 'InputNames',{'performance_function'},... - 'OutputNames',{'Vg'}); % Create the performance function object -Xperformance=opencossan.reliability.PerformanceFunction('Description','My Performance Function', ... - 'Xmio',Xmio_performance); +Xperformance=opencossan.reliability.PerformanceFunction('Description','My Performance Function', 'Script','Moutput=Minput;',... + 'InputNames',{'performance_function'},... + 'OutputName','Vg'); % Construct the Probabilisti Model object XprobModel=opencossan.reliability.ProbabilisticModel('Description','Defines our reliability analysis',... - 'Xmodel',Xmodel,'XperformanceFunction',Xperformance); + 'Model',Xmodel,'PerformanceFunction',Xperformance); %% Define an Important Direction % Construct the Local Sensitivity by Finite Difference Xlsfd=opencossan.sensitivity.LocalSensitivityFiniteDifference('Xtarget',XprobModel,'Coutputnames',{'Vg'}); diff --git a/docs/tutorials/CossanObjects/TutorialBobyqa.m b/docs/tutorials/CossanObjects/TutorialBobyqa.m index 25a2a428..5f95c359 100644 --- a/docs/tutorials/CossanObjects/TutorialBobyqa.m +++ b/docs/tutorials/CossanObjects/TutorialBobyqa.m @@ -35,14 +35,14 @@ %% Define a model SrosenbrockPath=fullfile(opencossan.OpenCossan.getRoot(),'lib','MatlabFunctions','Rosenbrock.m'); -Xm = opencossan.workers.Mio(... +Xm = opencossan.workers.MatlabWorker(... 'FullFileName', SrosenbrockPath,... 'IsFunction', true, ... 'Format', 'matrix', ... - 'InputNames', {'x1','x2'},... + 'InputNames', {'x1', 'x2'},... 'OutputNames', {'out'}); -evaluator = opencossan.workers.Evaluator('Xmio', Xm); +evaluator = opencossan.workers.Evaluator('Solver', Xm); model = opencossan.common.Model('Evaluator', evaluator, 'Input', input); %% Create objective function diff --git a/docs/tutorials/CossanObjects/TutorialCobyla.m b/docs/tutorials/CossanObjects/TutorialCobyla.m index 39e460b7..39fbc3f2 100644 --- a/docs/tutorials/CossanObjects/TutorialCobyla.m +++ b/docs/tutorials/CossanObjects/TutorialCobyla.m @@ -43,14 +43,14 @@ input = opencossan.common.inputs.Input('MembersNames',{'x1' 'x2'},'Members',{x1 x2}); %% Model -mio = opencossan.workers.Mio('Description','objective function of optimization problem', ... - 'FunctionHandle',@f, ... +mio = opencossan.workers.MatlabWorker('Description','objective function of optimization problem', ... + 'FunctionHandle',@myModel, ... 'Format','structure', ... 'IsFunction', true, ... 'InputNames',{'x1','x2'},... 'OutputNames',{'y'}); -evaluator = opencossan.workers.Evaluator('Xmio',mio); +evaluator = opencossan.workers.Evaluator('Solver',mio); model = opencossan.common.Model('evaluator',evaluator,'input',input); %% Objective function @@ -58,16 +58,14 @@ % optimization. objfun = opencossan.optimization.ObjectiveFunction('Description','objective function', ... - 'FunctionHandle',@g, ... - 'IsFunction', true, ... + 'FunctionHandle',@myObjective, ... 'Format', 'structure', ... 'InputNames',{'y'}, ... 'OutputNames',{'fobj'}); %% Create non linear inequality constraint constraint = opencossan.optimization.Constraint('Description','non linear inequality constraint', ... - 'FunctionHandle',@c, ... - 'IsFunction', true, ... + 'FunctionHandle',@myConstraint, ... 'Format', 'structure', ... 'OutputNames',{'con'},... 'InputNames',{'x1','x2'},... @@ -98,14 +96,14 @@ % To use a function handle as model/objective/constraint they have to be defined at the end of the % file. -function out = f(in) +function out = myModel(in) out.y = in.x1.^2 + in.x2.^2; end -function out = g(in) +function out = myObjective(in) out.fobj = in.y; end -function out = c(in) +function out = myConstraint(in) out.con = 2 - in.x1 - in.x2; end diff --git a/docs/tutorials/CossanObjects/TutorialConstraint.m b/docs/tutorials/CossanObjects/TutorialConstraint.m index c8ea028d..8569da62 100644 --- a/docs/tutorials/CossanObjects/TutorialConstraint.m +++ b/docs/tutorials/CossanObjects/TutorialConstraint.m @@ -5,21 +5,38 @@ % Please refer to the Mio tutorial and Optimization tutorial for more % examples of the constraints % -% See Also: http://cossan.co.uk/wiki/index.php/@Constraint +% See Also: Constraint, TutorialObjectiveFunction, TutorialOptimization % -% $Copyright~1993-2011,~COSSAN~Working~Group,~University~of~Liverpool,~UK$ -% $Author:~Edoardo~Patelli$ +%{ + This file is part of OpenCossan . + Copyright (C) 2006-2020 COSSAN WORKING GROUP + + OpenCossan is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License or, + (at your option) any later version. + + OpenCossan is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with OpenCossan. If not, see . +%} clear; close all clc; + +import opencossan.optimization.* %% Define a Constraint object % The fieds Linequality is used to define the type of constaint (equality % or inequality constraint) -Xcon = opencossan.optimization.Constraint('Description','non linear inequality constraint', ... +Xcon = Constraint('Description','non linear inequality constraint', ... 'Script','for n=1:length(Tinput),Toutput(n).Con1=2-Tinput(n).X1-Tinput(n).X2;end',... - 'Outputnames',{'Con1'},'Inputnames',{'X1','X2'},'Linequality',true); + 'Outputnames',{'Con1'},'Inputnames',{'X1','X2'},'IsInequality',true,'Format','structure'); % Show details of the Constraints display(Xcon) @@ -31,23 +48,32 @@ Xout=Xcon.run(Tinput); +%% Evaluate Constraint object +% Constraint can be evaluate by passing a table containing the name +% of variables defined in Constraint.InputNames + +TableInput=array2table([0 0; 4 3; 4, 1],'VariableName',{'X1','X2'}); +TableOutput=Xcon.evaluate(TableInput); + +% The constraint returns a table with name equal to Constraint.OutputNames +disp(TableOutput.Con1) display(Xout); %% Use a Constraint object in a Optimization Problem % Create input X1 = DesignVariable('Sdescription','design variable 1','value',7); X2 = DesignVariable('Sdescription','design variable 2','value',2); -Xinput = Input('CXmembers',{X1 X2},'CSobjectNames',{'X1' 'X2'}) +Xinput = Input('CXmembers',{X1 X2},'CSobjectNames',{'X1' 'X2'}); % Define an ObjectiveFunction -Xofun = ObjectiveFunction('Sdescription','objective function', ... - 'Cinputnames',{'X1','X2'},... % Define the inputs - 'Sscript','for n=1:length(Tinput),Toutput(n).fobj=Tinput(n).X1;end',... - 'Coutputnames',{'fobj'}); % Define the outputs +Xofun = ObjectiveFunction('Description','objective function', ... + 'Inputnames',{'X1','X2'},... % Define the inputs + 'Script','for n=1:length(Tinput),Toutput(n).fobj=Tinput(n).X1;end',... + 'OutputNames',{'fobj'},'Format','structure'); % Define the outputs % Define an Optimization Problem -Xop = OptimizationProblem('Sdescription','Optimization problem', ... - 'Xinput',Xin,'XobjectiveFunction',Xofun,'Xconstraint',Xcon); +Xop = OptimizationProblem('Description','Optimization problem', ... + 'Input',Xin,'ObjectiveFunction',Xofun,'Constraint',Xcon); % Show the optimization problem % The object contains 1 Objective Function and 1 Constraint display(Xop) diff --git a/docs/tutorials/CossanObjects/TutorialCovarianceFunction.m b/docs/tutorials/CossanObjects/TutorialCovarianceFunction.m index 1bd24efe..bfb07731 100644 --- a/docs/tutorials/CossanObjects/TutorialCovarianceFunction.m +++ b/docs/tutorials/CossanObjects/TutorialCovarianceFunction.m @@ -23,9 +23,6 @@ %% Define CovarianceFunction object Xcovfun1 = opencossan.common.inputs.stochasticprocess.CovarianceFunction('Description','covariance function', ... - 'IsFunction',false, ... -... 'Liostructure',true, ... % input and output are structures -... 'Liomatrix',false,... 'InputNames',{'t1','t2'},... % names of inputs 'Script',Sscript1,... % script with 1D exponential function 'OutputNames',{'fcov'}); % name of output @@ -33,7 +30,9 @@ %% Evaluate the values of the covariance function for time lags between 0 and 4 timesteps = linspace(0,4,100); -Vcov1 = Xcovfun1.evaluate([zeros(1, 100); timesteps]); +InputTable=array2table([zeros(1, 100); timesteps]','VariableNames',{'t1','t2'}); + +Vcov1 = Xcovfun1.evaluate(InputTable); %% Visualize and validate results @@ -57,12 +56,10 @@ %% Define CovarianceFunction object Xcovfun2 = CovarianceFunction('Sdescription','covariance function', ... - 'Lfunction',false, ... - 'Liostructure',true, ... % input and output are structures - 'Liomatrix',false,... - 'Cinputnames',{'x1','x2'},... % names of inputs - 'Sscript',Sscript2,... % script with 1D exponential function - 'Coutputnames',{'fcov'}); % name of output + 'format','structure', ... + 'InputNames',{'x1','x2'},... % names of inputs + 'Script',Sscript2,... % script with 1D exponential function + 'OutputNames',{'fcov'}); % name of output %% Visualize and validate results @@ -70,7 +67,10 @@ coordinates = meshgrid(linspace(0,4,100),linspace(0,4,100)); xcoordinates = coordinates(:)'; ycoordinates = repmat(linspace(0,4,100),1,100); -Vcov2 = Xcovfun2.evaluate([zeros(2, 10000);xcoordinates;ycoordinates]); + +InputTable=array2table([zeros(2, 10000);xcoordinates;ycoordinates]','VariableNames',{'x1','x2'}); + +Vcov2 = Xcovfun2.evaluate(InputTable); f2 = figure; plot3(xcoordinates,ycoordinates,Vcov2,'.') grid on diff --git a/docs/tutorials/CossanObjects/TutorialDesignOfExperiment.m b/docs/tutorials/CossanObjects/TutorialDesignOfExperiment.m index 746be9e0..abc12b19 100644 --- a/docs/tutorials/CossanObjects/TutorialDesignOfExperiment.m +++ b/docs/tutorials/CossanObjects/TutorialDesignOfExperiment.m @@ -4,15 +4,17 @@ % Please refer to the specific tutorials for the other objects available in % [COSSANEngine/examples/Tutorials] % -% See Also: http://cossan.cfd.liv.ac.uk/wiki/index.php/@DesignOfExperiment +% See Also: DesignOfExperiment % % -% $Copyright~1993-2011,~COSSAN~Working~Group,~University~of~Innsbruck,~Austria$ +% $Copyright~1993-2020,~COSSAN~Working~Group$ % $Author: Edoardo-Patelli$ clear; close all clc; + +import opencossan.simulations.* %% Example #1 - using only RandomVariables in Input % Define RandomVariablesSet diff --git a/docs/tutorials/CossanObjects/TutorialEvaluator.m b/docs/tutorials/CossanObjects/TutorialEvaluator.m index 66d8db28..926a2d15 100644 --- a/docs/tutorials/CossanObjects/TutorialEvaluator.m +++ b/docs/tutorials/CossanObjects/TutorialEvaluator.m @@ -4,29 +4,31 @@ % strategy and the interface with JobManager. % % -% See Also: http://cossan.co.uk/wiki/index.php/@Evaluator +% See Also: Evaluator TutorialModel % % $Author: Edoardo Patelli$ -% Copyright~1993-2015, COSSAN Working Group, University of Liverpool, UK +% COSSAN Working Group % email address: openengine@cossan.co.uk % Website: http://www.cossan.co.uk -% ===================================================================== -% This file is part of openCOSSAN. The open general purpose matlab -% toolbox for numerical analysis, risk and uncertainty quantification. -% -% openCOSSAN is free software: you can redistribute it and/or modify -% it under the terms of the GNU General Public License as published by -% the Free Software Foundation, either version 3 of the License. -% -% openCOSSAN is distributed in the hope that it will be useful, -% but WITHOUT ANY WARRANTY; without even the implied warranty of -% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -% GNU General Public License for more details. -% -% You should have received a copy of the GNU General Public License -% along with openCOSSAN. If not, see . -% ===================================================================== +%{ +This file is part of OpenCossan . +Copyright (C) 2006-2020 COSSAN WORKING GROUP + +OpenCossan is free software: you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free +Software Foundation, either version 3 of the License or, (at your option) +any later version. + +OpenCossan is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +more details. + +You should have received a copy of the GNU General Public License along +with OpenCossan. If not, see . +%} + clear; close all clc; @@ -40,14 +42,14 @@ % % Define a siple model based on a Matlab function % Construct a Mio object -Xm=opencossan.workers.Mio('description', 'Performance function', ... +Xm=opencossan.workers.MatlabWorker('description', 'Performance function', ... 'Script','for j=1:length(Tinput), Toutput(j).out1=sqrt(Tinput(j).RV1^2+Tinput(j).RV2^2); end', ... 'OutputNames',{'out1'},... 'InputNames',{'RV1' 'RV2'},... 'Format','structure',... 'IsFunction',false); % This flag specify if the .m file is a script or a function. % Construct the Evaluator -Xeval1 = opencossan.workers.Evaluator('Xmio',Xm,'Sdescription','fist Evaluator'); +Xeval1 = opencossan.workers.Evaluator("Solver",Xm,"SolverName","Xm","Description","fist Evaluator"); % In order to be able to test our Evaluator we need an Input object: % Define an Input @@ -66,7 +68,7 @@ % TestX evaluetor Xo1=Xeval1.apply(Xin); display(Xo1) -Vout=Xo1.getValues('Cnames',Xeval1.Coutputnames); +Vout=Xo1.getValues('Cnames',Xeval1.OutputNames); % Validate Solution @@ -84,7 +86,7 @@ % factors computed internally to the evaluator object. % Construct a Mio object -Xm2=opencossan.workers.Mio('description', 'Performance function', ... +Xm2=opencossan.workers.MatlabWorker('description', 'Performance function', ... 'Script','for j=1:length(Tinput), Toutput(j).out2=Tinput(j).out1+3; end', ... 'OutputNames',{'out2'},... 'InputNames',{'out1'},... @@ -92,24 +94,24 @@ 'IsFunction',false); % This flag specify if the .m file is a script or a function. % Construct a Mio object -Xm3=opencossan.workers.Mio('description', 'Performance function', ... +Xm3=opencossan.workers.MatlabWorker('description', 'Performance function', ... 'Script','for j=1:length(Tinput), Toutput(j).out3=Tinput(j).out2+Tinput(j).RV1; end', ... 'OutputNames',{'out3'},... 'InputNames',{'out2' 'RV1'},... 'Format','structure',... 'IsFunction',false); -XevALL=opencossan.workers.Evaluator('CXmembers',{Xm Xm2 Xm3}); -XevALL.Cinputnames +XevALL=opencossan.workers.Evaluator('Solver',[Xm Xm2 Xm3],'SolverName',["Xm" "Xm2" "Xm3"]); +XevALL.InputNames % The provided output are shows in the field Coutputnames -XevALL.Coutputnames +XevALL.OutputNames %% Deterministic Analysis % Xoutdet=XevALL.deterministicAnalysis(Xin); display(Xoutdet) -Vout=Xoutdet.getValues('Sname','out3'); +Vout=Xoutdet.getValues('Name','out3'); % Validate Solution Vreference= 3; @@ -129,7 +131,7 @@ % models are exetuted before processing the next samples). % The default option is HorizontalSplit -display(XevALL.LverticalSplit) +display(XevALL.VerticalSplit) XinTest(1) = sample(Xin,'Nsamples',1); XinTest(2) = sample(Xin,'Nsamples',10); diff --git a/docs/tutorials/CossanObjects/TutorialExtractor.m b/docs/tutorials/CossanObjects/TutorialExtractor.m index d04b235b..d3bf296b 100644 --- a/docs/tutorials/CossanObjects/TutorialExtractor.m +++ b/docs/tutorials/CossanObjects/TutorialExtractor.m @@ -3,11 +3,27 @@ % extract method. For this purpose, a dummy textfile called 'outputfile.txt' % is used. % -% See Also: http://cossan.cfd.liv.ac.uk/wiki/index.php/@Extractor +% See Also: Extractor, TutorialInjector, TutorialConnector % -% Copyright 1993-2015, COSSAN Working Group -% University of Liverpool, UK +%{ + This file is part of OpenCossan . + Copyright (C) 2006-2020 COSSAN WORKING GROUP + + OpenCossan is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License or, + (at your option) any later version. + + OpenCossan is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with OpenCossan. If not, see . +%} + clear; close all diff --git a/docs/tutorials/CossanObjects/TutorialFailureProbability.m b/docs/tutorials/CossanObjects/TutorialFailureProbability.m index 44a69eff..20128d38 100644 --- a/docs/tutorials/CossanObjects/TutorialFailureProbability.m +++ b/docs/tutorials/CossanObjects/TutorialFailureProbability.m @@ -4,9 +4,9 @@ % FailureProbability. % % -% See Also: http://cossan.cfd.liv.ac.uk/wiki/index.php/@FailureProbability +% See Also: FailureProbability % -% $Copyright~1993-2011,~COSSAN~Working~Group,~University~of~Innsbruck,~Austria$ +% $Copyright~1993-2020,~COSSAN~Working~Group$ % $Author: Edoardo-Patelli$ clear; diff --git a/docs/tutorials/CossanObjects/TutorialFunction.m b/docs/tutorials/CossanObjects/TutorialFunction.m index b4b98000..20ad59f4 100644 --- a/docs/tutorials/CossanObjects/TutorialFunction.m +++ b/docs/tutorials/CossanObjects/TutorialFunction.m @@ -4,11 +4,11 @@ % % % -% See Also: https://cossan.co.uk/wiki/index.php/Function +% See Also: Function % %{ This file is part of OpenCossan . -Copyright (C) 2006-2019 COSSAN WORKING GROUP +Copyright (C) 2006-2020 COSSAN WORKING GROUP OpenCossan is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -68,7 +68,7 @@ %retrieve the names of all objects that are associated with the Function %object and their type -[Cmembers Ctypes] = getMembers(Xfun1); +[Cmembers, Ctypes] = getMembers(Xfun1); disp(Cmembers) disp(Ctypes) diff --git a/docs/tutorials/CossanObjects/TutorialGaussianMixtureRandomVariableSet.m b/docs/tutorials/CossanObjects/TutorialGaussianMixtureRandomVariableSet.m index 8bc16b4f..bd77f38d 100644 --- a/docs/tutorials/CossanObjects/TutorialGaussianMixtureRandomVariableSet.m +++ b/docs/tutorials/CossanObjects/TutorialGaussianMixtureRandomVariableSet.m @@ -5,9 +5,9 @@ % This class is particolarly useful to crate a multivariate distrution starting % from realizations. % -% See Also: http://cossan.cfd.liv.ac.uk/wiki/index.php/@GaussianMixtureRandomVariableSet +% See Also: GaussianMixtureRandomVariableSet % -% $Copyright~1993-2011,~COSSAN~Working~Group,~University~of~Innsbruck,~Austria$ +% $Copyright~1993-2020,~COSSAN~Working~Group$ % $Author:~Edoardo~Patelli$ clear diff --git a/docs/tutorials/CossanObjects/TutorialGeneticAlgorithms.m b/docs/tutorials/CossanObjects/TutorialGeneticAlgorithms.m index e995189e..c674883f 100644 --- a/docs/tutorials/CossanObjects/TutorialGeneticAlgorithms.m +++ b/docs/tutorials/CossanObjects/TutorialGeneticAlgorithms.m @@ -2,9 +2,9 @@ % This tutorial shows how to optimize the Rastrigin's function using % GeneticAlgorithms % -% See Also: http://cossan.cfd.liv.ac.uk/wiki/index.php/@GeneticAlgorithms +% See Also: GeneticAlgorithms % -% $Copyright~1993-2014,~COSSAN~Working~Group,~University~of~Liverpool,~UK$ +% $Copyright~1993-2020,~COSSAN~Working~Group$ % % $Author: Edoardo~Patelli$ % $email address: openengine@cossan.co.uk$ diff --git a/docs/tutorials/CossanObjects/TutorialGlobalSensitivityRandomBalanceDesign.m b/docs/tutorials/CossanObjects/TutorialGlobalSensitivityRandomBalanceDesign.m index 0ca9bbb7..7c71dc6f 100644 --- a/docs/tutorials/CossanObjects/TutorialGlobalSensitivityRandomBalanceDesign.m +++ b/docs/tutorials/CossanObjects/TutorialGlobalSensitivityRandomBalanceDesign.m @@ -5,15 +5,15 @@ % % In this tutorial a very simplified model is considered. % -% See Also: http://cossan.cfd.liv.ac.uk/wiki/index.php/Infection_Dynamic_Model +% See Also: Infection_Dynamic_Model Sensitivity % % -% $Copyright~1993-2011,~COSSAN~Working~Group,~University~of~Innsbruck,~Austria$ +% $Copyright~1993-2020,~COSSAN~Working~Group$ % $Author: Edoardo-Patelli$ clear close all clc; - +import opencossan.sensitivity.* %% Problem setup % In this examples we consider only 3 uniform random variables Xrv1 = opencossan.common.inputs.random.UniformRandomVariable('bounds',[-1, 1]); @@ -23,14 +23,13 @@ Xin = opencossan.common.inputs.Input('members',Xrvset,'membersnames','Xrvset'); % The model is defined using a Mio object -Xm = opencossan.workers.Mio('Script','for j=1:length(Tinput), Toutput(j).out1=Tinput(j).Xrv1^2+2*Tinput(j).Xrv2-Tinput(j).Xrv3; end', ... +Xm = opencossan.workers.MatlabWorker('Script','for j=1:length(Tinput), Toutput(j).out1=Tinput(j).Xrv1^2+2*Tinput(j).Xrv2-Tinput(j).Xrv3; end', ... 'OutputNames',{'out1'},... 'InputNames',{'Xrv1' 'Xrv2' 'Xrv3'},... -... 'Liostructure',true,... 'IsFunction',false); -Xev = opencossan.workers.Evaluator('Xmio',Xm); -Xmdl = opencossan.common.Model('Xinput',Xin,'Xevaluator',Xev); +Xev = opencossan.workers.Evaluator('Solver',Xm); +Xmdl = opencossan.common.Model('Input',Xin,'Evaluator',Xev); %% Global Sensitivity Analysis diff --git a/docs/tutorials/CossanObjects/TutorialGlobalSensitivitySobol.m b/docs/tutorials/CossanObjects/TutorialGlobalSensitivitySobol.m index 818486cd..7010461d 100644 --- a/docs/tutorials/CossanObjects/TutorialGlobalSensitivitySobol.m +++ b/docs/tutorials/CossanObjects/TutorialGlobalSensitivitySobol.m @@ -5,14 +5,13 @@ % % In this tutorial a very simplified model is considered. % -% See Also: http://cossan.cfd.liv.ac.uk/wiki/index.php/Infection_Dynamic_Model +% See Also: Infection_Dynamic_Model GlobalSensitivitySobol % % -% $Copyright~1993-2011,~COSSAN~Working~Group,~University~of~Innsbruck,~Austria$ +% $Copyright~1993-2020,~COSSAN~Working~Group$ % $Author: Edoardo-Patelli$ -clear; -close all clc; +import opencossan.simulations.* %% Problem setup % In this examples we consider only 3 uniform random variables @@ -23,14 +22,13 @@ Xin = opencossan.common.inputs.Input('members',Xrvset,'membersnames','Xrvset'); % The model is defined using a Mio object -Xm = opencossan.workers.Mio('Script','for j=1:length(Tinput), Toutput(j).out1=Tinput(j).Xrv1^2+2*Tinput(j).Xrv2-Tinput(j).Xrv3; end', ... +Xm = opencossan.workers.MatlabWorker('Script','for j=1:length(Tinput), Toutput(j).out1=Tinput(j).Xrv1^2+2*Tinput(j).Xrv2-Tinput(j).Xrv3; end', ... 'OutputNames',{'out1'},... 'InputNames',{'Xrv1' 'Xrv2' 'Xrv3'},... -... 'Liostructure',true,... 'IsFunction',false); -Xev = opencossan.workers.Evaluator('Xmio',Xm); -Xmdl = opencossan.common.Model('Xinput',Xin,'Xevaluator',Xev); +Xev = opencossan.workers.Evaluator('Solver',Xm); +Xmdl = opencossan.common.Model('Input',Xin,'Evaluator',Xev); %% Global Sensitivity Analysis diff --git a/docs/tutorials/CossanObjects/TutorialGlobalSensitivityUpperBound.m b/docs/tutorials/CossanObjects/TutorialGlobalSensitivityUpperBound.m index 93066db6..786d4c7a 100644 --- a/docs/tutorials/CossanObjects/TutorialGlobalSensitivityUpperBound.m +++ b/docs/tutorials/CossanObjects/TutorialGlobalSensitivityUpperBound.m @@ -5,10 +5,10 @@ % % In this tutorial a very simplified model is considered. % -% See Also: http://cossan.cfd.liv.ac.uk/wiki/index.php/Infection_Dynamic_Model +% See Also: Infection_Dynamic_Model % % -% $Copyright~1993-2011,~COSSAN~Working~Group,~University~of~Innsbruck,~Austria$ +% $Copyright~1993-2020,~COSSAN~Working~Group$ % $Author: Edoardo-Patelli$ clear close all; @@ -23,10 +23,9 @@ Xin = opencossan.common.inputs.Input('members',Xrvset,'membersnames','Xrvset'); % The model is defined using a Mio object -Xm = opencossan.workers.Mio('Script','for j=1:length(Tinput), Toutput(j).out1=Tinput(j).Xrv1^2+2*Tinput(j).Xrv2-Tinput(j).Xrv3; end', ... +Xm = opencossan.workers.MatlabWorker('Script','for j=1:length(Tinput), Toutput(j).out1=Tinput(j).Xrv1^2+2*Tinput(j).Xrv2-Tinput(j).Xrv3; end', ... 'OutputNames',{'out1'},... 'InputNames',{'Xrv1' 'Xrv2' 'Xrv3'},... -... 'Liostructure',true,... 'IsFunction',false); Xev = opencossan.workers.Evaluator('Xmio',Xm); @@ -39,7 +38,7 @@ % The method upperBounds allows to estimate the upper bounds of teh total % sensitivity indices -Xgs=GlobalSensitivityUpperBound('Xmodel',Xmdl,'Nbootstrap',3,'Nsamples',4); +Xgs=GlobalSensitivityUpperBound('Model',Xmdl,'Nbootstrap',3,'Nsamples',4); display(Xgs) Xsm = Xgs.computeIndices; display(Xsm) diff --git a/docs/tutorials/CossanObjects/TutorialGradient.m b/docs/tutorials/CossanObjects/TutorialGradient.m index 1141c1ac..f5441513 100644 --- a/docs/tutorials/CossanObjects/TutorialGradient.m +++ b/docs/tutorials/CossanObjects/TutorialGradient.m @@ -8,9 +8,9 @@ % $y=2+x_1^2+5*x_2^2+0.01*x_4^2$ % where x1, x2, x3, x4 are modelled as random variables (RV1,...,RV4) % -% See Also: http://cossan.cfd.liv.ac.uk/wiki/index.php/@Gradient +% See Also: Gradient % -% $Copyright~1993-2014,~COSSAN~Working~Group,~University~of~Liverpool,~UK$ +% $Copyright~1993-2020,~COSSAN~Working~Group$ % $Author: Edoardo-Patelli$ clear; close all @@ -30,18 +30,17 @@ Xin = add(Xin,'name','Xrvs1','member',Xrvs1); % Construct a Mio object that defines the model -Xm=opencossan.workers.Mio('description', 'Performance function', ... +Xm=opencossan.workers.MatlabWorker('description', 'Performance function', ... 'Script','for j=1:length(Tinput), Toutput(j).out1=2+Tinput(j).RV1^2+5*Tinput(j).RV2^2+0.01*Tinput(j).RV4^2; end', ... -... 'Liostructure',true,... 'IsFunction',false, ... 'InputNames',{'RV1' 'RV2' 'RV3' 'RV4'},... 'OutputNames',{'out1'}); % This flag specify if the .m file is a script or a function. % Construct the Evaluator -Xeval1 = opencossan.workers.Evaluator('Xmio',Xm,'Sdescription','fist Evaluator'); +Xeval1 = opencossan.workers.Evaluator('Solver',Xm,'Description','fist Evaluator'); % Construct the Model -Xmdl=opencossan.common.Model('CXmembers',{Xin,Xeval1}); +Xmdl=opencossan.common.Model('Input',Xin,'Evaluator',Xeval1); % Show Model details display(Xmdl) diff --git a/docs/tutorials/CossanObjects/TutorialHaltonSampling.m b/docs/tutorials/CossanObjects/TutorialHaltonSampling.m index cb5e6ab5..5b5dd66d 100644 --- a/docs/tutorials/CossanObjects/TutorialHaltonSampling.m +++ b/docs/tutorials/CossanObjects/TutorialHaltonSampling.m @@ -16,8 +16,8 @@ %% Define the Input % Define RandomVariable -RV1=opencossan.common.inputs.random.NormalRandomVariable('mean',2,'std',1); %#ok -RV2=opencossan.common.inputs.random.NormalRandomVariable('mean',0,'std',2); %#ok +RV1=opencossan.common.inputs.random.NormalRandomVariable('mean',2,'std',1); +RV2=opencossan.common.inputs.random.NormalRandomVariable('mean',0,'std',2); % Define the RandomVariableSet Xrvs1=opencossan.common.inputs.random.RandomVariableSet('names',{'RV1', 'RV2'},'members',[RV1; RV2]); % Construct Input Object @@ -29,17 +29,16 @@ %% Define the Evaluator (i.e. how our model is evaluate) % Construct a Mio object -Xm=opencossan.workers.Mio( 'description', 'This is our Model', ... +Xm=opencossan.workers.MatlabWorker( 'description', 'This is our Model', ... 'Script','for j=1:length(Tinput), Toutput(j).out=-Tinput(j).RV1+Tinput(j).RV2; end', ... -... 'Liostructure',true,... 'OutputNames',{'out'},... 'InputNames',{'RV1','RV2'},... 'IsFunction',false); % This flag specify if the .m file is a script or a function. % Construct the Evaluator -Xeval = opencossan.workers.Evaluator('Xmio',Xm,'Sdescription','Evaluator for the IS tutorial'); +Xeval = opencossan.workers.Evaluator('Solver',Xm,'Description','Evaluator for the IS tutorial'); %% Define the Physical Model based on the Input and the Evaluator -Xmdl=opencossan.common.Model('Xevaluator',Xeval,'Xinput',Xin); +Xmdl=opencossan.common.Model('Evaluator',Xeval,'Input',Xin); %% Test the Model % Generate 10 random realization of the input diff --git a/docs/tutorials/CossanObjects/TutorialImportanceSampling.m b/docs/tutorials/CossanObjects/TutorialImportanceSampling.m index 8fe5264e..589caa8b 100644 --- a/docs/tutorials/CossanObjects/TutorialImportanceSampling.m +++ b/docs/tutorials/CossanObjects/TutorialImportanceSampling.m @@ -46,17 +46,16 @@ %% Define the Evaluator (i.e. how our model is evaluate) % Construct a Mio object -Xm=opencossan.workers.Mio( 'description', 'This is our Model', ... +Xm=opencossan.workers.MatlabWorker( 'description', 'This is our Model', ... 'Script','for j=1:length(Tinput), Toutput(j).out=-Tinput(j).RV1+Tinput(j).RV2; end', ... -... 'Liostructure',true,... 'OutputNames',{'out'},... 'InputNames',{'RV1','RV2'},... 'IsFunction',false); % This flag specify if the .m file is a script or a function. % Construct the Evaluator -Xeval = opencossan.workers.Evaluator('Xmio',Xm,'Sdescription','Evaluator for the IS tutorial'); +Xeval = opencossan.workers.Evaluator('Solver',Xm,'Description','Evaluator for the IS tutorial'); %% Define the Physical Model based on the Input and the Evaluator -Xmdl=opencossan.common.Model('Xevaluator',Xeval,'Xinput',Xin); +Xmdl=opencossan.common.Model('Evaluator',Xeval,'Input',Xin); %% Define ImportanceSampling object % The InportanceSampling object required the definition of a "Proposal @@ -110,7 +109,7 @@ Xpf=PerformanceFunction('OutputName','Vg','Capacity','Xthreshold','Demand','out'); % Construct a ProbabilisticModel Object -Xpm=ProbabilisticModel('Xmodel',Xmdl,'XperformanceFunction',Xpf); +Xpm=ProbabilisticModel('Model',Xmdl,'PerformanceFunction',Xpf); % now we can apply the ImportanceSampling object and estimate also % the performance function % ProbabilisticModel @@ -123,8 +122,8 @@ %% Check the weights % use as Importance Sampling density the same distribution of RV1 and RV2. % By doing so, all the weigth must be equal 1! -RV3=RV1; %#ok -RV4=RV2; %#ok +RV3=RV1; +RV4=RV2; XrvsIS=RandomVariableSet('Cmembers',{'RV3';'RV4'}); Cmapping={'RV3' 'RV1'; 'RV4' 'RV2'}; % Construct the Simulation object @@ -171,4 +170,4 @@ display(XpfIS) %% Validate Solutions -assert(abs(Xpf.pfhat-0.2)<1e-4,'openCOSSAN:Tutorials','Wrong results') +assert(abs(Xpf.pfhat-0.2)<1e-4,'OpenCossan:Tutorials','Wrong results') diff --git a/docs/tutorials/CossanObjects/TutorialJobManager.m b/docs/tutorials/CossanObjects/TutorialJobManager.m deleted file mode 100644 index 9b3738cc..00000000 --- a/docs/tutorials/CossanObjects/TutorialJobManager.m +++ /dev/null @@ -1,53 +0,0 @@ -% Tutorial for the JobManager Object -% This tutorial shows how to create and use a JobManager object - -% This tutorial shows only the most basic commands and features of -% JobMangarer. -clear -close all -clc; - -Xjm = opencossan.highperformancecomputing.JobManagerInterface('Stype','GridEngine'); - - Xg = opencossan.highperformancecomputing.JobManager('Sdescription','test #1',... - 'Squeue','pizzas64.q',... - 'Xjobmanagerinterface',Xjm); - - Xg = JobManager('Sdescription','test #1',... - 'Squeue','pizzas64.q',... - 'Xjobmanagerinterface',Xjm,... - 'SpreExeCmd','echo preexecmd ', ... - 'SpostExeCmd','echo postexecmd '); - [~,SuserName] = system('whoami'); SuserName=SuserName(1:end-1); - CSjobID(1)=Xg.submitJob; - - - %% Run a model - - - RV1=RandomVariable('Sdistribution','normal', 'mean',0,'std',1); %#ok -RV2=RandomVariable('Sdistribution','normal', 'mean',0,'std',1); %#ok -% Define the RVset -Xrvs1=RandomVariableSet('Cmembers',{'RV1', 'RV2'}); -% Define Xinput -Xin = Input('Sdescription','Input satellite_inp'); -Xin = add(Xin,Xrvs1); - - -Xm=Mio( 'Sdescription', 'This is our Model', ... - 'Sscript','for j=1:length(Tinput), Toutput(j).out=-Tinput(j).RV1+Tinput(j).RV2; end', ... - 'Liostructure',true,... - 'Coutputnames',{'out'},... - 'Cinputnames',{'RV1','RV2'},... - 'Lfunction',false); % This flag specify if the .m file is a script or a function. - -%% Construct the Evaluator -% First mode (The object are passed by reference) - -Xeval1 = Evaluator('CXmembers',{Xm}, ... - 'Xjobmanagerinterface',Xjm,... - 'CSqueues',{'MatlabPool',''},... - 'Vconcurrent',[6]); - -Xmdl=Model('Xinput',Xin,'Xevaluator',Xeval1,'Sdescription','The Model'); - % Construct a Mio object diff --git a/docs/tutorials/CossanObjects/TutorialJobManagerInterface.m b/docs/tutorials/CossanObjects/TutorialJobManagerInterface.m deleted file mode 100644 index b476ead5..00000000 --- a/docs/tutorials/CossanObjects/TutorialJobManagerInterface.m +++ /dev/null @@ -1,18 +0,0 @@ -%% Tutorial for the JobManagerInterface object -% JobManagerInterface is used as an input of an Evaluator object. When a -% JobManagerInterface is introduced, the simulations are executed remotely -% on a cluster. -% JobManagerInterface has to be used then in conjunction of an Evaluator -% and one or more solvers. An example of his usage is available in the -% tutorial of Connector "TutorialConnectorABAQUS.m". -% Please refer to this tutorial for additional instructions. -% -% See Also: http://cossan.cfd.liv.ac.uk/wiki/index.php/@JobManagerInterface -% -% $Copyright~1993-2011,~COSSAN~Working~Group,~University~of~Innsbruck,~Austria$ -% $Author: $ - -clear -close all; -clc; - diff --git a/docs/tutorials/CossanObjects/TutorialJobManagerSlurm.m b/docs/tutorials/CossanObjects/TutorialJobManagerSlurm.m index 6db2b65b..0e6ef2d1 100644 --- a/docs/tutorials/CossanObjects/TutorialJobManagerSlurm.m +++ b/docs/tutorials/CossanObjects/TutorialJobManagerSlurm.m @@ -20,7 +20,7 @@ queues=Xjm.getQueues; % Submit job system('sbactch ....') -[]=Xjm.submit() +[ID]=Xjm.submit() diff --git a/docs/tutorials/CossanObjects/TutorialKarhunenLoeve.m b/docs/tutorials/CossanObjects/TutorialKarhunenLoeve.m index f242701d..507e42b6 100644 --- a/docs/tutorials/CossanObjects/TutorialKarhunenLoeve.m +++ b/docs/tutorials/CossanObjects/TutorialKarhunenLoeve.m @@ -7,7 +7,7 @@ % % See Also: STOCHASTICPROCESS TutorialInput % -% $Copyright~1993-2018,~COSSAN~Working~Group$ +% $Copyright~1993-2020,~COSSAN~Working~Group$ % $Author:~Barbara~Goller$ % $Author:~Edoardo~Patelli$ @@ -34,7 +34,7 @@ 'IsFunction',false,'Format','structure',... 'Script', strcat('sigma = 1; b = 0.5; for i=1:length(Tinput), ',... 'Toutput(i).fcov = sigma^2*exp(-1/b*abs(Tinput(i).x_2-Tinput(i).x_1));',... - 'end'), 'OutputNames',{'fcov'}); % Define the outputs + 'end'), 'OutputNames',{'fcov'},'InputNames',{'x_1' 'x_2'}); % Define the outputs % Define the time steps of the process Vtime = linspace(0,5,101); % time steps @@ -136,7 +136,7 @@ 'Script',strcat('sigma = 1; b = 0.5; for i=1:length(Tinput),',... 'Toutput(i).fcov = sigma^2*exp(-1/b*sqrt((Tinput(i).x_2-Tinput(i).x_1)*',... '(Tinput(i).x_2-Tinput(i).x_1)'')); end'), ... - 'OutputNames',{'fcov'}); % Define the outputs + 'OutputNames',{'fcov'},'InputNames',{'x_1','x_2'}); % Define the outputs % Define StochasticProcess object by passing the covariance function and assembling the matrix [Vx ,Vy] = meshgrid(0:0.5:5,0:0.5:4); @@ -160,7 +160,7 @@ TableInput=array2table([Mindex1(:) Mindex1(:)],'VariableNames',Xcovfun.InputNames); TableOutput = evaluate(Xcovfun,TableInput); -Mcovariance = reshape(table2array(TableOutput),length(Vtime),length(Vtime)); +Mcovariance = reshape(table2array(TableOutput),length(Mindex1),length(Mindex2)); f2=figure; mesh(Mindex1,Mindex2,Mcovariance) @@ -195,7 +195,7 @@ 'Script',strcat('sigma = 1; b = 0.5; for i=1:length(Tinput), ', ... 'Toutput(i).fcov = sigma^2*exp(-1/b*sqrt((Tinput(i).x_2-Tinput(i).x_1)',... '*(Tinput(i).x_2-Tinput(i).x_1)'')); end'),... - 'OutputNames',{'fcov'}); % Define the outputs + 'OutputNames',{'fcov'},'InputNames',{'x_1','x_2'}); % Define the outputs % Define 3d-coordinates for random field [Vx, Vy, Vz] = meshgrid(0:0.5:5,0:0.5:5,0:0.5:5); diff --git a/docs/tutorials/CossanObjects/TutorialKriging.m b/docs/tutorials/CossanObjects/TutorialKriging.m index 3e7fadf7..7e612b4b 100644 --- a/docs/tutorials/CossanObjects/TutorialKriging.m +++ b/docs/tutorials/CossanObjects/TutorialKriging.m @@ -4,10 +4,9 @@ % % Prepared by IM % -% Copyright 1993-2012, +% Copyright 1993-2020, % -% See Also: -% http://cossan.cfd.liv.ac.uk/wiki/index.php/@Kriging +% See Also: Kriging % Reset the random number generator in order to always obtain the same results. % DO NOT CHANGE THE VALUES OF THE SEED diff --git a/docs/tutorials/CossanObjects/TutorialLatinHypercubeSampling.m b/docs/tutorials/CossanObjects/TutorialLatinHypercubeSampling.m index 6c8f9f31..bb96bdbb 100644 --- a/docs/tutorials/CossanObjects/TutorialLatinHypercubeSampling.m +++ b/docs/tutorials/CossanObjects/TutorialLatinHypercubeSampling.m @@ -3,9 +3,9 @@ % This tutorial is focus on the use and definition of the % LatinHypercubeSampling class % -% See Also: http://cossan.cfd.liv.ac.uk/wiki/index.php/@LatinHypercubeSampling +% See Also: LatinHypercubeSampling % -% $Copyright~1993-2011,~COSSAN~Working~Group,~University~of~Innsbruck,~Austria$ +% $Copyright~1993-2020,~COSSAN~Working~Group$ % $Author: Edoardo~Patelli$ clear @@ -15,8 +15,8 @@ % Here we define our problem % Define RandomVariable -RV1=opencossan.common.inputs.random.NormalRandomVariable('mean',2,'std',1); %#ok -RV2=opencossan.common.inputs.random.NormalRandomVariable('mean',0,'std',2); %#ok +RV1=opencossan.common.inputs.random.NormalRandomVariable('mean',2,'std',1); +RV2=opencossan.common.inputs.random.NormalRandomVariable('mean',0,'std',2); Xdemand=opencossan.common.inputs.Parameter('description','Define threshold','value',0); % Define the RandomVariableSet @@ -29,17 +29,16 @@ %% Define the Evaluator (i.e. how our model is evaluate) % Construct a Mio object -Xm=opencossan.workers.Mio( 'description', 'This is our Model', ... +Xm=opencossan.workers.MatlabWorker( 'description', 'This is our Model', ... 'Script','for j=1:length(Tinput), Toutput(j).out=-Tinput(j).RV1+Tinput(j).RV2; end', ... -... 'Liostructure',true,... 'OutputNames',{'out'},... 'InputNames',{'RV1','RV2'},... 'IsFunction',false); % This flag specify if the .m file is a script or a function. % Construct the Evaluator -Xeval = opencossan.workers.Evaluator('Xmio',Xm,'Sdescription','Evaluator for the IS tutorial'); +Xeval = opencossan.workers.Evaluator('Solver',Xm,'Description','Evaluator for the IS tutorial'); %% Define the Physical Model based on the Input and the Evaluator -Xmdl=opencossan.common.Model('Xevaluator',Xeval,'Xinput',Xin); +Xmdl=opencossan.common.Model('Evaluator',Xeval,'Input',Xin); %% Test the Model % Generate 10 random realization of the input diff --git a/docs/tutorials/CossanObjects/TutorialLineSampling.m b/docs/tutorials/CossanObjects/TutorialLineSampling.m index 2f6a22c6..c5d20a6e 100644 --- a/docs/tutorials/CossanObjects/TutorialLineSampling.m +++ b/docs/tutorials/CossanObjects/TutorialLineSampling.m @@ -4,9 +4,9 @@ % The line sampling is not applicable to simulate the Model since it % required a performace function. % -% See Also: http://cossan.cfd.liv.ac.uk/wiki/index.php/@LineSampling +% See Also: LineSampling % -% $Copyright~1993-2011,~COSSAN~Working~Group,~University~of~Innsbruck,~Austria$ +% $Copyright~1993-2011,~COSSAN~Working~Group$ % $Author: Edoardo-Patelli and Marco-de-Angelis$ clear close all @@ -22,20 +22,20 @@ % Add parameter for the performance function Xthreshold=opencossan.common.inputs.Parameter('value',2); % Construct Input Object -Xin = opencossan.common.inputs.Input('description','Input Object of our model', ... - 'members',{Xrvs1 Xthreshold},'membersnames',{'Xrvs1' 'Xthreshold'}); +Xin = opencossan.common.inputs.Input('Description','Input Object of our model', ... + 'Members',{Xrvs1 Xthreshold},'MembersNames',{'Xrvs1' 'Xthreshold'}); %% Define the Evaluator (i.e. how our model is evaluate) % Construct a Mio object -Xm=opencossan.workers.Mio( 'description', 'This define our Model', ... +Xm=opencossan.workers.MatlabWorker( 'description', 'This define our Model', ... 'Script','for j=1:length(Tinput), Toutput(j).out=-Tinput(j).RV1+Tinput(j).RV2+Tinput(j).RV2.^2; end', ... 'Format','structure',... 'OutputNames',{'out'},... 'InputNames',{'RV1' 'RV2'},... 'IsFunction',false); % This flag specify if the .m file is a script or a function. % Construct the Evaluator -Xeval = opencossan.workers.Evaluator('Xmio',Xm,'Sdescription','Evaluator for the IS tutorial'); +Xeval = opencossan.workers.Evaluator('Solver',Xm,'Description','Evaluator for the IS tutorial'); %% Define the Physical Model based on the Input and the Evaluator -Xmdl=opencossan.common.Model('Xevaluator',Xeval,'Xinput',Xin); +Xmdl=opencossan.common.Model('Evaluator',Xeval,'Input',Xin); %% Test the Model % Generate 10 random realization of the input Xin = sample(Xin,'Nsamples',10); @@ -61,7 +61,7 @@ % Construct the performance function Xpf=PerformanceFunction('OutputName','Vg','Capacity','Xthreshold','Demand','out'); % Construct a ProbabilisticModel Object -Xpm=ProbabilisticModel('Xmodel',Xmdl,'XperformanceFunction',Xpf); +Xpm=ProbabilisticModel('Model',Xmdl,'PerformanceFunction',Xpf); %% Compute Reference Solution % This can take a while Xmc=MonteCarlo('Nsamples',5e4); diff --git a/docs/tutorials/CossanObjects/TutorialLocalSensitivityFiniteDifference.m b/docs/tutorials/CossanObjects/TutorialLocalSensitivityFiniteDifference.m index 95ceb774..c9dcc806 100644 --- a/docs/tutorials/CossanObjects/TutorialLocalSensitivityFiniteDifference.m +++ b/docs/tutorials/CossanObjects/TutorialLocalSensitivityFiniteDifference.m @@ -29,14 +29,13 @@ Xin = opencossan.common.inputs.Input('members',Xrvset, 'membersnames','Xrvset'); % The model is defined using a Mio object -Xm = opencossan.workers.Mio('Script','for j=1:length(Tinput), Toutput(j).out1=Tinput(j).Xrv1^2+2*Tinput(j).Xrv2-Tinput(j).Xrv3; end', ... +Xm = opencossan.workers.MatlabWorker('Script','for j=1:length(Tinput), Toutput(j).out1=Tinput(j).Xrv1^2+2*Tinput(j).Xrv2-Tinput(j).Xrv3; end', ... 'OutputNames',{'out1'},... 'InputNames',{'Xrv1' 'Xrv2' 'Xrv3'},... -... 'Liostructure',true,... 'IsFunction',false); -Xev = opencossan.workers.Evaluator('Xmio',Xm); -Xmdl = opencossan.common.Model('Xinput',Xin,'Xevaluator',Xev); +Xev = opencossan.workers.Evaluator('Solver',Xm); +Xmdl = opencossan.common.Model('Input',Xin,'Evaluator',Xev); % Here we go!!! %% Local Sensitivity Analysis @@ -77,17 +76,15 @@ 'Toutput(i).Out1 = 2*Tinput(i).DV1-Tinput(i).DV2^2;'... 'end']; -XmDV = Mio('Sdescription', 'Performance function', ... - 'Sscript',Sscript, ... % Define the script - 'Coutputnames',{'Out1'},... % This field is mandatory - 'Cinputnames',{'DV1';'DV2'},... % This field is mandatory - 'Liostructure',true,... % This flag specify the type of I/O - 'Liomatrix',false, ... % This flag specify the type of I/O - 'Lfunction',false); % This flag specify if the .m file is a script or a function. - -Xev = Evaluator('Xmio',XmDV); +XmDV = opencossan.workers.MatlabWorker('Description', 'Performance function', ... + 'Script',Sscript, ... % Define the script + 'OutputNames',{'Out1'},... % This field is mandatory + 'InputNames',{'DV1';'DV2'},... % This field is mandatory + 'Format','structure'); + +Xev = opencossan.workers.Evaluator('Solver',XmDV); % Define probmodel -Xmodel = Model('XInput',Xin,'XEvaluator',Xev); +Xmodel = opencossan.common.Model('Xnput',Xin,'Evaluator',Xev); %% Select a user defined point XlsFD2=LocalSensitivityFiniteDifference('Xtarget',Xmodel,... @@ -122,14 +119,13 @@ % The figures show the correlation of the sample in physical space. % The model is defined using a Mio object -Xm = Mio('Sscript','for j=1:length(Tinput), Toutput(j).out1=-2*Tinput(j).Xrv1+(Tinput(j).Xrv2-2); end', ... - 'Coutputnames',{'out1'},... - 'Cinputnames',{'Xrv1' 'Xrv2'},... - 'Liostructure',true,... - 'Lfunction',false); +Xm = opencossan.workers.MatlabWorker('Script','for j=1:length(Tinput), Toutput(j).out1=-2*Tinput(j).Xrv1+(Tinput(j).Xrv2-2); end', ... + 'OutputNames',{'out1'},... + 'InputNames',{'Xrv1' 'Xrv2'},... + 'Format','structure'); -Xev = Evaluator('Xmio',Xm); -Xmdl = Model('Xinput',Xin,'Xevaluator',Xev); +Xev = Evaluator('Solver',Xm); +Xmdl = Model('Input',Xin,'Evaluator',Xev); % Define the Local Sensitivity method XlsFD=LocalSensitivityFiniteDifference('Xmodel',Xmdl); diff --git a/docs/tutorials/CossanObjects/TutorialLocalSensitivityMonteCarlo.m b/docs/tutorials/CossanObjects/TutorialLocalSensitivityMonteCarlo.m index 26d5e9a8..52a8e7eb 100644 --- a/docs/tutorials/CossanObjects/TutorialLocalSensitivityMonteCarlo.m +++ b/docs/tutorials/CossanObjects/TutorialLocalSensitivityMonteCarlo.m @@ -5,12 +5,12 @@ % % In this tutorial a very simplified model is considered. % -% See Also: http://cossan.cfd.liv.ac.uk/wiki/index.php/Infection_Dynamic_Model +% See Also: Infection_Dynamic_Model % % %{ This file is part of OpenCossan . -Copyright (C) 2006-2019 COSSAN WORKING GROUP +Copyright (C) 2006-2020 COSSAN WORKING GROUP OpenCossan is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -41,14 +41,13 @@ Xin = add(Xin,'member',Xrvset,'name','Xrvset'); % The model is defined using a Mio object -Xm = opencossan.workers.Mio('Script','for j=1:length(Tinput), Toutput(j).out1=Tinput(j).Xrv1^2+2*Tinput(j).Xrv2-Tinput(j).Xrv3; end', ... +Xm = opencossan.workers.MatlabWorker('Script','for j=1:length(Tinput), Toutput(j).out1=Tinput(j).Xrv1^2+2*Tinput(j).Xrv2-Tinput(j).Xrv3; end', ... 'OutputNames',{'out1'},... 'InputNames',{'Xrv1' 'Xrv2' 'Xrv3'},... - ... 'Liostructure',true,... 'IsFunction',false); -Xev = opencossan.workers.Evaluator('Xmio',Xm); -Xmdl = opencossan.common.Model('Input',Xin,'evaluator',Xev); +Xev = opencossan.workers.Evaluator('Solver',Xm); +Xmdl = opencossan.common.Model('Input',Xin,'Evaluator',Xev); % Here we go!!! %% Local Sensitivity Analysis diff --git a/docs/tutorials/CossanObjects/TutorialMarkovChain.m b/docs/tutorials/CossanObjects/TutorialMarkovChain.m index 1aa1ea7f..04248fcb 100644 --- a/docs/tutorials/CossanObjects/TutorialMarkovChain.m +++ b/docs/tutorials/CossanObjects/TutorialMarkovChain.m @@ -8,9 +8,9 @@ % another state (or remain in the same state) according to a % probability distribution. % -% See Also: http://cossan.cfd.liv.ac.uk/wiki/index.php/@MarkovChain +% See Also: MarkovChain % -% $Copyright~1993-2011,~COSSAN~Working~Group,~University~of~Innsbruck,~Austria$ +% $Copyright~1993-2020,~COSSAN~Working~Group,~University~of~Innsbruck,~Austria$ % $Author:~Edoardo~Patelli$ clear diff --git a/docs/tutorials/CossanObjects/TutorialMatlabWorker.m b/docs/tutorials/CossanObjects/TutorialMatlabWorker.m index ec61d16b..3525dac0 100644 --- a/docs/tutorials/CossanObjects/TutorialMatlabWorker.m +++ b/docs/tutorials/CossanObjects/TutorialMatlabWorker.m @@ -1,9 +1,7 @@ % Tutorial for MatlabWorker. % This tutorial shows how to construct and use a MatlabWorker object. % -clear -close all -clc; +clear, close all, clc; import opencossan.workers.* import opencossan.common.inputs.* @@ -73,7 +71,7 @@ %% First test - Use MonteCarlo simulation % Define Evaluator -Xev = Evaluator('Solvers',Xm); +Xev = Evaluator('Solver',Xm); % Define probmodel Xmodel = opencossan.common.Model('Input',Xinp,'Evaluator',Xev); % Apply @@ -109,7 +107,7 @@ %% Second - Use MonteCarlo simulation % Define Evaluator -Xev = Evaluator('Solvers',XmB); +Xev = Evaluator('Solver',XmB); % Define probmodel Xmodel = opencossan.common.Model('Input',Xinp,'Evaluator',Xev); % Apply @@ -172,7 +170,7 @@ %% Third test - Use MonteCarlo simulation %8.1. Define Evaluator -Xev = Evaluator('Solvers',XmD); +Xev = Evaluator('Solver',XmD); %8.2. Define probmodel Xmodel = opencossan.common.Model('Input',Xinp,'Evaluator',Xev); %8.3. Apply @@ -220,7 +218,7 @@ %% Third test - Use MonteCarlo simulation %8.1. Define Evaluator -Xev = Evaluator('Solvers',XmDfun); +Xev = Evaluator('Solver',XmDfun); %8.2. Define probmodel Xmodel = opencossan.common.Model('Input',Xinp,'Evaluator',Xev); %8.3. Apply diff --git a/docs/tutorials/CossanObjects/TutorialMetaModel.m b/docs/tutorials/CossanObjects/TutorialMetaModel.m index cdf747f3..9a8bf39e 100644 --- a/docs/tutorials/CossanObjects/TutorialMetaModel.m +++ b/docs/tutorials/CossanObjects/TutorialMetaModel.m @@ -2,11 +2,10 @@ % The object MetaModel is use to replace a computational demanding model or % to provide relationships between input and output. % -% See Also: http://cossan.cfd.liv.ac.uk/wiki/index.php/@MetaModel +% See Also: MetaModel % % % Author: Edoardo Patelli -% Institute for Risk and Uncertainty, University of Liverpool, UK % email address: openengine@cossan.co.uk % Website: http://www.cossan.co.uk diff --git a/docs/tutorials/CossanObjects/TutorialMiniMax.m b/docs/tutorials/CossanObjects/TutorialMiniMax.m index e881994b..ce3abe1a 100644 --- a/docs/tutorials/CossanObjects/TutorialMiniMax.m +++ b/docs/tutorials/CossanObjects/TutorialMiniMax.m @@ -6,7 +6,7 @@ % the Aim of this tutorial is to find x that minimize the maximum value of 5 % objective functions. % -% See Also: http://cossan.cfd.liv.ac.uk/wiki/index.php/@MiniMax +% See Also: MiniMax x1 = opencossan.optimization.ContinuousDesignVariable('Description','design variable 1','value',0.1); x2 = opencossan.optimization.ContinuousDesignVariable('Description','design variable 2','value',0.1); diff --git a/docs/tutorials/CossanObjects/TutorialModel.m b/docs/tutorials/CossanObjects/TutorialModel.m index 1562d202..4f14c136 100644 --- a/docs/tutorials/CossanObjects/TutorialModel.m +++ b/docs/tutorials/CossanObjects/TutorialModel.m @@ -4,7 +4,7 @@ % The Model object defines the user defined model compoused by an Input % object and an Evaluator object. % -% See Also: https://cossan.co.uk/wiki/index.php/Model +% See Also: Model % %{ This file is part of OpenCossan . @@ -37,7 +37,7 @@ % user define model based on a Matlab function % % Construct a Mio object -Xm=opencossan.workers.Mio( 'description', 'This is our Model', ... +Xm=opencossan.workers.MatlabWorker( 'description', 'This is our Model', ... 'Script','for j=1:length(Tinput), Toutput(j).out=-Tinput(j).RV1+Tinput(j).RV2; end', ... 'format','structure',... 'OutputNames',{'out'},... @@ -46,15 +46,15 @@ %% Construct the Evaluator % First mode (The object are passed by reference) -Xeval1 = opencossan.workers.Evaluator('Xmio',Xm,'Sdescription','fist Evaluator'); +Xeval1 = opencossan.workers.Evaluator('Solver',Xm,'Description','fist Evaluator'); % In order to be able to construct our Model an Input object must be % defined %% Define an Input % Define RVs -RV1=opencossan.common.inputs.random.NormalRandomVariable('mean',1,'std',1); %#ok -RV2=opencossan.common.inputs.random.NormalRandomVariable('mean',3,'std',1); %#ok +RV1=opencossan.common.inputs.random.NormalRandomVariable('mean',1,'std',1); +RV2=opencossan.common.inputs.random.NormalRandomVariable('mean',3,'std',1); % Define the RVset Xrvs1=opencossan.common.inputs.random.RandomVariableSet('names',{'RV1', 'RV2'}, 'members', [RV1; RV2]); % Define Xinput @@ -63,9 +63,9 @@ Xin = sample(Xin,'Nsamples',10); %% Construct the Model -Xmdl=opencossan.common.Model('Input',Xin,'evaluator',Xeval1); %#ok +Xmdl=opencossan.common.Model('Input',Xin,'Evaluator',Xeval1); %#ok % or -Xmdl=opencossan.common.Model('input',Xin,'evaluator',Xeval1,'description','The Model'); +Xmdl=opencossan.common.Model('input',Xin,'Evaluator',Xeval1,'Description','The Model'); % Show Model details display(Xmdl) @@ -87,7 +87,7 @@ MX=Xo2.getValues('Cnames',Xo2.Cnames); % % Check solution -assert(all(MX(:,3)-MX(:,2)==MX(:,1)),'openCOSSAN:Tutorial','wrong results'); +assert(all(MX(:,3)-MX(:,2)==MX(:,1)),'OpenCossan:Tutorial','wrong results'); disp('Tutorial terminated successfully') diff --git a/docs/tutorials/CossanObjects/TutorialModes.m b/docs/tutorials/CossanObjects/TutorialModes.m index f60f3a4b..084029dc 100644 --- a/docs/tutorials/CossanObjects/TutorialModes.m +++ b/docs/tutorials/CossanObjects/TutorialModes.m @@ -3,7 +3,7 @@ % compute frequency response functions % % -% See Also: http://cossan.cfd.liv.ac.uk/wiki/index.php/@Modes +% See Also: Modes % % $Copyright~1993-2011,~COSSAN~Working~Group,~University~of~Innsbruck,~Austria$ % $Author: Barbara-Goller$ @@ -19,7 +19,7 @@ % Determine modal properties opts.disp = 0; Nmodes = 20; % number of modes to be retained -[Phi Lam]=eigs(stiff,mass,Nmodes,0,opts); +[Phi. Lam]=eigs(stiff,mass,Nmodes,0,opts); Vlam = diag(Lam); % Construct object Modes diff --git a/docs/tutorials/CossanObjects/TutorialMonteCarlo.m b/docs/tutorials/CossanObjects/TutorialMonteCarlo.m index 7b1394d1..45ccfeb8 100644 --- a/docs/tutorials/CossanObjects/TutorialMonteCarlo.m +++ b/docs/tutorials/CossanObjects/TutorialMonteCarlo.m @@ -35,7 +35,7 @@ %% Define the evaluator % Construct a Mio object -Xm=opencossan.workers.Mio( 'description', 'Performance function', ... +Xm=opencossan.workers.MatlabWorker('Description', 'Performance function', ... 'script','for j=1:length(Tinput), Toutput(j).out1=sqrt(Tinput(j).RV1^2+Tinput(j).RV2^2); end', ... ...'Liostructure',true,... 'format','structure',... @@ -45,10 +45,10 @@ % Construct the Evaluator -Xeval = opencossan.workers.Evaluator('Xmio',Xm,'Sdescription','Evaluator xmio'); +Xeval = opencossan.workers.Evaluator('Solver',Xm,'Description','Evaluator xmio'); %% Define a Model -Xmdl=opencossan.common.Model('evaluator',Xeval,'input',Xin); +Xmdl=opencossan.common.Model('Evaluator',Xeval,'Input',Xin); %% Define a Monte Carlo object % The montecarlo object defines the number of simulations to be used, the number @@ -65,7 +65,7 @@ % Define performance function Xpf=opencossan.reliability.PerformanceFunction('OutputName','Vg','Capacity','Xthreshold','Demand','out1'); % Construct the model -Xpm=opencossan.reliability.ProbabilisticModel('model',Xmdl,'performanceFunction',Xpf); +Xpm=opencossan.reliability.ProbabilisticModel('Model',Xmdl,'PerformanceFunction',Xpf); % Apply MC object to generate samples of the ProbabilisticModel Xo=Xmc.apply(Xpm); @@ -184,4 +184,4 @@ display(Xof1) %% Validate Solutions -assert(abs(Xof1.pfhat-0.6823)<1e-4,'openCOSSAN:Tutorials','Wrong results') +assert(abs(Xof1.pfhat-0.6823)<1e-4,'OpemCossan:Tutorials','Wrong results') diff --git a/docs/tutorials/CossanObjects/TutorialObjectiveFunction.m b/docs/tutorials/CossanObjects/TutorialObjectiveFunction.m index 534d8303..63dc197a 100644 --- a/docs/tutorials/CossanObjects/TutorialObjectiveFunction.m +++ b/docs/tutorials/CossanObjects/TutorialObjectiveFunction.m @@ -5,19 +5,36 @@ % Please refer to the Mio tutorial and Optimization tutorial for more % examples of objective function % -% See Also: http://cossan.cfd.liv.ac.uk/wiki/index.php/@ObjectiveFunction +% See Also: ObjectiveFunction TutorialConstraint +% TutorialOptimizationProblem % -% $Copyright~1993-2011,~COSSAN~Working~Group,~University~of~Innsbruck,~Austria$ -% $Author:~Edoardo~Patelli$ -clear -close all -clc; +% Author: Edoardo Patelli + +%{ + This file is part of OpenCossan . + Copyright (C) 2006-2020 COSSAN WORKING GROUP + + OpenCossan is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License or, + (at your option) any later version. + + OpenCossan is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with OpenCossan. If not, see . +%} + +clear, clc; +import opencossan.optimization.* + %% Constructor -Xofun = opencossan.optimization.ObjectiveFunction('description','objective function', ... - 'IsFunction',true, ... -... 'Liostructure',true,'Liomatrix',false,... +Xofun = ObjectiveFunction('description','objective function', ... 'InputNames',{'X1','X2'},... % Define the inputs - 'function',@rastriginsfcn,... + 'FunctionHandle',@rastriginsfcn,... 'OutputNames',{'fobj'}); % Define the outputs % Show details of the ObjectiveFunction @@ -27,11 +44,8 @@ ScurrentPath=which('TutorialObjectiveFunction'); [Spath, ~ ]=fileparts(ScurrentPath); -Xofun1 = opencossan.optimization.ObjectiveFunction('description','objective function of optimization problem', ... - 'FullFileName',fullfile(Spath,'Files4Mio','ExampleMioStructure') ,... -... 'Sfile','ExampleMioStructure',... -... 'Liostructure',true,... -... 'Lfunction',true,... +Xofun1 = ObjectiveFunction('description','objective function of optimization problem', ... + 'FullFileName',fullfile(Spath,'Files4MatlabWorker','ExampleMatlabWorkerStructure.m') ,... 'InputNames',{'X1','X2'},... 'OutputNames',{'mioout'}); @@ -39,6 +53,10 @@ %% Use ObjectiveFunction -% In order to be able to use the method evaluate of ObjectiveFunction an -% OptimizationProblem needs to be defined. -Xofun1.evaluate +% ObjectiveFunction can be evaluate by passing a table containing the name +% of variables defined in ObjectiveFunction.InputNames + +TableInput=array2table([0 0; 4 3; 4, 1],'VariableName',{'X1','X2'}); +TableOutput=Xofun1.evaluate(TableInput); + +disp(TableOutput.fobj) \ No newline at end of file diff --git a/docs/tutorials/CossanObjects/TutorialTimer.m b/docs/tutorials/CossanObjects/TutorialTimer.m index 0d58bc4d..046f92e7 100644 --- a/docs/tutorials/CossanObjects/TutorialTimer.m +++ b/docs/tutorials/CossanObjects/TutorialTimer.m @@ -49,61 +49,60 @@ ... opencossan.OpenCossan.setLaptime %% Get the Timer object -% A timer object can be retrieved via the Analysis object -Xanalysis=opencossan.OpenCossan.getAnalysis; -% The timer is self is -Xanalysis.Xtimer +% A timer object can be retrieved via the OpenCossan object +Xtimer=opencossan.OpenCossan.getTimer; + % Reset timer -Xanalysis.Xtimer.reset -display(Xanalysis.Xtimer) +Xtimer.reset +display(Xtimer) % Get total time of the analyis -Xanalysis.Xtimer.totalTime +Xtimer.TotalTime pause(1) % The total time does not change because the Timer is not running -Xanalysis.Xtimer.totalTime +Xtimer.TotalTime % start the total time of the analyis -Xanalysis.Xtimer.starttime -Xanalysis.Xtimer.totalTime +Xtimer.start +Xtimer.TotalTime pause(1) -Xanalysis.Xtimer.totalTime +Xtimer.TotalTime % It is possible to assign labels to the Timer -Xanalysis.Xtimer.starttime('Sdescription','MyLabel 1') +Xtimer.start('Description','MyLabel 1') pause(1) -Xanalysis.Xtimer.starttime('Sdescription','MyLabel 2') -display(Xanalysis.Xtimer) +Xtimer.start('Description','MyLabel 2') +display(Xtimer) %% Get the lap time % returns the time elapsed from previous counter and start a new counter -counterID=laptime(Xanalysis.Xtimer,'Sdescription','MyLabel 3'); -display(Xanalysis.Xtimer) +counterID=lap(Xtimer,'Description','MyLabel 3'); +display(Xtimer) %% Get enlapsed time between two counters -Xanalysis.Xtimer.deltatime(counterID-1,counterID) -display(Xanalysis.Xtimer) +Xtimer.delta(counterID-1,counterID) +display(Xtimer) -counterID2=laptime(Xanalysis.Xtimer,'Sdescription','MyLabel 4'); -Xanalysis.Xtimer.deltatime(counterID,counterID2) -display(Xanalysis.Xtimer) +counterID2=lap(Xtimer,'Description','MyLabel 4'); +Xtimer.delta(counterID,counterID2) +display(Xtimer) - % The method returns as optional output argument the number of - % the new counter started +% The method returns as optional output argument the number of +% the new counter started %% Plot time SfigureName='PlotTimer'; Sformat='pdf'; -Xanalysis.Xtimer.plot('SfigureName',SfigureName,'Stitle','Tutorial Timer','SexportFormat',Sformat) +Xtimer.plot('FigureName',SfigureName,'Title','Tutorial Timer','ExportFormat',Sformat) % The figure is export in the Cossan working directory assert (logical(exist(fullfile(OpenCossan.getCossanWorkingPath,[SfigureName '.' Sformat]),'file')),... 'OpenCossan:Tutorial:TutorialTimer:nofigurecreated','Figure Not created') % remove created file -delete(fullfile(OpenCossan.getCossanWorkingPath,[SfigureName '.' Sformat])) +delete(fullfile(opencossan.OpenCossan.getWorkingPath,[SfigureName '.' Sformat])) diff --git a/test/unit/+opencossan/+workers/EvaluatorTest.m b/test/unit/+opencossan/+workers/EvaluatorTest.m index 1eaaf0ab..947361b5 100644 --- a/test/unit/+opencossan/+workers/EvaluatorTest.m +++ b/test/unit/+opencossan/+workers/EvaluatorTest.m @@ -53,7 +53,7 @@ function emptyConstructor(testCase) function constructorShouldSetDescription(testCase) Xe = opencossan.workers.Evaluator('Description','Evaluator',... - 'Solver',testCase.MatWorker ); + 'Solver',testCase.MatWorker,'SolverName','test'); testCase.assertEqual(Xe.Description,"Evaluator"); end @@ -63,7 +63,7 @@ function constructorShouldSetMatlabWorker(testCase) testCase.assertEqual(Xe.Solver(1),testCase.MatWorker); end - function constructorShouldFail(testCase) + function constructorTwoWorkers(testCase) Xe = opencossan.workers.Evaluator('Solver',[testCase.MatWorker testCase.MatWorker2],... 'SolverName',["MatlabWorker" "ExtraName"]); @@ -72,85 +72,24 @@ function constructorShouldFail(testCase) function constructorShouldSetJobManagerInterface(testCase) Xjmi = opencossan.highperformancecomputing.JobManagerInterface(); - Xe = opencossan.workers.Evaluator('Solver',testCase.MatWorker,'JobManager',Xjmi); + Xe = opencossan.workers.Evaluator('Solver',testCase.MatWorker,'JobManager',Xjmi,'SolverName','MatWorker'); testCase.assertEqual(Xe.JobManager,Xjmi); end - function constructorShouldSetLremoteInjectExtract(testCase) - Xe = opencossan.workers.Evaluator('Solver',testCase.MatWorker,... - 'RemoteInjectExtract',true); - testCase.assertTrue(Xe.RemoteInjectExtract); - end - - function constructorShouldSetHostNames(testCase) - - Squeues = ["Queue1" "Queue2"]; - Shostnames = ["Host1","Host2"]; - Xe = opencossan.workers.Evaluator('Solver',[testCase.ConWorker testCase.MatWorker],... - 'Hostnames',Shostnames,... - 'Queues',Squeues); - testCase.assertEqual(Xe.Hostnames,Shostnames); - end - - function constructorShouldSetQueues(testCase) - Squeues = ["Queue1" "Queue2"]; - Xe = opencossan.workers.Evaluator('Solver',... - [testCase.ConWorker testCase.MatWorker],'Queues',Squeues); - testCase.assertEqual(Xe.Queues,Squeues); - end - - function constructorShouldSetVconcurrent(testCase) - Xe = opencossan.workers.Evaluator('Solver',[testCase.ConWorker testCase.MatWorker],... - 'MaxCuncurrentJobs',[Inf 4]); - testCase.assertEqual(Xe.MaxCuncurrentJobs,[Inf 4]); - end - - function constructorShouldSetMembers(testCase) - Xe = opencossan.workers.Evaluator('Solver',[testCase.ConWorker testCase.MatWorker]); - testCase.assertEqual(Xe.Solver,[testCase.ConWorker testCase.MatWorker]); - end - - function constructorShouldSetNames(testCase) - CSnames = ["Connector Name", "Mio name"]; - Xe = opencossan.workers.Evaluator('Solver',[testCase.ConWorker testCase.MatWorker],'SolverName',CSnames); - testCase.assertEqual(Xe.Solver,[testCase.ConWorker testCase.MatWorker]); - testCase.assertEqual(Xe.SolverName,CSnames); - end - - function constructorShouldSetSolutionSequence(testCase) - Xss = opencossan.workers.SolutionSequence(); - Xe = opencossan.workers.Evaluator('Solver',Xss); - testCase.assertEqual(Xe.Solver(1),Xss); - end - - function constructorShouldSetMetaModel(testCase) - Xrs = opencossan.metamodels.ResponseSurface(); - Xe = opencossan.workers.Evaluator('Solver',Xrs); - testCase.assertEqual(Xe.Solver(1),Xrs); - end - - function constructorShouldSetParallelEnvironment(testCase) - Xe = opencossan.workers.Evaluator('Solver',[testCase.ConWorker testCase.MatWorker],... - 'ParallelEnvironments',["parallel" "parallel"]); - testCase.assertEqual(Xe.ParallelEnvironments,["parallel";"parallel"]); - end - - function constructorShouldSetVSlots(testCase) - Xe = opencossan.workers.Evaluator('Solver',[testCase.ConWorker testCase.MatWorker],... - 'Slots',[1 1]); - testCase.assertEqual(Xe.Slots,[1 1]); - end + function constructorTestInputNames(testCase) - Xe = opencossan.workers.Evaluator('Solver',[testCase.MatWorker testCase.MatWorker2]); + Xe = opencossan.workers.Evaluator('Solver',[testCase.MatWorker testCase.MatWorker2],... + 'SolverName',["MatWorker" "MatWorker2"]); Cinput=[testCase.MatWorker.InputNames testCase.MatWorker2.InputNames]; testCase.assertEqual(Xe.InputNames,Cinput); end function constructorTestOutputNames(testCase) - Xe = opencossan.workers.Evaluator('Solver',[testCase.MatWorker testCase.MatWorker2]); + Xe = opencossan.workers.Evaluator('Solver',[testCase.MatWorker testCase.MatWorker2],... + 'SolverName',["MatWorker" "MatWorker2"]); Coutput=[testCase.MatWorker.OutputNames testCase.MatWorker2.OutputNames]; testCase.assertEqual(Xe.OutputNames,Coutput); end