From 30c33d167497d8a721e625d2febff131efac6391 Mon Sep 17 00:00:00 2001 From: shepf Date: Thu, 13 Dec 2018 11:21:45 +0800 Subject: [PATCH 1/5] example for generating FTP job(.kjb) --- .../samples/embedding/GeneratingFtpJobs.java | 188 ++++++++++++++++++ 1 file changed, 188 insertions(+) create mode 100644 kettle-sdk-embedding-samples/src/main/java/org/pentaho/di/sdk/samples/embedding/GeneratingFtpJobs.java diff --git a/kettle-sdk-embedding-samples/src/main/java/org/pentaho/di/sdk/samples/embedding/GeneratingFtpJobs.java b/kettle-sdk-embedding-samples/src/main/java/org/pentaho/di/sdk/samples/embedding/GeneratingFtpJobs.java new file mode 100644 index 000000000..eb4c8e453 --- /dev/null +++ b/kettle-sdk-embedding-samples/src/main/java/org/pentaho/di/sdk/samples/embedding/GeneratingFtpJobs.java @@ -0,0 +1,188 @@ +package org.pentaho.di.sdk.samples.embedding; + +import java.io.File; + +import org.apache.commons.io.FileUtils; +import org.pentaho.di.core.KettleEnvironment; +import org.pentaho.di.core.logging.LogLevel; +import org.pentaho.di.job.JobHopMeta; +import org.pentaho.di.job.JobMeta; +import org.pentaho.di.job.entries.ftp.JobEntryFTP; +import org.pentaho.di.job.entries.special.JobEntrySpecial; +import org.pentaho.di.job.entries.success.JobEntrySuccess; +import org.pentaho.di.job.entries.writetolog.JobEntryWriteToLog; +import org.pentaho.di.job.entry.JobEntryCopy; + +/** + * This class demonstrates how to create a PDI FTP job definition + * in code, and save it to a kjb file. + */ +public class GeneratingFtpJobs { + + public static GeneratingFtpJobs instance; + + /** + * @param args not used + */ + public static void main( String[] args ) { + + try { + // Kettle Environment must always be initialized first when using PDI + // It bootstraps the PDI engine by loading settings, appropriate plugins + // etc. + KettleEnvironment.init( false ); + + // Create an instance of this demo class for convenience + instance = new GeneratingFtpJobs(); + + // generates a simple job, returning the JobMeta object describing it + JobMeta jobMeta = instance.generateJob(); + + // get the xml of the definition and save it to a file for inspection in spoon + String outputFilename = "etl/generated_ftp_job.kjb"; + System.out.println( "- Saving to " + outputFilename ); + String xml = jobMeta.getXML(); + File file = new File( outputFilename ); + FileUtils.writeStringToFile( file, xml, "UTF-8" ); + + System.out.println( "DONE" ); + } catch ( Exception e ) { + e.printStackTrace(); + return; + } + + } + + /** + * This method generates a job definition from scratch. + * + * It demonstrates the following: + * + * - Creating a new job + * - Creating and connecting job entries + * + * @return the generated job definition + */ + public JobMeta generateJob() { + + try { + System.out.println( "Generating a FTP job definition" ); + + // create empty transformation definition + JobMeta jobMeta = new JobMeta(); + jobMeta.setName( "Generated Demo FTP Job" ); + + // ------------------------------------------------------------------------------------ + // Create start entry and put it into the job + // ------------------------------------------------------------------------------------ + System.out.println( "- Adding Start Entry" ); + + // Create and configure start entry + JobEntrySpecial start = new JobEntrySpecial(); + start.setName( "START" ); + start.setStart( true ); + + // wrap into JobEntryCopy object, which holds generic job entry information + JobEntryCopy startEntry = new JobEntryCopy( start ); + + // place it on Spoon canvas properly + startEntry.setDrawn( true ); + startEntry.setLocation( 100, 100 ); + + jobMeta.addJobEntry( startEntry ); + + // ------------------------------------------------------------------------------------ + // Create "write to log" entry and put it into the job + // ------------------------------------------------------------------------------------ + System.out.println( "- Adding Write To Log Entry" ); + + // Create and configure entry + JobEntryWriteToLog writeToLog = new JobEntryWriteToLog(); + writeToLog.setName( "This is FTP job example" ); + writeToLog.setLogLevel( LogLevel.MINIMAL ); + writeToLog.setLogSubject( "Logging PDI Build Information:" ); + writeToLog.setLogMessage( "Version: ${Internal.Kettle.Version}\n" + + "Build Date: ${Internal.Kettle.Build.Date}" ); + + // wrap into JobEntryCopy object, which holds generic job entry information + JobEntryCopy writeToLogEntry = new JobEntryCopy( writeToLog ); + + // place it on Spoon canvas properly + writeToLogEntry.setDrawn( true ); + writeToLogEntry.setLocation( 300, 100 ); + + jobMeta.addJobEntry( writeToLogEntry ); + + // connect start entry to logging entry using simple hop + jobMeta.addJobHop( new JobHopMeta( startEntry, writeToLogEntry ) ); + + + // ------------------------------------------------------------------------------------ + // Create "FTP" entry and put it into the job + // ------------------------------------------------------------------------------------ + System.out.println( "- Adding FTP Entry" ); + + // crate and configure entry + // + JobEntryFTP ftp = new JobEntryFTP(); + ftp.setName( "FTP Job" ); + //set ftp parameters + ftp.setServerName("127.0.0.1"); + ftp.setPort("21"); + ftp.setUserName("ftp1"); + // + ftp.setFtpDirectory("/"); + ftp.setWildcard(".*"); + ftp.setTargetDirectory("/tmp/"); + + + // wrap into JobEntryCopy object, which holds generic job entry information + JobEntryCopy ftpEntry = new JobEntryCopy( ftp ); + + // place it on Spoon canvas properly + ftpEntry.setDrawn( true ); + ftpEntry.setLocation( 500, 100 ); + + jobMeta.addJobEntry( ftpEntry ); + + // connect logging entry to FTP entry on true evaluation + JobHopMeta greenHop1 = new JobHopMeta( writeToLogEntry, ftpEntry ); + greenHop1.setEvaluation( true ); + jobMeta.addJobHop( greenHop1 ); + + + // ------------------------------------------------------------------------------------ + // Create "success" entry and put it into the job + // ------------------------------------------------------------------------------------ + System.out.println( "- Adding Success Entry" ); + + // crate and configure entry + JobEntrySuccess success = new JobEntrySuccess(); + success.setName( "Success" ); + + // wrap into JobEntryCopy object, which holds generic job entry information + JobEntryCopy successEntry = new JobEntryCopy( success ); + + // place it on Spoon canvas properly + successEntry.setDrawn( true ); + successEntry.setLocation( 700, 100 ); + + jobMeta.addJobEntry( successEntry ); + + // connect logging entry to success entry on TRUE evaluation + JobHopMeta greenHop = new JobHopMeta( ftpEntry, successEntry ); + greenHop.setEvaluation( true ); + jobMeta.addJobHop( greenHop ); + + + + return jobMeta; + + } catch ( Exception e ) { + + // something went wrong, just log and return + e.printStackTrace(); + return null; + } + } +} From 6d23acdfafb86f601a60fead80e35c2655f8ac03 Mon Sep 17 00:00:00 2001 From: shepf Date: Mon, 17 Dec 2018 19:50:04 +0800 Subject: [PATCH 2/5] =?UTF-8?q?=20json=E9=85=8D=E7=BD=AE=E9=80=9A=E7=94=A8?= =?UTF-8?q?JobEntry=20kettle=E6=8F=92=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 4 + .../samples/embedding/GeneratingFtpJobs.java | 4 +- .../embedding/GeneratingFtpPlusJobs.java | 186 +++ .../di/sdk/samples/embedding/RunningJobs.java | 4 +- .../entries/ftpplus/JobEntryFtpPlus.java | 1272 +++++++++++++++++ kettle-sdk-jobentry-plugin/pom.xml | 22 + .../jobentries/ftpplus/JobEntryFtpPlus.java | 102 ++ .../ftpplus/JobEntryFtpPlusDialog.java | 361 +++++ .../ftpplus/JobEntryFtpPlusParamsDO.java | 68 + .../samples/jobentries/demo/JobEntryDemo.java | 20 +- .../messages/messages_en_US.properties | 17 + .../jobentries/ftpplus/resources/demo.png | Bin 0 -> 916 bytes .../jobentries/ftpplus/resources/demo.svg | 25 + 13 files changed, 2071 insertions(+), 14 deletions(-) create mode 100644 kettle-sdk-embedding-samples/src/main/java/org/pentaho/di/sdk/samples/embedding/GeneratingFtpPlusJobs.java create mode 100644 kettle-sdk-embedding-samples/src/main/java/org/pentaho/di/sdk/samples/embedding/entries/ftpplus/JobEntryFtpPlus.java create mode 100644 kettle-sdk-jobentry-plugin/src/main/java/org/pentaho/di/sdk/myplugins/jobentries/ftpplus/JobEntryFtpPlus.java create mode 100644 kettle-sdk-jobentry-plugin/src/main/java/org/pentaho/di/sdk/myplugins/jobentries/ftpplus/JobEntryFtpPlusDialog.java create mode 100644 kettle-sdk-jobentry-plugin/src/main/java/org/pentaho/di/sdk/myplugins/jobentries/ftpplus/JobEntryFtpPlusParamsDO.java create mode 100644 kettle-sdk-jobentry-plugin/src/main/resources/org/pentaho/di/sdk/myplugins/jobentries/ftpplus/messages/messages_en_US.properties create mode 100644 kettle-sdk-jobentry-plugin/src/main/resources/org/pentaho/di/sdk/myplugins/jobentries/ftpplus/resources/demo.png create mode 100644 kettle-sdk-jobentry-plugin/src/main/resources/org/pentaho/di/sdk/myplugins/jobentries/ftpplus/resources/demo.svg diff --git a/README.md b/README.md index a9f4eb4ae..40def5d90 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,8 @@ # PDI SDK Samples +fork官方demo，编写通用dialog +操作界面统一使用一个JSON对象作为参数设置途径。 +采用JSON作为参数配置，省去ui调试设计步骤，只需一个类就可以开发一个插件。 + ### Pre-requisites for building the project: * Maven, version 3+ diff --git a/kettle-sdk-embedding-samples/src/main/java/org/pentaho/di/sdk/samples/embedding/GeneratingFtpJobs.java b/kettle-sdk-embedding-samples/src/main/java/org/pentaho/di/sdk/samples/embedding/GeneratingFtpJobs.java index eb4c8e453..fbac41c7a 100644 --- a/kettle-sdk-embedding-samples/src/main/java/org/pentaho/di/sdk/samples/embedding/GeneratingFtpJobs.java +++ b/kettle-sdk-embedding-samples/src/main/java/org/pentaho/di/sdk/samples/embedding/GeneratingFtpJobs.java @@ -27,7 +27,7 @@ public class GeneratingFtpJobs { public static void main( String[] args ) { try { - // Kettle Environment must always be initialized first when using PDI + // Kettle Environment must be initialized first when using PDI // It bootstraps the PDI engine by loading settings, appropriate plugins // etc. KettleEnvironment.init( false ); @@ -127,7 +127,7 @@ public JobMeta generateJob() { JobEntryFTP ftp = new JobEntryFTP(); ftp.setName( "FTP Job" ); //set ftp parameters - ftp.setServerName("127.0.0.1"); + ftp.setServerName("11.12.112.84"); ftp.setPort("21"); ftp.setUserName("ftp1"); // diff --git a/kettle-sdk-embedding-samples/src/main/java/org/pentaho/di/sdk/samples/embedding/GeneratingFtpPlusJobs.java b/kettle-sdk-embedding-samples/src/main/java/org/pentaho/di/sdk/samples/embedding/GeneratingFtpPlusJobs.java new file mode 100644 index 000000000..9bbfda660 --- /dev/null +++ b/kettle-sdk-embedding-samples/src/main/java/org/pentaho/di/sdk/samples/embedding/GeneratingFtpPlusJobs.java @@ -0,0 +1,186 @@ +package org.pentaho.di.sdk.samples.embedding; + +import org.apache.commons.io.FileUtils; +import org.pentaho.di.core.KettleEnvironment; +import org.pentaho.di.core.logging.LogLevel; +import org.pentaho.di.job.JobHopMeta; +import org.pentaho.di.job.JobMeta; +import org.pentaho.di.job.entries.special.JobEntrySpecial; +import org.pentaho.di.job.entries.success.JobEntrySuccess; +import org.pentaho.di.job.entries.writetolog.JobEntryWriteToLog; +import org.pentaho.di.job.entry.JobEntryCopy; +import org.pentaho.di.sdk.samples.embedding.entries.ftpplus.JobEntryFtpPlus; + +import java.io.File; + +public class GeneratingFtpPlusJobs { + + public static GeneratingFtpPlusJobs instance; + + /** + * @param args not used + */ + public static void main( String[] args ) { + + try { + // Kettle Environment must be initialized first when using PDI + // It bootstraps the PDI engine by loading settings, appropriate plugins + // etc. + KettleEnvironment.init( false ); + + // Create an instance of this demo class for convenience + instance = new GeneratingFtpPlusJobs(); + + // generates a simple job, returning the JobMeta object describing it + JobMeta jobMeta = instance.generateJob(); + + // get the xml of the definition and save it to a file for inspection in spoon + String outputFilename = "etl/generated_ftp_job.kjb"; + System.out.println( "- Saving to " + outputFilename ); + String xml = jobMeta.getXML(); + File file = new File( outputFilename ); + FileUtils.writeStringToFile( file, xml, "UTF-8" ); + + System.out.println( "DONE" ); + } catch ( Exception e ) { + e.printStackTrace(); + return; + } + + } + + /** + * This method generates a job definition from scratch. + * + * It demonstrates the following: + * + * - Creating a new job + * - Creating and connecting job entries + * + * @return the generated job definition + */ + public JobMeta generateJob() { + + try { + System.out.println( "Generating a FTP job definition" ); + + // create empty transformation definition + JobMeta jobMeta = new JobMeta(); + jobMeta.setName( "Generated Demo FtpPlus Job" ); + + // ------------------------------------------------------------------------------------ + // Create start entry and put it into the job + // ------------------------------------------------------------------------------------ + System.out.println( "- Adding Start Entry" ); + + // Create and configure start entry + JobEntrySpecial start = new JobEntrySpecial(); + start.setName( "START" ); + start.setStart( true ); + + // wrap into JobEntryCopy object, which holds generic job entry information + JobEntryCopy startEntry = new JobEntryCopy( start ); + + // place it on Spoon canvas properly + startEntry.setDrawn( true ); + startEntry.setLocation( 100, 100 ); + + jobMeta.addJobEntry( startEntry ); + + // ------------------------------------------------------------------------------------ + // Create "write to log" entry and put it into the job + // ------------------------------------------------------------------------------------ + System.out.println( "- Adding Write To Log Entry" ); + + // Create and configure entry + JobEntryWriteToLog writeToLog = new JobEntryWriteToLog(); + writeToLog.setName( "This is FTP job example" ); + writeToLog.setLogLevel( LogLevel.MINIMAL ); + writeToLog.setLogSubject( "Logging PDI Build Information:" ); + writeToLog.setLogMessage( "Version: ${Internal.Kettle.Version}\n" + + "Build Date: ${Internal.Kettle.Build.Date}" ); + + // wrap into JobEntryCopy object, which holds generic job entry information + JobEntryCopy writeToLogEntry = new JobEntryCopy( writeToLog ); + + // place it on Spoon canvas properly + writeToLogEntry.setDrawn( true ); + writeToLogEntry.setLocation( 300, 100 ); + + jobMeta.addJobEntry( writeToLogEntry ); + + // connect start entry to logging entry using simple hop + jobMeta.addJobHop( new JobHopMeta( startEntry, writeToLogEntry ) ); + + + // ------------------------------------------------------------------------------------ + // Create "FTP" entry and put it into the job + // ------------------------------------------------------------------------------------ + System.out.println( "- Adding FTP Entry" ); + + // crate and configure entry + // + JobEntryFtpPlus ftp = new JobEntryFtpPlus(); + ftp.setName( "FtpPlus Job" ); + //set ftp parameters + ftp.setServerName("127.0.0.1"); + ftp.setPort("21"); + ftp.setUserName("ftp1"); + // + ftp.setFtpDirectory("/"); + ftp.setWildcard(".*"); + ftp.setTargetDirectory("/tmp"); + + + // wrap into JobEntryCopy object, which holds generic job entry information + JobEntryCopy ftpEntry = new JobEntryCopy( ftp ); + + // place it on Spoon canvas properly + ftpEntry.setDrawn( true ); + ftpEntry.setLocation( 500, 100 ); + + jobMeta.addJobEntry( ftpEntry ); + + // connect logging entry to FTP entry on true evaluation + JobHopMeta greenHop1 = new JobHopMeta( writeToLogEntry, ftpEntry ); + greenHop1.setEvaluation( true ); + jobMeta.addJobHop( greenHop1 ); + + + // ------------------------------------------------------------------------------------ + // Create "success" entry and put it into the job + // ------------------------------------------------------------------------------------ + System.out.println( "- Adding Success Entry" ); + + // crate and configure entry + JobEntrySuccess success = new JobEntrySuccess(); + success.setName( "Success" ); + + // wrap into JobEntryCopy object, which holds generic job entry information + JobEntryCopy successEntry = new JobEntryCopy( success ); + + // place it on Spoon canvas properly + successEntry.setDrawn( true ); + successEntry.setLocation( 700, 100 ); + + jobMeta.addJobEntry( successEntry ); + + // connect logging entry to success entry on TRUE evaluation + JobHopMeta greenHop = new JobHopMeta( ftpEntry, successEntry ); + greenHop.setEvaluation( true ); + jobMeta.addJobHop( greenHop ); + + + + return jobMeta; + + } catch ( Exception e ) { + + // something went wrong, just log and return + e.printStackTrace(); + return null; + } + } + + +} diff --git a/kettle-sdk-embedding-samples/src/main/java/org/pentaho/di/sdk/samples/embedding/RunningJobs.java b/kettle-sdk-embedding-samples/src/main/java/org/pentaho/di/sdk/samples/embedding/RunningJobs.java index 5eba91649..8f09c0386 100644 --- a/kettle-sdk-embedding-samples/src/main/java/org/pentaho/di/sdk/samples/embedding/RunningJobs.java +++ b/kettle-sdk-embedding-samples/src/main/java/org/pentaho/di/sdk/samples/embedding/RunningJobs.java @@ -68,7 +68,7 @@ public static void main( String[] args ) { instance = new RunningJobs(); // run a job from the file system - Job job = instance.runJobFromFileSystem( "etl/parameterized_job.kjb" ); + Job job = instance.runJobFromFileSystem( "etl/generated_ftp_job.kjb" ); // retrieve logging appender LoggingBuffer appender = KettleLogStore.getAppender(); @@ -146,7 +146,7 @@ public Job runJobFromFileSystem( String filename ) { Job job = new Job( null, jobMeta ); // adjust the log level - job.setLogLevel( LogLevel.MINIMAL ); + job.setLogLevel( LogLevel.ROWLEVEL ); System.out.println( "\nStarting job" ); diff --git a/kettle-sdk-embedding-samples/src/main/java/org/pentaho/di/sdk/samples/embedding/entries/ftpplus/JobEntryFtpPlus.java b/kettle-sdk-embedding-samples/src/main/java/org/pentaho/di/sdk/samples/embedding/entries/ftpplus/JobEntryFtpPlus.java new file mode 100644 index 000000000..8a1f59501 --- /dev/null +++ b/kettle-sdk-embedding-samples/src/main/java/org/pentaho/di/sdk/samples/embedding/entries/ftpplus/JobEntryFtpPlus.java @@ -0,0 +1,1272 @@ +/*! ****************************************************************************** + * + * Pentaho Data Integration + * + * Copyright (C) 2002-2017 by Hitachi Vantara : http://www.pentaho.com + * + ******************************************************************************* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + + +package org.pentaho.di.sdk.samples.embedding.entries.ftpplus; + + +import org.pentaho.di.job.entries.ftp.MVSFileParser; +import org.pentaho.di.job.entry.validator.AndValidator; +import org.pentaho.di.job.entry.validator.JobEntryValidatorUtils; + +import java.io.File; +import java.io.IOException; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import com.google.common.annotations.VisibleForTesting; +import org.apache.commons.vfs2.FileObject; +import org.pentaho.di.cluster.SlaveServer; +import org.pentaho.di.core.CheckResultInterface; +import org.pentaho.di.core.Const; +import org.pentaho.di.core.util.Utils; +import org.pentaho.di.core.Result; +import org.pentaho.di.core.ResultFile; +import org.pentaho.di.core.database.DatabaseMeta; +import org.pentaho.di.core.encryption.Encr; +import org.pentaho.di.core.exception.KettleDatabaseException; +import org.pentaho.di.core.exception.KettleException; +import org.pentaho.di.core.exception.KettleXMLException; +import org.pentaho.di.core.util.StringUtil; +import org.pentaho.di.core.variables.VariableSpace; +import org.pentaho.di.core.vfs.KettleVFS; +import org.pentaho.di.core.xml.XMLHandler; +import org.pentaho.di.i18n.BaseMessages; +import org.pentaho.di.job.Job; +import org.pentaho.di.job.JobMeta; +import org.pentaho.di.job.entry.JobEntryBase; +import org.pentaho.di.job.entry.JobEntryInterface; +import org.pentaho.di.repository.ObjectId; +import org.pentaho.di.repository.Repository; +import org.pentaho.di.resource.ResourceEntry; +import org.pentaho.di.resource.ResourceReference; +import org.pentaho.metastore.api.IMetaStore; +import org.w3c.dom.Node; + +import com.enterprisedt.net.ftp.FTPClient; +import com.enterprisedt.net.ftp.FTPException; +import com.enterprisedt.net.ftp.FTPFileFactory; +import com.enterprisedt.net.ftp.FTPFileParser; + +/** + * This defines an FtpPlus job entry. + * + * @author shepf + * @since 12-18-2018 + * + */ + +public class JobEntryFtpPlus extends JobEntryBase implements Cloneable, JobEntryInterface { + private static Class PKG = JobEntryFtpPlus.class; // for i18n purposes, needed by Translator2!! + + private String serverName; + + private String userName; + + private String password; + + private String ftpDirectory; + + private String targetDirectory; + + private String wildcard; + + private boolean binaryMode; + + private int timeout; + + private boolean remove; + + private boolean onlyGettingNewFiles; /* Don't overwrite files */ + + private boolean activeConnection; + + private String controlEncoding; /* how to convert list of filenames e.g. */ + + /** + * Implicit encoding used before PDI v2.4.1 + */ + private static String LEGACY_CONTROL_ENCODING = "US-ASCII"; + + /** + * Default encoding when making a new ftp job entry instance. + */ + private static String DEFAULT_CONTROL_ENCODING = "ISO-8859-1"; + + private boolean movefiles; + + private String movetodirectory; + + private boolean adddate; + + private boolean addtime; + + private boolean SpecifyFormat; + + private String date_time_format; + + private boolean AddDateBeforeExtension; + + private boolean isaddresult; + + private boolean createmovefolder; + + private String port; + + private String proxyHost; + + private String proxyPort; /* string to allow variable substitution */ + + private String proxyUsername; + + private String proxyPassword; + + private String socksProxyHost; + + private String socksProxyPort; + + private String socksProxyUsername; + + private String socksProxyPassword; + + public int ifFileExistsSkip = 0; + + public String SifFileExistsSkip = "ifFileExistsSkip"; + + public int ifFileExistsCreateUniq = 1; + + public String SifFileExistsCreateUniq = "ifFileExistsCreateUniq"; + + public int ifFileExistsFail = 2; + + public String SifFileExistsFail = "ifFileExistsFail"; + + public int ifFileExists; + + public String SifFileExists; + + public String SUCCESS_IF_AT_LEAST_X_FILES_DOWNLOADED = "success_when_at_least"; + + public String SUCCESS_IF_ERRORS_LESS = "success_if_errors_less"; + + public String SUCCESS_IF_NO_ERRORS = "success_if_no_errors"; + + private String nr_limit; + + private String success_condition; + + long NrErrors = 0; + + long NrfilesRetrieved = 0; + + boolean successConditionBroken = false; + + int limitFiles = 0; + + String targetFilename = null; + + static String FILE_SEPARATOR = "/"; + + public JobEntryFtpPlus( String n ) { + super( n, "" ); + nr_limit = "10"; + port = "21"; + socksProxyPort = "1080"; + success_condition = SUCCESS_IF_NO_ERRORS; + ifFileExists = ifFileExistsSkip; + SifFileExists = SifFileExistsSkip; + + serverName = null; + movefiles = false; + movetodirectory = null; + adddate = false; + addtime = false; + SpecifyFormat = false; + AddDateBeforeExtension = false; + isaddresult = true; + createmovefolder = false; + + setControlEncoding( DEFAULT_CONTROL_ENCODING ); + } + + public JobEntryFtpPlus() { + this( "" ); + } + + public Object clone() { + org.pentaho.di.job.entries.ftp.JobEntryFTP je = (org.pentaho.di.job.entries.ftp.JobEntryFTP) super.clone(); + return je; + } + + public String getXML() { + StringBuilder retval = new StringBuilder( 650 ); // 528 chars in spaces and tags alone + + retval.append( super.getXML() ); + retval.append( " " ).append( XMLHandler.addTagValue( "port", port ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "servername", serverName ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "username", userName ) ); + retval.append( " " ).append( + XMLHandler.addTagValue( "password", Encr.encryptPasswordIfNotUsingVariables( password ) ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "ftpdirectory", ftpDirectory ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "targetdirectory", targetDirectory ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "wildcard", wildcard ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "binary", binaryMode ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "timeout", timeout ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "remove", remove ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "only_new", onlyGettingNewFiles ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "active", activeConnection ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "control_encoding", controlEncoding ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "movefiles", movefiles ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "movetodirectory", movetodirectory ) ); + + retval.append( " " ).append( XMLHandler.addTagValue( "adddate", adddate ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "addtime", addtime ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "SpecifyFormat", SpecifyFormat ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "date_time_format", date_time_format ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "AddDateBeforeExtension", AddDateBeforeExtension ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "isaddresult", isaddresult ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "createmovefolder", createmovefolder ) ); + + retval.append( " " ).append( XMLHandler.addTagValue( "proxy_host", proxyHost ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "proxy_port", proxyPort ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "proxy_username", proxyUsername ) ); + retval.append( " " ).append( + XMLHandler.addTagValue( "proxy_password", Encr.encryptPasswordIfNotUsingVariables( proxyPassword ) ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "socksproxy_host", socksProxyHost ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "socksproxy_port", socksProxyPort ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "socksproxy_username", socksProxyUsername ) ); + retval.append( " " ).append( + XMLHandler.addTagValue( "socksproxy_password", Encr + .encryptPasswordIfNotUsingVariables( socksProxyPassword ) ) ); + + retval.append( " " ).append( XMLHandler.addTagValue( "ifFileExists", SifFileExists ) ); + + retval.append( " " ).append( XMLHandler.addTagValue( "nr_limit", nr_limit ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "success_condition", success_condition ) ); + + return retval.toString(); + } + + public void loadXML(Node entrynode, List databases, List slaveServers, + Repository rep, IMetaStore metaStore ) throws KettleXMLException { + try { + super.loadXML( entrynode, databases, slaveServers ); + port = XMLHandler.getTagValue( entrynode, "port" ); + serverName = XMLHandler.getTagValue( entrynode, "servername" ); + userName = XMLHandler.getTagValue( entrynode, "username" ); + password = Encr.decryptPasswordOptionallyEncrypted( XMLHandler.getTagValue( entrynode, "password" ) ); + ftpDirectory = XMLHandler.getTagValue( entrynode, "ftpdirectory" ); + targetDirectory = XMLHandler.getTagValue( entrynode, "targetdirectory" ); + wildcard = XMLHandler.getTagValue( entrynode, "wildcard" ); + binaryMode = "Y".equalsIgnoreCase( XMLHandler.getTagValue( entrynode, "binary" ) ); + timeout = Const.toInt( XMLHandler.getTagValue( entrynode, "timeout" ), 10000 ); + remove = "Y".equalsIgnoreCase( XMLHandler.getTagValue( entrynode, "remove" ) ); + onlyGettingNewFiles = "Y".equalsIgnoreCase( XMLHandler.getTagValue( entrynode, "only_new" ) ); + activeConnection = "Y".equalsIgnoreCase( XMLHandler.getTagValue( entrynode, "active" ) ); + controlEncoding = XMLHandler.getTagValue( entrynode, "control_encoding" ); + if ( controlEncoding == null ) { + // if we couldn't retrieve an encoding, assume it's an old instance and + // put in the the encoding used before v 2.4.0 + controlEncoding = LEGACY_CONTROL_ENCODING; + } + movefiles = "Y".equalsIgnoreCase( XMLHandler.getTagValue( entrynode, "movefiles" ) ); + movetodirectory = XMLHandler.getTagValue( entrynode, "movetodirectory" ); + + adddate = "Y".equalsIgnoreCase( XMLHandler.getTagValue( entrynode, "adddate" ) ); + addtime = "Y".equalsIgnoreCase( XMLHandler.getTagValue( entrynode, "addtime" ) ); + SpecifyFormat = "Y".equalsIgnoreCase( XMLHandler.getTagValue( entrynode, "SpecifyFormat" ) ); + date_time_format = XMLHandler.getTagValue( entrynode, "date_time_format" ); + AddDateBeforeExtension = + "Y".equalsIgnoreCase( XMLHandler.getTagValue( entrynode, "AddDateBeforeExtension" ) ); + + String addresult = XMLHandler.getTagValue( entrynode, "isaddresult" ); + + if ( Utils.isEmpty( addresult ) ) { + isaddresult = true; + } else { + isaddresult = "Y".equalsIgnoreCase( addresult ); + } + + createmovefolder = "Y".equalsIgnoreCase( XMLHandler.getTagValue( entrynode, "createmovefolder" ) ); + + proxyHost = XMLHandler.getTagValue( entrynode, "proxy_host" ); + proxyPort = XMLHandler.getTagValue( entrynode, "proxy_port" ); + proxyUsername = XMLHandler.getTagValue( entrynode, "proxy_username" ); + proxyPassword = + Encr.decryptPasswordOptionallyEncrypted( XMLHandler.getTagValue( entrynode, "proxy_password" ) ); + socksProxyHost = XMLHandler.getTagValue( entrynode, "socksproxy_host" ); + socksProxyPort = XMLHandler.getTagValue( entrynode, "socksproxy_port" ); + socksProxyUsername = XMLHandler.getTagValue( entrynode, "socksproxy_username" ); + socksProxyPassword = + Encr.decryptPasswordOptionallyEncrypted( XMLHandler.getTagValue( entrynode, "socksproxy_password" ) ); + SifFileExists = XMLHandler.getTagValue( entrynode, "ifFileExists" ); + if ( Utils.isEmpty( SifFileExists ) ) { + ifFileExists = ifFileExistsSkip; + } else { + if ( SifFileExists.equals( SifFileExistsCreateUniq ) ) { + ifFileExists = ifFileExistsCreateUniq; + } else if ( SifFileExists.equals( SifFileExistsFail ) ) { + ifFileExists = ifFileExistsFail; + } else { + ifFileExists = ifFileExistsSkip; + } + + } + nr_limit = XMLHandler.getTagValue( entrynode, "nr_limit" ); + success_condition = + Const.NVL( XMLHandler.getTagValue( entrynode, "success_condition" ), SUCCESS_IF_NO_ERRORS ); + + } catch ( KettleXMLException xe ) { + throw new KettleXMLException( "Unable to load job entry of type 'ftp' from XML node", xe ); + } + } + + public void loadRep(Repository rep, IMetaStore metaStore, ObjectId id_jobentry, List databases, + List slaveServers ) throws KettleException { + try { + port = rep.getJobEntryAttributeString( id_jobentry, "port" ); + serverName = rep.getJobEntryAttributeString( id_jobentry, "servername" ); + userName = rep.getJobEntryAttributeString( id_jobentry, "username" ); + password = + Encr.decryptPasswordOptionallyEncrypted( rep.getJobEntryAttributeString( id_jobentry, "password" ) ); + ftpDirectory = rep.getJobEntryAttributeString( id_jobentry, "ftpdirectory" ); + targetDirectory = rep.getJobEntryAttributeString( id_jobentry, "targetdirectory" ); + wildcard = rep.getJobEntryAttributeString( id_jobentry, "wildcard" ); + binaryMode = rep.getJobEntryAttributeBoolean( id_jobentry, "binary" ); + timeout = (int) rep.getJobEntryAttributeInteger( id_jobentry, "timeout" ); + remove = rep.getJobEntryAttributeBoolean( id_jobentry, "remove" ); + onlyGettingNewFiles = rep.getJobEntryAttributeBoolean( id_jobentry, "only_new" ); + activeConnection = rep.getJobEntryAttributeBoolean( id_jobentry, "active" ); + controlEncoding = rep.getJobEntryAttributeString( id_jobentry, "control_encoding" ); + if ( controlEncoding == null ) { + // if we couldn't retrieve an encoding, assume it's an old instance and + // put in the the encoding used before v 2.4.0 + controlEncoding = LEGACY_CONTROL_ENCODING; + } + + movefiles = rep.getJobEntryAttributeBoolean( id_jobentry, "movefiles" ); + movetodirectory = rep.getJobEntryAttributeString( id_jobentry, "movetodirectory" ); + + adddate = rep.getJobEntryAttributeBoolean( id_jobentry, "adddate" ); + addtime = rep.getJobEntryAttributeBoolean( id_jobentry, "addtime" ); + SpecifyFormat = rep.getJobEntryAttributeBoolean( id_jobentry, "SpecifyFormat" ); + date_time_format = rep.getJobEntryAttributeString( id_jobentry, "date_time_format" ); + AddDateBeforeExtension = rep.getJobEntryAttributeBoolean( id_jobentry, "AddDateBeforeExtension" ); + + String addToResult = rep.getJobEntryAttributeString( id_jobentry, "isaddresult" ); + if ( Utils.isEmpty( addToResult ) ) { + isaddresult = true; + } else { + isaddresult = rep.getJobEntryAttributeBoolean( id_jobentry, "isaddresult" ); + } + + createmovefolder = rep.getJobEntryAttributeBoolean( id_jobentry, "createmovefolder" ); + + proxyHost = rep.getJobEntryAttributeString( id_jobentry, "proxy_host" ); + proxyPort = rep.getJobEntryAttributeString( id_jobentry, "proxy_port" ); + proxyUsername = rep.getJobEntryAttributeString( id_jobentry, "proxy_username" ); + proxyPassword = + Encr.decryptPasswordOptionallyEncrypted( rep.getJobEntryAttributeString( id_jobentry, "proxy_password" ) ); + socksProxyHost = rep.getJobEntryAttributeString( id_jobentry, "socksproxy_host" ); + socksProxyPort = rep.getJobEntryAttributeString( id_jobentry, "socksproxy_port" ); + socksProxyUsername = rep.getJobEntryAttributeString( id_jobentry, "socksproxy_username" ); + socksProxyPassword = + Encr.decryptPasswordOptionallyEncrypted( rep.getJobEntryAttributeString( + id_jobentry, "socksproxy_password" ) ); + SifFileExists = rep.getJobEntryAttributeString( id_jobentry, "ifFileExists" ); + if ( Utils.isEmpty( SifFileExists ) ) { + ifFileExists = ifFileExistsSkip; + } else { + if ( SifFileExists.equals( SifFileExistsCreateUniq ) ) { + ifFileExists = ifFileExistsCreateUniq; + } else if ( SifFileExists.equals( SifFileExistsFail ) ) { + ifFileExists = ifFileExistsFail; + } else { + ifFileExists = ifFileExistsSkip; + } + } + nr_limit = rep.getJobEntryAttributeString( id_jobentry, "nr_limit" ); + success_condition = + Const.NVL( rep.getJobEntryAttributeString( id_jobentry, "success_condition" ), SUCCESS_IF_NO_ERRORS ); + + } catch ( KettleException dbe ) { + throw new KettleException( "Unable to load job entry of type 'ftp' from the repository for id_jobentry=" + + id_jobentry, dbe ); + } + } + + public void saveRep( Repository rep, IMetaStore metaStore, ObjectId id_job ) throws KettleException { + try { + rep.saveJobEntryAttribute( id_job, getObjectId(), "port", port ); + rep.saveJobEntryAttribute( id_job, getObjectId(), "servername", serverName ); + rep.saveJobEntryAttribute( id_job, getObjectId(), "username", userName ); + rep.saveJobEntryAttribute( id_job, getObjectId(), "password", Encr + .encryptPasswordIfNotUsingVariables( password ) ); + rep.saveJobEntryAttribute( id_job, getObjectId(), "ftpdirectory", ftpDirectory ); + rep.saveJobEntryAttribute( id_job, getObjectId(), "targetdirectory", targetDirectory ); + rep.saveJobEntryAttribute( id_job, getObjectId(), "wildcard", wildcard ); + rep.saveJobEntryAttribute( id_job, getObjectId(), "binary", binaryMode ); + rep.saveJobEntryAttribute( id_job, getObjectId(), "timeout", timeout ); + rep.saveJobEntryAttribute( id_job, getObjectId(), "remove", remove ); + rep.saveJobEntryAttribute( id_job, getObjectId(), "only_new", onlyGettingNewFiles ); + rep.saveJobEntryAttribute( id_job, getObjectId(), "active", activeConnection ); + rep.saveJobEntryAttribute( id_job, getObjectId(), "control_encoding", controlEncoding ); + + rep.saveJobEntryAttribute( id_job, getObjectId(), "movefiles", movefiles ); + rep.saveJobEntryAttribute( id_job, getObjectId(), "movetodirectory", movetodirectory ); + + rep.saveJobEntryAttribute( id_job, getObjectId(), "addtime", addtime ); + rep.saveJobEntryAttribute( id_job, getObjectId(), "adddate", adddate ); + rep.saveJobEntryAttribute( id_job, getObjectId(), "SpecifyFormat", SpecifyFormat ); + rep.saveJobEntryAttribute( id_job, getObjectId(), "date_time_format", date_time_format ); + + rep.saveJobEntryAttribute( id_job, getObjectId(), "AddDateBeforeExtension", AddDateBeforeExtension ); + rep.saveJobEntryAttribute( id_job, getObjectId(), "isaddresult", isaddresult ); + rep.saveJobEntryAttribute( id_job, getObjectId(), "createmovefolder", createmovefolder ); + + rep.saveJobEntryAttribute( id_job, getObjectId(), "proxy_host", proxyHost ); + rep.saveJobEntryAttribute( id_job, getObjectId(), "proxy_port", proxyPort ); + rep.saveJobEntryAttribute( id_job, getObjectId(), "proxy_username", proxyUsername ); + rep.saveJobEntryAttribute( id_job, getObjectId(), "proxy_password", Encr + .encryptPasswordIfNotUsingVariables( proxyPassword ) ); + rep.saveJobEntryAttribute( id_job, getObjectId(), "socksproxy_host", socksProxyHost ); + rep.saveJobEntryAttribute( id_job, getObjectId(), "socksproxy_port", socksProxyPort ); + rep.saveJobEntryAttribute( id_job, getObjectId(), "socksproxy_username", socksProxyUsername ); + rep.saveJobEntryAttribute( id_job, getObjectId(), "socksproxy_password", Encr + .encryptPasswordIfNotUsingVariables( socksProxyPassword ) ); + rep.saveJobEntryAttribute( id_job, getObjectId(), "ifFileExists", SifFileExists ); + + rep.saveJobEntryAttribute( id_job, getObjectId(), "nr_limit", nr_limit ); + rep.saveJobEntryAttribute( id_job, getObjectId(), "success_condition", success_condition ); + } catch ( KettleDatabaseException dbe ) { + throw new KettleException( + "Unable to save job entry of type 'ftp' to the repository for id_job=" + id_job, dbe ); + } + } + + public void setLimit( String nr_limitin ) { + this.nr_limit = nr_limitin; + } + + public String getLimit() { + return nr_limit; + } + + public void setSuccessCondition( String success_condition ) { + this.success_condition = success_condition; + } + + public String getSuccessCondition() { + return success_condition; + } + + public void setCreateMoveFolder( boolean createmovefolderin ) { + this.createmovefolder = createmovefolderin; + } + + public boolean isCreateMoveFolder() { + return createmovefolder; + } + + public void setAddDateBeforeExtension( boolean AddDateBeforeExtension ) { + this.AddDateBeforeExtension = AddDateBeforeExtension; + } + + public boolean isAddDateBeforeExtension() { + return AddDateBeforeExtension; + } + + public void setAddToResult( boolean isaddresultin ) { + this.isaddresult = isaddresultin; + } + + public boolean isAddToResult() { + return isaddresult; + } + + public void setDateInFilename( boolean adddate ) { + this.adddate = adddate; + } + + public boolean isDateInFilename() { + return adddate; + } + + public void setTimeInFilename( boolean addtime ) { + this.addtime = addtime; + } + + public boolean isTimeInFilename() { + return addtime; + } + + public boolean isSpecifyFormat() { + return SpecifyFormat; + } + + public void setSpecifyFormat( boolean SpecifyFormat ) { + this.SpecifyFormat = SpecifyFormat; + } + + public String getDateTimeFormat() { + return date_time_format; + } + + public void setDateTimeFormat( String date_time_format ) { + this.date_time_format = date_time_format; + } + + /** + * @return Returns the movefiles. + */ + public boolean isMoveFiles() { + return movefiles; + } + + /** + * @param movefilesin + * The movefiles to set. + */ + public void setMoveFiles( boolean movefilesin ) { + this.movefiles = movefilesin; + } + + /** + * @return Returns the movetodirectory. + */ + public String getMoveToDirectory() { + return movetodirectory; + } + + /** + * @param movetoin + * The movetodirectory to set. + */ + public void setMoveToDirectory( String movetoin ) { + this.movetodirectory = movetoin; + } + + /** + * @return Returns the binaryMode. + */ + public boolean isBinaryMode() { + return binaryMode; + } + + /** + * @param binaryMode + * The binaryMode to set. + */ + public void setBinaryMode( boolean binaryMode ) { + this.binaryMode = binaryMode; + } + + /** + * @return Returns the directory. + */ + public String getFtpDirectory() { + return ftpDirectory; + } + + /** + * @param directory + * The directory to set. + */ + public void setFtpDirectory( String directory ) { + this.ftpDirectory = directory; + } + + /** + * @return Returns the password. + */ + public String getPassword() { + return password; + } + + /** + * @param password + * The password to set. + */ + public void setPassword( String password ) { + this.password = password; + } + + /** + * @return Returns the serverName. + */ + public String getServerName() { + return serverName; + } + + /** + * @param serverName + * The serverName to set. + */ + public void setServerName( String serverName ) { + this.serverName = serverName; + } + + /** + * @return Returns the port. + */ + public String getPort() { + return port; + } + + /** + * @param port + * The port to set. + */ + public void setPort( String port ) { + this.port = port; + } + + /** + * @return Returns the userName. + */ + public String getUserName() { + return userName; + } + + /** + * @param userName + * The userName to set. + */ + public void setUserName( String userName ) { + this.userName = userName; + } + + /** + * @return Returns the wildcard. + */ + public String getWildcard() { + return wildcard; + } + + /** + * @param wildcard + * The wildcard to set. + */ + public void setWildcard( String wildcard ) { + this.wildcard = wildcard; + } + + /** + * @return Returns the targetDirectory. + */ + public String getTargetDirectory() { + return targetDirectory; + } + + /** + * @param targetDirectory + * The targetDirectory to set. + */ + public void setTargetDirectory( String targetDirectory ) { + this.targetDirectory = targetDirectory; + } + + /** + * @param timeout + * The timeout to set. + */ + public void setTimeout( int timeout ) { + this.timeout = timeout; + } + + /** + * @return Returns the timeout. + */ + public int getTimeout() { + return timeout; + } + + /** + * @param remove + * The remove to set. + */ + public void setRemove( boolean remove ) { + this.remove = remove; + } + + /** + * @return Returns the remove. + */ + public boolean getRemove() { + return remove; + } + + /** + * @return Returns the onlyGettingNewFiles. + */ + public boolean isOnlyGettingNewFiles() { + return onlyGettingNewFiles; + } + + /** + * @param onlyGettingNewFiles + * The onlyGettingNewFiles to set. + */ + public void setOnlyGettingNewFiles( boolean onlyGettingNewFilesin ) { + this.onlyGettingNewFiles = onlyGettingNewFilesin; + } + + /** + * Get the control encoding to be used for ftp'ing + * + * @return the used encoding + */ + public String getControlEncoding() { + return controlEncoding; + } + + /** + * Set the encoding to be used for ftp'ing. This determines how names are translated in dir e.g. It does impact the + * contents of the files being ftp'ed. + * + * @param encoding + * The encoding to be used. + */ + public void setControlEncoding( String encoding ) { + this.controlEncoding = encoding; + } + + /** + * @return Returns the hostname of the ftp-proxy. + */ + public String getProxyHost() { + return proxyHost; + } + + /** + * @param proxyHost + * The hostname of the proxy. + */ + public void setProxyHost( String proxyHost ) { + this.proxyHost = proxyHost; + } + + /** + * @param proxyPassword + * The password which is used to authenticate at the socks proxy. + */ + public void setProxyPassword( String proxyPassword ) { + this.proxyPassword = proxyPassword; + } + + /** + * @return Returns the password which is used to authenticate at the proxy. + */ + public String getProxyPassword() { + return proxyPassword; + } + + /** + * @param proxyPassword + * The password which is used to authenticate at the proxy. + */ + public void setSocksProxyPassword( String socksProxyPassword ) { + this.socksProxyPassword = socksProxyPassword; + } + + /** + * @return Returns the password which is used to authenticate at the socks proxy. + */ + public String getSocksProxyPassword() { + return socksProxyPassword; + } + + /** + * @param proxyPort + * The port of the ftp-proxy. + */ + public void setProxyPort( String proxyPort ) { + this.proxyPort = proxyPort; + } + + /** + * @return Returns the port of the ftp-proxy. + */ + public String getProxyPort() { + return proxyPort; + } + + /** + * @return Returns the username which is used to authenticate at the proxy. + */ + public String getProxyUsername() { + return proxyUsername; + } + + /** + * @param proxyUsername + * The username which is used to authenticate at the proxy. + */ + public void setProxyUsername( String proxyUsername ) { + this.proxyUsername = proxyUsername; + } + + /** + * @return Returns the username which is used to authenticate at the socks proxy. + */ + public String getSocksProxyUsername() { + return socksProxyUsername; + } + + /** + * @param proxyUsername + * The username which is used to authenticate at the socks proxy. + */ + public void setSocksProxyUsername( String socksPoxyUsername ) { + this.socksProxyUsername = socksPoxyUsername; + } + + /** + * + * @param socksProxyHost + * The host name of the socks proxy host + */ + public void setSocksProxyHost( String socksProxyHost ) { + this.socksProxyHost = socksProxyHost; + } + + /** + * @return The host name of the socks proxy host + */ + public String getSocksProxyHost() { + return this.socksProxyHost; + } + + /** + * @param socksProxyPort + * The port number the socks proxy host is using + */ + public void setSocksProxyPort( String socksProxyPort ) { + this.socksProxyPort = socksProxyPort; + } + + /** + * @return The port number the socks proxy host is using + */ + public String getSocksProxyPort() { + return this.socksProxyPort; + } + + protected FTPClient initFTPClient() { + return new FTPClient(); + } + + protected InetAddress getInetAddress(String realServername ) throws UnknownHostException { + return InetAddress.getByName( realServername ); + } + + /** + * + * @param previousResult + * @param nr + * @return + */ + public Result execute(Result previousResult, int nr ) { + log.logBasic( BaseMessages.getString( PKG, "JobEntryFTP.Started", serverName ) ); + + Result result = previousResult; + result.setNrErrors( 1 ); + result.setResult( false ); + NrErrors = 0; + NrfilesRetrieved = 0; + successConditionBroken = false; + boolean exitjobentry = false; + limitFiles = Const.toInt( environmentSubstitute( getLimit() ), 10 ); + + + if ( isDetailed() ) { + logDetailed( BaseMessages.getString( PKG, "JobEntryFtpPlus.Start" ) ); + } + + //ftp download + System.out.println("good news"); + + result.setNrErrors( NrErrors ); + result.setNrFilesRetrieved( NrfilesRetrieved ); + if ( getSuccessStatus() ) { + result.setResult( true ); + } + if ( exitjobentry ) { + result.setResult( false ); + } + displayResults(); + return result; + } + + private void downloadFile( FTPClient ftpclient, String filename, String realMoveToFolder, Job parentJob, + Result result ) throws Exception { + String localFilename = filename; + targetFilename = KettleVFS.getFilename( KettleVFS.getFileObject( returnTargetFilename( localFilename ) ) ); + + if ( ( !onlyGettingNewFiles ) || ( onlyGettingNewFiles && needsDownload( targetFilename ) ) ) { + if ( isDetailed() ) { + logDetailed( BaseMessages.getString( + PKG, "JobEntryFTP.GettingFile", filename, environmentSubstitute( targetDirectory ) ) ); + } + ftpclient.get( targetFilename, filename ); + + // Update retrieved files + updateRetrievedFiles(); + if ( isDetailed() ) { + logDetailed( BaseMessages.getString( PKG, "JobEntryFTP.GotFile", filename ) ); + } + + // Add filename to result filenames + addFilenameToResultFilenames( result, parentJob, targetFilename ); + + // Delete the file if this is needed! + if ( remove ) { + ftpclient.delete( filename ); + if ( isDetailed() ) { + if ( isDetailed() ) { + logDetailed( BaseMessages.getString( PKG, "JobEntryFTP.DeletedFile", filename ) ); + } + } + } else { + if ( movefiles ) { + // Try to move file to destination folder ... + ftpclient.rename( filename, realMoveToFolder + FILE_SEPARATOR + filename ); + + if ( isDetailed() ) { + logDetailed( BaseMessages.getString( PKG, "JobEntryFTP.MovedFile", filename, realMoveToFolder ) ); + } + } + } + } + } + + /** + * normalize / to \ and remove trailing slashes from a path + * + * @param path + * @return normalized path + * @throws Exception + */ + public String normalizePath( String path ) throws Exception { + + String normalizedPath = path.replaceAll( "\\\\", FILE_SEPARATOR ); + while ( normalizedPath.endsWith( "\\" ) || normalizedPath.endsWith( FILE_SEPARATOR ) ) { + normalizedPath = normalizedPath.substring( 0, normalizedPath.length() - 1 ); + } + + return normalizedPath; + } + + private void addFilenameToResultFilenames( Result result, Job parentJob, String filename ) throws KettleException { + if ( isaddresult ) { + FileObject targetFile = null; + try { + targetFile = KettleVFS.getFileObject( filename, this ); + + // Add to the result files... + ResultFile resultFile = + new ResultFile( ResultFile.FILE_TYPE_GENERAL, targetFile, parentJob.getJobname(), toString() ); + resultFile.setComment( BaseMessages.getString( PKG, "JobEntryFTP.Downloaded", serverName ) ); + result.getResultFiles().put( resultFile.getFile().toString(), resultFile ); + + if ( isDetailed() ) { + logDetailed( BaseMessages.getString( PKG, "JobEntryFTP.FileAddedToResult", filename ) ); + } + } catch ( Exception e ) { + throw new KettleException( e ); + } finally { + try { + targetFile.close(); + targetFile = null; + } catch ( Exception e ) { + // Ignore close errors + } + } + } + } + + private void displayResults() { + if ( isDetailed() ) { + logDetailed( "=======================================" ); + logDetailed( BaseMessages.getString( PKG, "JobEntryFTP.Log.Info.FilesInError", "" + NrErrors ) ); + logDetailed( BaseMessages.getString( PKG, "JobEntryFTP.Log.Info.FilesRetrieved", "" + NrfilesRetrieved ) ); + logDetailed( "=======================================" ); + } + } + + private boolean getSuccessStatus() { + boolean retval = false; + + if ( ( NrErrors == 0 && getSuccessCondition().equals( SUCCESS_IF_NO_ERRORS ) ) + || ( NrfilesRetrieved >= limitFiles && getSuccessCondition().equals( + SUCCESS_IF_AT_LEAST_X_FILES_DOWNLOADED ) ) + || ( NrErrors <= limitFiles && getSuccessCondition().equals( SUCCESS_IF_ERRORS_LESS ) ) ) { + retval = true; + } + + return retval; + } + + private void updateErrors() { + NrErrors++; + if ( checkIfSuccessConditionBroken() ) { + // Success condition was broken + successConditionBroken = true; + } + } + + private boolean checkIfSuccessConditionBroken() { + boolean retval = false; + if ( ( NrErrors > 0 && getSuccessCondition().equals( SUCCESS_IF_NO_ERRORS ) ) + || ( NrErrors >= limitFiles && getSuccessCondition().equals( SUCCESS_IF_ERRORS_LESS ) ) ) { + retval = true; + } + return retval; + } + + private void updateRetrievedFiles() { + NrfilesRetrieved++; + } + + /** + * @param string + * the filename from the FTP server + * + * @return the calculated target filename + */ + @VisibleForTesting + String returnTargetFilename( String filename ) { + String retval = null; + // Replace possible environment variables... + if ( filename != null ) { + retval = filename; + } else { + return null; + } + + int lenstring = retval.length(); + int lastindexOfDot = retval.lastIndexOf( "." ); + if ( lastindexOfDot == -1 ) { + lastindexOfDot = lenstring; + } + + String fileExtension = retval.substring( lastindexOfDot, lenstring ); + + if ( isAddDateBeforeExtension() ) { + retval = retval.substring( 0, lastindexOfDot ); + } + + SimpleDateFormat daf = new SimpleDateFormat(); + Date now = new Date(); + + if ( SpecifyFormat && !Utils.isEmpty( date_time_format ) ) { + daf.applyPattern( date_time_format ); + String dt = daf.format( now ); + retval += dt; + } else { + if ( adddate ) { + daf.applyPattern( "yyyyMMdd" ); + String d = daf.format( now ); + retval += "_" + d; + } + if ( addtime ) { + daf.applyPattern( "HHmmssSSS" ); + String t = daf.format( now ); + retval += "_" + t; + } + } + + if ( isAddDateBeforeExtension() ) { + retval += fileExtension; + } + + // Add foldername to filename + retval = environmentSubstitute( targetDirectory ) + Const.FILE_SEPARATOR + retval; + return retval; + } + + public boolean evaluates() { + return true; + } + + /** + * See if the filename on the FTP server needs downloading. The default is to check the presence of the file in the + * target directory. If you need other functionality, extend this class and build it into a plugin. + * + * @param filename + * The local filename to check + * @param remoteFileSize + * The size of the remote file + * @return true if the file needs downloading + */ + protected boolean needsDownload( String filename ) { + boolean retval = false; + + File file = new File( filename ); + + if ( !file.exists() ) { + // Local file not exists! + if ( isDebug() ) { + logDebug( BaseMessages.getString( PKG, "JobEntryFTP.LocalFileNotExists" ), filename ); + } + return true; + } else { + + // Local file exists! + if ( ifFileExists == ifFileExistsCreateUniq ) { + if ( isDebug() ) { + logDebug( toString(), BaseMessages.getString( PKG, "JobEntryFTP.LocalFileExists" ), filename ); + // Create file with unique name + } + + int lenstring = targetFilename.length(); + int lastindexOfDot = targetFilename.lastIndexOf( '.' ); + if ( lastindexOfDot == -1 ) { + lastindexOfDot = lenstring; + } + + targetFilename = + targetFilename.substring( 0, lastindexOfDot ) + + StringUtil.getFormattedDateTimeNow( true ) + + targetFilename.substring( lastindexOfDot, lenstring ); + + return true; + } else if ( ifFileExists == ifFileExistsFail ) { + log.logError( BaseMessages.getString( PKG, "JobEntryFTP.LocalFileExists" ), filename ); + updateErrors(); + } else { + if ( isDebug() ) { + logDebug( toString(), BaseMessages.getString( PKG, "JobEntryFTP.LocalFileExists" ), filename ); + } + } + } + + return retval; + } + + /** + * @return the activeConnection + */ + public boolean isActiveConnection() { + return activeConnection; + } + + /** + * @param activeConnection + * the activeConnection to set + */ + public void setActiveConnection( boolean passive ) { + this.activeConnection = passive; + } + + public void check(List remarks, JobMeta jobMeta, VariableSpace space, + Repository repository, IMetaStore metaStore ) { + JobEntryValidatorUtils.andValidator().validate( this, "serverName", remarks, + AndValidator.putValidators( JobEntryValidatorUtils.notBlankValidator() ) ); + JobEntryValidatorUtils.andValidator().validate( + this, "targetDirectory", remarks, AndValidator.putValidators( JobEntryValidatorUtils.notBlankValidator(), + JobEntryValidatorUtils.fileExistsValidator() ) ); + JobEntryValidatorUtils.andValidator().validate( this, "userName", remarks, + AndValidator.putValidators( JobEntryValidatorUtils.notBlankValidator() ) ); + JobEntryValidatorUtils.andValidator().validate( this, "password", remarks, + AndValidator.putValidators( JobEntryValidatorUtils.notNullValidator() ) ); + } + + public List getResourceDependencies(JobMeta jobMeta ) { + List references = super.getResourceDependencies( jobMeta ); + if ( !Utils.isEmpty( serverName ) ) { + String realServername = jobMeta.environmentSubstitute( serverName ); + ResourceReference reference = new ResourceReference( this ); + reference.getEntries().add( new ResourceEntry( realServername, ResourceEntry.ResourceType.SERVER ) ); + references.add( reference ); + } + return references; + } + + /** + * Hook in known parsers, and then those that have been specified in the variable ftp.file.parser.class.names + * + * @param ftpClient + * @throws FTPException + * @throws IOException + */ + protected void hookInOtherParsers( FTPClient ftpClient ) throws FTPException, IOException { + if ( log.isDebug() ) { + logDebug( BaseMessages.getString( PKG, "JobEntryFTP.DEBUG.Hooking.Parsers" ) ); + } + String system = ftpClient.system(); + MVSFileParser parser = new MVSFileParser( log ); + if ( log.isDebug() ) { + logDebug( BaseMessages.getString( PKG, "JobEntryFTP.DEBUG.Created.MVS.Parser" ) ); + } + FTPFileFactory factory = new FTPFileFactory( system ); + if ( log.isDebug() ) { + logDebug( BaseMessages.getString( PKG, "JobEntryFTP.DEBUG.Created.Factory" ) ); + } + factory.addParser( parser ); + ftpClient.setFTPFileFactory( factory ); + if ( log.isDebug() ) { + logDebug( BaseMessages.getString( PKG, "JobEntryFTP.DEBUG.Get.Variable.Space" ) ); + } + VariableSpace vs = this.getVariables(); + if ( vs != null ) { + if ( log.isDebug() ) { + logDebug( BaseMessages.getString( PKG, "JobEntryFTP.DEBUG.Getting.Other.Parsers" ) ); + } + String otherParserNames = vs.getVariable( "ftp.file.parser.class.names" ); + if ( otherParserNames != null ) { + if ( log.isDebug() ) { + logDebug( BaseMessages.getString( PKG, "JobEntryFTP.DEBUG.Creating.Parsers" ) ); + } + String[] parserClasses = otherParserNames.split( "|" ); + String cName = null; + Class clazz = null; + Object parserInstance = null; + for ( int i = 0; i < parserClasses.length; i++ ) { + cName = parserClasses[i].trim(); + if ( cName.length() > 0 ) { + try { + clazz = Class.forName( cName ); + parserInstance = clazz.newInstance(); + if ( parserInstance instanceof FTPFileParser ) { + if ( log.isDetailed() ) { + logDetailed( BaseMessages.getString( PKG, "JobEntryFTP.DEBUG.Created.Other.Parser", cName ) ); + } + factory.addParser( (FTPFileParser) parserInstance ); + } + } catch ( Exception ignored ) { + if ( log.isDebug() ) { + ignored.printStackTrace(); + logError( BaseMessages.getString( PKG, "JobEntryFTP.ERROR.Creating.Parser", cName ) ); + } + } + } + } + } + } + } +} diff --git a/kettle-sdk-jobentry-plugin/pom.xml b/kettle-sdk-jobentry-plugin/pom.xml index f97318da8..617e18ae1 100644 --- a/kettle-sdk-jobentry-plugin/pom.xml +++ b/kettle-sdk-jobentry-plugin/pom.xml @@ -63,6 +63,28 @@ ${mockito.version} test + + + + com.alibaba + fastjson + 1.2.47 + + + + commons-net + commons-net + 3.6 + + + + org.projectlombok + lombok + 1.18.2 + provided + + + diff --git a/kettle-sdk-jobentry-plugin/src/main/java/org/pentaho/di/sdk/myplugins/jobentries/ftpplus/JobEntryFtpPlus.java b/kettle-sdk-jobentry-plugin/src/main/java/org/pentaho/di/sdk/myplugins/jobentries/ftpplus/JobEntryFtpPlus.java new file mode 100644 index 000000000..cd3d7a472 --- /dev/null +++ b/kettle-sdk-jobentry-plugin/src/main/java/org/pentaho/di/sdk/myplugins/jobentries/ftpplus/JobEntryFtpPlus.java @@ -0,0 +1,102 @@ +package org.pentaho.di.sdk.myplugins.jobentries.ftpplus; + +import org.apache.commons.net.ftp.FTPClient; +import org.apache.commons.net.ftp.FTPClientConfig; +import org.apache.commons.net.ftp.FTPReply; +import org.pentaho.di.cluster.SlaveServer; +import org.pentaho.di.core.Result; +import org.pentaho.di.core.annotations.JobEntry; +import org.pentaho.di.core.database.DatabaseMeta; +import org.pentaho.di.core.exception.KettleException; +import org.pentaho.di.core.exception.KettleXMLException; +import org.pentaho.di.core.xml.XMLHandler; +import org.pentaho.di.i18n.BaseMessages; +import org.pentaho.di.job.entry.JobEntryBase; +import org.pentaho.di.job.entry.JobEntryInterface; +import org.pentaho.di.repository.Repository; +import org.pentaho.metastore.api.IMetaStore; +import org.w3c.dom.Node; + +import java.io.IOException; +import java.net.SocketException; +import java.util.List; + +/** + * @author shepf + */ +@JobEntry( + id = "JobEntryFtpPlus", + name = "JobEntryFtpPlus.Name", + description = "JobEntryFtpPlus.TooltipDesc", + categoryDescription = "i18n:org.pentaho.di.job:JobCategory.Category.FileTransfer", + i18nPackageName = "org.pentaho.di.sdk.myplugins.jobentries.ftpplus", + documentationUrl = "JobEntryFtpPlus.DocumentationURL", + casesUrl = "JobEntryFtpPlus.CasesURL", + forumUrl = "JobEntryFtpPlus.ForumURL" +) +public class JobEntryFtpPlus extends JobEntryBase implements Cloneable, JobEntryInterface { + + private String jsonConfStr = "{}"; + private String className = "JobEntryFtpPlus"; + + /** + * for i18n + */ + private static Class PKG = JobEntryFtpPlus.class; + + public String getJsonConfStr() { + return jsonConfStr; + } + + public void setJsonConfStr(String jsonConfStr) { + this.jsonConfStr = jsonConfStr; + } + + + @Override + public Result execute(Result prev_result, int nr) throws KettleException { + + //TODO ftp业务代码 + + // indicate there are no errors + prev_result.setNrErrors( 0 ); + // indicate the result as configured + prev_result.setResult( true ); + return prev_result; + } + + + @Override + public String getXML() { + StringBuffer retval = new StringBuffer(); + + retval.append(super.getXML()); + retval.append(" ").append( + XMLHandler.addTagValue("configInfo", jsonConfStr)); + retval.append(" ").append( + XMLHandler.addTagValue("className", className)); + + return retval.toString(); + } + + @Override + public void loadXML(Node entrynode, List databases, + List slaveServers, Repository rep, IMetaStore metaStore) + throws KettleXMLException { + try { + super.loadXML(entrynode, databases, slaveServers); + jsonConfStr = XMLHandler.getTagValue(entrynode, "configInfo"); + + + + className = XMLHandler.getTagValue(entrynode, "className"); + } catch (Exception e) { + throw new KettleXMLException(BaseMessages.getString(PKG, + "JobEntryKettleUtil.UnableToLoadFromXml"), e); + } + } + + + + +} diff --git a/kettle-sdk-jobentry-plugin/src/main/java/org/pentaho/di/sdk/myplugins/jobentries/ftpplus/JobEntryFtpPlusDialog.java b/kettle-sdk-jobentry-plugin/src/main/java/org/pentaho/di/sdk/myplugins/jobentries/ftpplus/JobEntryFtpPlusDialog.java new file mode 100644 index 000000000..58d4839f4 --- /dev/null +++ b/kettle-sdk-jobentry-plugin/src/main/java/org/pentaho/di/sdk/myplugins/jobentries/ftpplus/JobEntryFtpPlusDialog.java @@ -0,0 +1,361 @@ +package org.pentaho.di.sdk.myplugins.jobentries.ftpplus; + +import org.apache.commons.lang.StringUtils; +import org.eclipse.swt.SWT; +import org.eclipse.swt.events.*; +import org.eclipse.swt.layout.FormAttachment; +import org.eclipse.swt.layout.FormData; +import org.eclipse.swt.layout.FormLayout; +import org.eclipse.swt.widgets.*; +import org.pentaho.di.core.Const; +import org.pentaho.di.core.Props; +import org.pentaho.di.core.annotations.PluginDialog; +import org.pentaho.di.i18n.BaseMessages; +import org.pentaho.di.job.JobMeta; +import org.pentaho.di.job.entry.JobEntryDialogInterface; +import org.pentaho.di.job.entry.JobEntryInterface; +import org.pentaho.di.repository.Repository; +import org.pentaho.di.ui.core.gui.WindowProperty; +import org.pentaho.di.ui.core.widget.StyledTextComp; +import org.pentaho.di.ui.core.widget.TextVar; +import org.pentaho.di.ui.job.dialog.JobDialog; +import org.pentaho.di.ui.job.entry.JobEntryDialog; +import org.pentaho.di.ui.trans.step.BaseStepDialog; + +/** + * @author shepf + */ +@PluginDialog(id = "JobEntryFtpPlus", + image = "org/pentaho/di/sdk/myplugins/jobentries/ftpplus/resources/demo.svg", + pluginType = PluginDialog.PluginType.JOBENTRY) +public class JobEntryFtpPlusDialog extends JobEntryDialog implements JobEntryDialogInterface { + + // for i18n purposes + private static Class PKG = JobEntryFtpPlus.class; + // the job entry configuration object + private JobEntryFtpPlus jobEntry; + + //配合界面 + private Label wlName; + private Text wText; + private FormData fdlName, fdName; + private Label wlConfigInfo; + private StyledTextComp wConfigInfo; + private FormData fdlConfigInfo, fdConfigInfo; + private Label wlPosition; + private FormData fdlPosition; + private Button wOK, wGet, wCancel; + private Listener lsOK, lsGet, lsCancel; + private Shell shell; + private SelectionAdapter lsDef; + private boolean changed; + + + /** + * 配置名称 + */ + private TextVar wClassName; + private Label wlClassName; + private FormData fdlClassName, fdClassName; + + /** + * Instantiates a new job entry dialog. + * + * @param parent the parent shell + * @param jobEntryInt the job entry interface + * @param rep the repository + * @param jobMeta + */ + public JobEntryFtpPlusDialog(Shell parent, JobEntryInterface jobEntryInt, Repository rep, JobMeta jobMeta) { + super(parent, jobEntryInt, rep, jobMeta); + // it is safe to cast the JobEntryInterface object to the object handled by this dialog + jobEntry = (JobEntryFtpPlus) jobEntryInt; + // ensure there is a default name for new job entries + if (this.jobEntry.getName() == null) { + this.jobEntry.setName(BaseMessages.getString(PKG, "FtpPlus.Default.Name")); + } + } + + @Override + public JobEntryInterface open() { + Shell parent = getParent(); + Display display = parent.getDisplay(); + + shell = new Shell(parent, props.getJobsDialogStyle()); + props.setLook(shell); + JobDialog.setShellImage(shell, jobEntry); + + ModifyListener lsMod = new ModifyListener() { + @Override + public void modifyText(ModifyEvent e) { + jobEntry.setChanged(); + } + }; + changed = jobEntry.hasChanged(); + + FormLayout formLayout = new FormLayout(); + formLayout.marginWidth = Const.FORM_MARGIN; + formLayout.marginHeight = Const.FORM_MARGIN; + + shell.setLayout(formLayout); + shell.setText(BaseMessages.getString(PKG, "JobEntryKettleUtil.Title")); + + int middle = props.getMiddlePct(); + int margin = Const.MARGIN; + + wGet = new Button(shell, SWT.PUSH); + wGet.setText("获取默认配置"); + wGet.setToolTipText("在输入类名称后再通过此按钮获取对应默认配置信息"); + + wOK = new Button(shell, SWT.PUSH); + wOK.setText(BaseMessages.getString(PKG, "System.Button.OK")); + wCancel = new Button(shell, SWT.PUSH); + wCancel.setText(BaseMessages.getString(PKG, "System.Button.Cancel")); + + // at the bottom + BaseStepDialog.positionBottomButtons(shell, new Button[]{wOK, wCancel, wGet}, margin, null); + + // Filename line + wlName = new Label(shell, SWT.RIGHT); + wlName.setText(BaseMessages.getString(PKG, "JobEntryKettleUtil.Jobname.Label")); + props.setLook(wlName); + fdlName = new FormData(); + fdlName.left = new FormAttachment(0, 0); + fdlName.right = new FormAttachment(middle, -margin); + fdlName.top = new FormAttachment(0, margin); + wlName.setLayoutData(fdlName); + wText = new Text(shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + props.setLook(wText); + wText.addModifyListener(lsMod); + fdName = new FormData(); + fdName.left = new FormAttachment(middle, 0); + fdName.top = new FormAttachment(0, margin); + fdName.right = new FormAttachment(100, 0); + wText.setLayoutData(fdName); + + wlClassName = new Label(shell, SWT.RIGHT); + wlClassName.setText(BaseMessages.getString(PKG, "JobEntryKettleUtil.ClassName.Label") + " "); + props.setLook(wlClassName); + fdlClassName = new FormData(); + fdlClassName.left = new FormAttachment(0, 0); + fdlClassName.right = new FormAttachment(middle, -margin); + fdlClassName.top = new FormAttachment(wText, margin); + wlClassName.setLayoutData(fdlClassName); + + wClassName = new TextVar(jobEntry, shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + props.setLook(wClassName); + wClassName.addModifyListener(lsMod); + fdClassName = new FormData(); + fdClassName.left = new FormAttachment(middle, 0); + fdClassName.top = new FormAttachment(wText, margin); + fdClassName.right = new FormAttachment(100, margin); + wClassName.setLayoutData(fdClassName); + + wlPosition = new Label(shell, SWT.NONE); + wlPosition.setText(BaseMessages.getString(PKG, "JobEntryKettleUtil.LineNr.Label", "0")); + props.setLook(wlPosition); + fdlPosition = new FormData(); + fdlPosition.left = new FormAttachment(0, 0); + fdlPosition.bottom = new FormAttachment(wOK, -margin); + wlPosition.setLayoutData(fdlPosition); + + // Script line + wlConfigInfo = new Label(shell, SWT.NONE); + wlConfigInfo.setText(BaseMessages.getString(PKG, "JobEntryKettleUtil.Script.Label")); + props.setLook(wlConfigInfo); + fdlConfigInfo = new FormData(); + fdlConfigInfo.left = new FormAttachment(0, 0); + fdlConfigInfo.top = new FormAttachment(wClassName, margin); + wlConfigInfo.setLayoutData(fdlConfigInfo); + wConfigInfo = + new StyledTextComp(jobEntry, shell, SWT.MULTI | SWT.LEFT | SWT.BORDER | SWT.H_SCROLL | SWT.V_SCROLL, ""); + wConfigInfo.setText(BaseMessages.getString(PKG, "JobEntryKettleUtil.Script.Default")); + props.setLook(wConfigInfo, Props.WIDGET_STYLE_FIXED); + wConfigInfo.addModifyListener(lsMod); + fdConfigInfo = new FormData(); + fdConfigInfo.left = new FormAttachment(0, 0); + fdConfigInfo.top = new FormAttachment(wlConfigInfo, margin); + fdConfigInfo.right = new FormAttachment(100, -10); + fdConfigInfo.bottom = new FormAttachment(wlPosition, -margin); + wConfigInfo.setLayoutData(fdConfigInfo); + wConfigInfo.addModifyListener(new ModifyListener() { + @Override + public void modifyText(ModifyEvent arg0) { + setPosition(); + } + + }); + + wConfigInfo.addKeyListener(new KeyAdapter() { + @Override + public void keyPressed(KeyEvent e) { + setPosition(); + } + + @Override + public void keyReleased(KeyEvent e) { + setPosition(); + } + }); + wConfigInfo.addFocusListener(new FocusAdapter() { + @Override + public void focusGained(FocusEvent e) { + setPosition(); + } + + @Override + public void focusLost(FocusEvent e) { + setPosition(); + } + }); + wConfigInfo.addMouseListener(new MouseAdapter() { + @Override + public void mouseDoubleClick(MouseEvent e) { + setPosition(); + } + + @Override + public void mouseDown(MouseEvent e) { + setPosition(); + } + + @Override + public void mouseUp(MouseEvent e) { + setPosition(); + } + }); + wConfigInfo.addModifyListener(lsMod); + // Add listeners + lsCancel = new Listener() { + @Override + public void handleEvent(Event e) { + cancel(); + } + }; + lsOK = new Listener() { + @Override + public void handleEvent(Event e) { + ok(); + } + }; + lsGet = new Listener() { + @Override + public void handleEvent(Event e) { +// jobEntry.setClassName(wClassName.getText()); +// jobEntry.setConfigInfo(wConfigInfo.getText()); + String conf = null; + String msg = "获取默认配置失败，请输入正确的类名称"; + try { + //conf = jobEntry.getDefaultConfigInfo(); + } catch (Exception e1) { + msg = e1.getMessage(); + } + if (StringUtils.isBlank(conf)) { + wConfigInfo.setText("{}"); + MessageBox mb = new MessageBox(shell, SWT.OK | SWT.ICON_ERROR); + mb.setMessage(msg); + mb.setText("错误"); + mb.open(); + } else { + wConfigInfo.setText(conf); + } + } + }; + + wCancel.addListener(SWT.Selection, lsCancel); + wOK.addListener(SWT.Selection, lsOK); + wGet.addListener(SWT.Selection, lsGet); + + lsDef = new SelectionAdapter() { + @Override + public void widgetDefaultSelected(SelectionEvent e) { + ok(); + } + }; + + wText.addSelectionListener(lsDef); + + // Detect X or ALT-F4 or something that kills this window... + shell.addShellListener(new ShellAdapter() { + @Override + public void shellClosed(ShellEvent e) { + cancel(); + } + }); + + getData(); + + BaseStepDialog.setSize(shell, 250, 250, false); + + shell.open(); + props.setDialogSize(shell, "JobEvalDialogSize"); + while (!shell.isDisposed()) { + if (!display.readAndDispatch()) { + display.sleep(); + } + } + return jobEntry; + } + + public void setPosition() { + + String scr = wConfigInfo.getText(); + int linenr = wConfigInfo.getLineAtOffset(wConfigInfo.getCaretOffset()) + 1; + int posnr = wConfigInfo.getCaretOffset(); + + // Go back from position to last CR: how many positions? + int colnr = 0; + while (posnr > 0 && scr.charAt(posnr - 1) != '\n' && scr.charAt(posnr - 1) != '\r') { + posnr--; + colnr++; + } + wlPosition.setText(BaseMessages.getString(PKG, "JobEntryKettleUtil.Position.Label", "" + linenr, "" + colnr)); + + } + + public void dispose() { + WindowProperty winprop = new WindowProperty(shell); + props.setScreen(winprop); + shell.dispose(); + } + + /** + * Copy information from the meta-data input to the dialog fields. + */ + public void getData() { + if (jobEntry.getName() != null) { + wText.setText(jobEntry.getName()); + } +// if ( jobEntry.getClassName() != null ) { +// wClassName.setText( jobEntry.getClassName() ); +// } +// if ( jobEntry.getConfigInfo() != null ) { +// wConfigInfo.setText( jobEntry.getConfigInfo() ); +// } + + wText.selectAll(); + wText.setFocus(); + } + + private void cancel() { + jobEntry.setChanged(changed); + jobEntry = null; + dispose(); + } + + private void ok() { + if (Const.isEmpty(wText.getText())) { + MessageBox mb = new MessageBox(shell, SWT.OK | SWT.ICON_ERROR); + mb.setText(BaseMessages.getString(PKG, "System.StepJobEntryNameMissing.Title")); + mb.setMessage(BaseMessages.getString(PKG, "System.JobEntryNameMissing.Msg")); + mb.open(); + return; + } + + jobEntry.setJsonConfStr(wText.getText()); + //jobEntry.setConfigInfo( wConfigInfo.getText() ); + //jobEntry.setClassName( wClassName.getText() ); + dispose(); + } + +} diff --git a/kettle-sdk-jobentry-plugin/src/main/java/org/pentaho/di/sdk/myplugins/jobentries/ftpplus/JobEntryFtpPlusParamsDO.java b/kettle-sdk-jobentry-plugin/src/main/java/org/pentaho/di/sdk/myplugins/jobentries/ftpplus/JobEntryFtpPlusParamsDO.java new file mode 100644 index 000000000..f695a0965 --- /dev/null +++ b/kettle-sdk-jobentry-plugin/src/main/java/org/pentaho/di/sdk/myplugins/jobentries/ftpplus/JobEntryFtpPlusParamsDO.java @@ -0,0 +1,68 @@ +package org.pentaho.di.sdk.myplugins.jobentries.ftpplus; + + + + +/** + * + */ +public class JobEntryFtpPlusParamsDO { + private String serverName; + private String userName; + private String password; + private String ftpDirectory; + private String targetDirectory; + private String wildcard; + private boolean binaryMode; + private int timeout; + private boolean remove; + private boolean onlyGettingNewFiles; /* Don't overwrite files */ + private boolean activeConnection; + private String controlEncoding; /* how to convert list of filenames e.g. */ + /** + * Implicit encoding used before PDI v2.4.1 + */ + private static String LEGACY_CONTROL_ENCODING = "US-ASCII"; + /** + * Default encoding when making a new ftp job entry instance. + */ + private static String DEFAULT_CONTROL_ENCODING = "ISO-8859-1"; + private boolean movefiles; + private String movetodirectory; + private boolean adddate; + private boolean addtime; + private boolean SpecifyFormat; + private String date_time_format; + private boolean AddDateBeforeExtension; + private boolean isaddresult; + private boolean createmovefolder; + private String port; + private String proxyHost; + private String proxyPort; /* string to allow variable substitution */ + private String proxyUsername; + private String proxyPassword; + private String socksProxyHost; + private String socksProxyPort; + private String socksProxyUsername; + private String socksProxyPassword; + public int ifFileExistsSkip = 0; + public String SifFileExistsSkip = "ifFileExistsSkip"; + public int ifFileExistsCreateUniq = 1; + public String SifFileExistsCreateUniq = "ifFileExistsCreateUniq"; + public int ifFileExistsFail = 2; + public String SifFileExistsFail = "ifFileExistsFail"; + public int ifFileExists; + public String SifFileExists; + public String SUCCESS_IF_AT_LEAST_X_FILES_DOWNLOADED = "success_when_at_least"; + public String SUCCESS_IF_ERRORS_LESS = "success_if_errors_less"; + public String SUCCESS_IF_NO_ERRORS = "success_if_no_errors"; + private String nr_limit; + private String success_condition; + long NrErrors = 0; + long NrfilesRetrieved = 0; + boolean successConditionBroken = false; + int limitFiles = 0; + String targetFilename = null; + static String FILE_SEPARATOR = "/"; + +} diff --git a/kettle-sdk-jobentry-plugin/src/main/java/org/pentaho/di/sdk/samples/jobentries/demo/JobEntryDemo.java b/kettle-sdk-jobentry-plugin/src/main/java/org/pentaho/di/sdk/samples/jobentries/demo/JobEntryDemo.java index fa49c81de..3fc5a6ec5 100644 --- a/kettle-sdk-jobentry-plugin/src/main/java/org/pentaho/di/sdk/samples/jobentries/demo/JobEntryDemo.java +++ b/kettle-sdk-jobentry-plugin/src/main/java/org/pentaho/di/sdk/samples/jobentries/demo/JobEntryDemo.java @@ -58,16 +58,16 @@ * */ @JobEntry( - id = "DemoJobEntry", - name = "DemoJobEntry.Name", - description = "DemoJobEntry.TooltipDesc", - image = "org/pentaho/di/sdk/samples/jobentries/demo/resources/demo.svg", - categoryDescription = "i18n:org.pentaho.di.job:JobCategory.Category.Conditions", - i18nPackageName = "org.pentaho.di.sdk.samples.jobentries.demo", - documentationUrl = "DemoJobEntry.DocumentationURL", - casesUrl = "DemoJobEntry.CasesURL", - forumUrl = "DemoJobEntry.ForumURL" - ) + id = "DemoJobEntry", + name = "DemoJobEntry.Name", + description = "DemoJobEntry.TooltipDesc", + image = "org/pentaho/di/sdk/samples/jobentries/demo/resources/demo.svg", + categoryDescription = "i18n:org.pentaho.di.job:JobCategory.Category.Conditions", + i18nPackageName = "org.pentaho.di.sdk.samples.jobentries.demo", + documentationUrl = "DemoJobEntry.DocumentationURL", + casesUrl = "DemoJobEntry.CasesURL", + forumUrl = "DemoJobEntry.ForumURL" +) public class JobEntryDemo extends JobEntryBase implements Cloneable, JobEntryInterface { /** diff --git a/kettle-sdk-jobentry-plugin/src/main/resources/org/pentaho/di/sdk/myplugins/jobentries/ftpplus/messages/messages_en_US.properties b/kettle-sdk-jobentry-plugin/src/main/resources/org/pentaho/di/sdk/myplugins/jobentries/ftpplus/messages/messages_en_US.properties new file mode 100644 index 000000000..16e29f932 --- /dev/null +++ b/kettle-sdk-jobentry-plugin/src/main/resources/org/pentaho/di/sdk/myplugins/jobentries/ftpplus/messages/messages_en_US.properties @@ -0,0 +1,17 @@ +#ý +JobEntryKettleUtil.Title=JsonConfJobEntryDialog +JobEntryKettleUtil.Jobname.Label=JobName +JobEntryKettleUtil.ClassName.Label=JobEntryClassName +JobEntryKettleUtil.Script.Label=JsonConfigure + +JobEntryKettleUtil.LineNr.Label=LineNr: {0} +JobEntryKettleUtil.Script.Default=please input json String +JobEntryKettleUtil.Position.Label=x {0} y {1} +FtpPlus.Default.Name=FtpPlusDialog + +# +JobEntryFtpPlus.Name=FtpPlus +JobEntryFtpPlus.TooltipDesc=Demo FtpPlusDialog Job Entry +JobEntryFtpPlus.DocumentationURL=https://help.pentaho.com/Documentation/6.1/0R0/0V0 +JobEntryFtpPlus.CasesURL=http://jira.pentaho.com/browse/PDI/ +JobEntryFtpPlus.ForumURL=http://forums.pentaho.com/ \ No newline at end of file diff --git a/kettle-sdk-jobentry-plugin/src/main/resources/org/pentaho/di/sdk/myplugins/jobentries/ftpplus/resources/demo.png b/kettle-sdk-jobentry-plugin/src/main/resources/org/pentaho/di/sdk/myplugins/jobentries/ftpplus/resources/demo.png new file mode 100644 index 0000000000000000000000000000000000000000..841591051f3a8d4a8c517f2d9277e353379a6cf6 GIT binary patch literal 916 zcmeAS@N?(olHy`uVBq!ia0vp^3LwnE3?yBabR7dyEa{HEjtmSN`?>!lvI6;x#X;^) z4C~IxyacIC_6YK2V5m}KU}$JzVE6?TYIwoGP-?)y@G60U!D^WWPae{X(zy7Sqg!>`Wq zY~nZDXF2EY;@3ZZ>Fm}w*=IiQ-jZXlE*^SyHsVAK?_3&&qwiaV8b;`QYZ-+xC|Pnx~yz>$l$o<4s$Vd<7@cOJZX`)>1rQ*YnB z%jukZ<<|Z4*X~@s{ovK>H}BuSfB5A2t^1GCT4ojZEjo7o+Tk--dKYavbLG~`okyH9 zntcj7?NS?d9zExn-sG9vUN?Q^od-|cbJ}_qZQQu;#HAZ|w;VjZ?eLk`Z{F@YcFrZU z<<*|q!Wn%ftrq{r89Q2*OxeOC#~Kr}spwD8Ck0lnt{Wm6nyWvo zUbT|hQcB~K0qd-&KewzDGWB9@*>bqP@bW17xlC`@FfcM}{B=}#<0|OI_C*3F-a`RI% Y(<*Uma7~Rd2WnvOboFyt=akR{06|iy;Q#;t literal 0 HcmV?d00001 diff --git a/kettle-sdk-jobentry-plugin/src/main/resources/org/pentaho/di/sdk/myplugins/jobentries/ftpplus/resources/demo.svg b/kettle-sdk-jobentry-plugin/src/main/resources/org/pentaho/di/sdk/myplugins/jobentries/ftpplus/resources/demo.svg new file mode 100644 index 000000000..95639ccf0 --- /dev/null +++ b/kettle-sdk-jobentry-plugin/src/main/resources/org/pentaho/di/sdk/myplugins/jobentries/ftpplus/resources/demo.svg @@ -0,0 +1,25 @@ + + + + + + + + + + + + + + + + From 1aa843c577dbd4e485f4591ff8e8a630d24a81e0 Mon Sep 17 00:00:00 2001 From: shepf Date: Wed, 19 Dec 2018 22:44:22 +0800 Subject: [PATCH 3/5] =?UTF-8?q?=20json=E9=85=8D=E7=BD=AE=E9=80=9A=E7=94=A8?= =?UTF-8?q?JobEntry=20kettle=E6=8F=92=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kettle-sdk-embedding-samples/pom.xml | 36 + .../embedding/GeneratingFtpPlusJobs.java | 23 +- .../entries/ftpplus/JobEntryFtpPlus.java | 1286 +---------------- .../ftpplus/JobEntryFtpPlusParamsDO.java | 76 + .../jobentries/ftpplus/JobEntryFtpPlus.java | 18 +- .../ftpplus/JobEntryFtpPlusDialog.java | 3 +- .../ftpplus/JobEntryFtpPlusParamsDO.java | 10 +- .../messages/messages_en_US.properties | 6 +- pom.xml | 4 +- 9 files changed, 217 insertions(+), 1245 deletions(-) create mode 100644 kettle-sdk-embedding-samples/src/main/java/org/pentaho/di/sdk/samples/embedding/entries/ftpplus/JobEntryFtpPlusParamsDO.java diff --git a/kettle-sdk-embedding-samples/pom.xml b/kettle-sdk-embedding-samples/pom.xml index a094b14c4..3a9abf70c 100644 --- a/kettle-sdk-embedding-samples/pom.xml +++ b/kettle-sdk-embedding-samples/pom.xml @@ -28,12 +28,26 @@ commons-httpclient commons-httpclient + + commons-codec + commons-codec + pentaho-kettle kettle-engine ${pdi.version} + + + commons-dbcp + commons-dbcp + + + commons-codec + commons-codec + + org.pentaho.di.plugins @@ -105,5 +119,27 @@ 3.0.1 test + + + + + com.alibaba + fastjson + 1.2.47 + + + + commons-net + commons-net + 3.6 + + + + org.projectlombok + lombok + 1.18.2 + provided + + diff --git a/kettle-sdk-embedding-samples/src/main/java/org/pentaho/di/sdk/samples/embedding/GeneratingFtpPlusJobs.java b/kettle-sdk-embedding-samples/src/main/java/org/pentaho/di/sdk/samples/embedding/GeneratingFtpPlusJobs.java index 9bbfda660..cb543a907 100644 --- a/kettle-sdk-embedding-samples/src/main/java/org/pentaho/di/sdk/samples/embedding/GeneratingFtpPlusJobs.java +++ b/kettle-sdk-embedding-samples/src/main/java/org/pentaho/di/sdk/samples/embedding/GeneratingFtpPlusJobs.java @@ -1,5 +1,6 @@ package org.pentaho.di.sdk.samples.embedding; +import com.alibaba.fastjson.JSONObject; import org.apache.commons.io.FileUtils; import org.pentaho.di.core.KettleEnvironment; import org.pentaho.di.core.logging.LogLevel; @@ -10,6 +11,7 @@ import org.pentaho.di.job.entries.writetolog.JobEntryWriteToLog; import org.pentaho.di.job.entry.JobEntryCopy; import org.pentaho.di.sdk.samples.embedding.entries.ftpplus.JobEntryFtpPlus; +import org.pentaho.di.sdk.samples.embedding.entries.ftpplus.JobEntryFtpPlusParamsDO; import java.io.File; @@ -121,15 +123,18 @@ public JobMeta generateJob() { // crate and configure entry // JobEntryFtpPlus ftp = new JobEntryFtpPlus(); - ftp.setName( "FtpPlus Job" ); - //set ftp parameters - ftp.setServerName("127.0.0.1"); - ftp.setPort("21"); - ftp.setUserName("ftp1"); - // - ftp.setFtpDirectory("/"); - ftp.setWildcard(".*"); - ftp.setTargetDirectory("/tmp"); + JobEntryFtpPlusParamsDO jobEntryFtpPlusParamsDO = new JobEntryFtpPlusParamsDO(); + jobEntryFtpPlusParamsDO.setServerName("127.0.0.1"); + jobEntryFtpPlusParamsDO.setPort("21"); + jobEntryFtpPlusParamsDO.setUserName("ftp1"); + jobEntryFtpPlusParamsDO.setFtpDirectory("/"); + jobEntryFtpPlusParamsDO.setWildcard(".*"); + jobEntryFtpPlusParamsDO.setTargetDirectory("/tmp"); + + ftp.setName("FtpPlus Download"); + //ftp.setPluginId("JobEntryFtpPlus"); + String jsonString = JSONObject.toJSONString(jobEntryFtpPlusParamsDO); + ftp.setJsonConfStr(jsonString); // wrap into JobEntryCopy object, which holds generic job entry information diff --git a/kettle-sdk-embedding-samples/src/main/java/org/pentaho/di/sdk/samples/embedding/entries/ftpplus/JobEntryFtpPlus.java b/kettle-sdk-embedding-samples/src/main/java/org/pentaho/di/sdk/samples/embedding/entries/ftpplus/JobEntryFtpPlus.java index 8a1f59501..6546f7b77 100644 --- a/kettle-sdk-embedding-samples/src/main/java/org/pentaho/di/sdk/samples/embedding/entries/ftpplus/JobEntryFtpPlus.java +++ b/kettle-sdk-embedding-samples/src/main/java/org/pentaho/di/sdk/samples/embedding/entries/ftpplus/JobEntryFtpPlus.java @@ -1,1272 +1,116 @@ -/*! ****************************************************************************** - * - * Pentaho Data Integration - * - * Copyright (C) 2002-2017 by Hitachi Vantara : http://www.pentaho.com - * - ******************************************************************************* - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - - package org.pentaho.di.sdk.samples.embedding.entries.ftpplus; - -import org.pentaho.di.job.entries.ftp.MVSFileParser; -import org.pentaho.di.job.entry.validator.AndValidator; -import org.pentaho.di.job.entry.validator.JobEntryValidatorUtils; - -import java.io.File; -import java.io.IOException; -import java.net.InetAddress; -import java.net.UnknownHostException; -import java.text.SimpleDateFormat; -import java.util.Date; -import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import com.google.common.annotations.VisibleForTesting; -import org.apache.commons.vfs2.FileObject; +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONObject; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; +import org.apache.commons.net.ftp.FTPClient; +import org.apache.commons.net.ftp.FTPClientConfig; +import org.apache.commons.net.ftp.FTPReply; import org.pentaho.di.cluster.SlaveServer; -import org.pentaho.di.core.CheckResultInterface; -import org.pentaho.di.core.Const; -import org.pentaho.di.core.util.Utils; import org.pentaho.di.core.Result; -import org.pentaho.di.core.ResultFile; +import org.pentaho.di.core.annotations.JobEntry; import org.pentaho.di.core.database.DatabaseMeta; -import org.pentaho.di.core.encryption.Encr; -import org.pentaho.di.core.exception.KettleDatabaseException; import org.pentaho.di.core.exception.KettleException; import org.pentaho.di.core.exception.KettleXMLException; -import org.pentaho.di.core.util.StringUtil; -import org.pentaho.di.core.variables.VariableSpace; -import org.pentaho.di.core.vfs.KettleVFS; import org.pentaho.di.core.xml.XMLHandler; import org.pentaho.di.i18n.BaseMessages; -import org.pentaho.di.job.Job; -import org.pentaho.di.job.JobMeta; import org.pentaho.di.job.entry.JobEntryBase; import org.pentaho.di.job.entry.JobEntryInterface; -import org.pentaho.di.repository.ObjectId; import org.pentaho.di.repository.Repository; -import org.pentaho.di.resource.ResourceEntry; -import org.pentaho.di.resource.ResourceReference; import org.pentaho.metastore.api.IMetaStore; import org.w3c.dom.Node; -import com.enterprisedt.net.ftp.FTPClient; -import com.enterprisedt.net.ftp.FTPException; -import com.enterprisedt.net.ftp.FTPFileFactory; -import com.enterprisedt.net.ftp.FTPFileParser; +import java.io.IOException; +import java.net.SocketException; +import java.util.List; /** - * This defines an FtpPlus job entry. - * * @author shepf - * @since 12-18-2018 - * */ - +@JobEntry( + id = "JobEntryFtpPlus", + name = "JobEntryFtpPlus.Name", + description = "JobEntryFtpPlus.TooltipDesc", + categoryDescription = "i18n:org.pentaho.di.job:JobCategory.Category.FileTransfer", + i18nPackageName = "org.pentaho.di.sdk.myplugins.jobentries.ftpplus", + documentationUrl = "JobEntryFtpPlus.DocumentationURL", + casesUrl = "JobEntryFtpPlus.CasesURL", + forumUrl = "JobEntryFtpPlus.ForumURL" +) +@Getter +@Setter +@ToString public class JobEntryFtpPlus extends JobEntryBase implements Cloneable, JobEntryInterface { - private static Class PKG = JobEntryFtpPlus.class; // for i18n purposes, needed by Translator2!! - - private String serverName; - - private String userName; - - private String password; - private String ftpDirectory; - - private String targetDirectory; - - private String wildcard; - - private boolean binaryMode; - - private int timeout; - - private boolean remove; - - private boolean onlyGettingNewFiles; /* Don't overwrite files */ - - private boolean activeConnection; - - private String controlEncoding; /* how to convert list of filenames e.g. */ - - /** - * Implicit encoding used before PDI v2.4.1 - */ - private static String LEGACY_CONTROL_ENCODING = "US-ASCII"; + private String jsonConfStr = "{}"; + private String className = "JobEntryFtpPlus"; + private JobEntryFtpPlusParamsDO jobEntryFtpPlusParamsDO; /** - * Default encoding when making a new ftp job entry instance. + * for i18n */ - private static String DEFAULT_CONTROL_ENCODING = "ISO-8859-1"; - - private boolean movefiles; - - private String movetodirectory; - - private boolean adddate; - - private boolean addtime; - - private boolean SpecifyFormat; - - private String date_time_format; - - private boolean AddDateBeforeExtension; - - private boolean isaddresult; - - private boolean createmovefolder; - - private String port; - - private String proxyHost; - - private String proxyPort; /* string to allow variable substitution */ - - private String proxyUsername; - - private String proxyPassword; - - private String socksProxyHost; - - private String socksProxyPort; - - private String socksProxyUsername; - - private String socksProxyPassword; - - public int ifFileExistsSkip = 0; - - public String SifFileExistsSkip = "ifFileExistsSkip"; - - public int ifFileExistsCreateUniq = 1; - - public String SifFileExistsCreateUniq = "ifFileExistsCreateUniq"; - - public int ifFileExistsFail = 2; - - public String SifFileExistsFail = "ifFileExistsFail"; - - public int ifFileExists; - - public String SifFileExists; - - public String SUCCESS_IF_AT_LEAST_X_FILES_DOWNLOADED = "success_when_at_least"; + private static Class PKG = JobEntryFtpPlus.class; - public String SUCCESS_IF_ERRORS_LESS = "success_if_errors_less"; - - public String SUCCESS_IF_NO_ERRORS = "success_if_no_errors"; - - private String nr_limit; - - private String success_condition; - - long NrErrors = 0; - - long NrfilesRetrieved = 0; - - boolean successConditionBroken = false; - - int limitFiles = 0; - - String targetFilename = null; - - static String FILE_SEPARATOR = "/"; - - public JobEntryFtpPlus( String n ) { - super( n, "" ); - nr_limit = "10"; - port = "21"; - socksProxyPort = "1080"; - success_condition = SUCCESS_IF_NO_ERRORS; - ifFileExists = ifFileExistsSkip; - SifFileExists = SifFileExistsSkip; - - serverName = null; - movefiles = false; - movetodirectory = null; - adddate = false; - addtime = false; - SpecifyFormat = false; - AddDateBeforeExtension = false; - isaddresult = true; - createmovefolder = false; - - setControlEncoding( DEFAULT_CONTROL_ENCODING ); + public String getJsonConfStr() { + return jsonConfStr; } - public JobEntryFtpPlus() { - this( "" ); + public void setJsonConfStr(String jsonConfStr) { + this.jsonConfStr = jsonConfStr; } - public Object clone() { - org.pentaho.di.job.entries.ftp.JobEntryFTP je = (org.pentaho.di.job.entries.ftp.JobEntryFTP) super.clone(); - return je; - } - - public String getXML() { - StringBuilder retval = new StringBuilder( 650 ); // 528 chars in spaces and tags alone - - retval.append( super.getXML() ); - retval.append( " " ).append( XMLHandler.addTagValue( "port", port ) ); - retval.append( " " ).append( XMLHandler.addTagValue( "servername", serverName ) ); - retval.append( " " ).append( XMLHandler.addTagValue( "username", userName ) ); - retval.append( " " ).append( - XMLHandler.addTagValue( "password", Encr.encryptPasswordIfNotUsingVariables( password ) ) ); - retval.append( " " ).append( XMLHandler.addTagValue( "ftpdirectory", ftpDirectory ) ); - retval.append( " " ).append( XMLHandler.addTagValue( "targetdirectory", targetDirectory ) ); - retval.append( " " ).append( XMLHandler.addTagValue( "wildcard", wildcard ) ); - retval.append( " " ).append( XMLHandler.addTagValue( "binary", binaryMode ) ); - retval.append( " " ).append( XMLHandler.addTagValue( "timeout", timeout ) ); - retval.append( " " ).append( XMLHandler.addTagValue( "remove", remove ) ); - retval.append( " " ).append( XMLHandler.addTagValue( "only_new", onlyGettingNewFiles ) ); - retval.append( " " ).append( XMLHandler.addTagValue( "active", activeConnection ) ); - retval.append( " " ).append( XMLHandler.addTagValue( "control_encoding", controlEncoding ) ); - retval.append( " " ).append( XMLHandler.addTagValue( "movefiles", movefiles ) ); - retval.append( " " ).append( XMLHandler.addTagValue( "movetodirectory", movetodirectory ) ); - - retval.append( " " ).append( XMLHandler.addTagValue( "adddate", adddate ) ); - retval.append( " " ).append( XMLHandler.addTagValue( "addtime", addtime ) ); - retval.append( " " ).append( XMLHandler.addTagValue( "SpecifyFormat", SpecifyFormat ) ); - retval.append( " " ).append( XMLHandler.addTagValue( "date_time_format", date_time_format ) ); - retval.append( " " ).append( XMLHandler.addTagValue( "AddDateBeforeExtension", AddDateBeforeExtension ) ); - retval.append( " " ).append( XMLHandler.addTagValue( "isaddresult", isaddresult ) ); - retval.append( " " ).append( XMLHandler.addTagValue( "createmovefolder", createmovefolder ) ); - - retval.append( " " ).append( XMLHandler.addTagValue( "proxy_host", proxyHost ) ); - retval.append( " " ).append( XMLHandler.addTagValue( "proxy_port", proxyPort ) ); - retval.append( " " ).append( XMLHandler.addTagValue( "proxy_username", proxyUsername ) ); - retval.append( " " ).append( - XMLHandler.addTagValue( "proxy_password", Encr.encryptPasswordIfNotUsingVariables( proxyPassword ) ) ); - retval.append( " " ).append( XMLHandler.addTagValue( "socksproxy_host", socksProxyHost ) ); - retval.append( " " ).append( XMLHandler.addTagValue( "socksproxy_port", socksProxyPort ) ); - retval.append( " " ).append( XMLHandler.addTagValue( "socksproxy_username", socksProxyUsername ) ); - retval.append( " " ).append( - XMLHandler.addTagValue( "socksproxy_password", Encr - .encryptPasswordIfNotUsingVariables( socksProxyPassword ) ) ); - - retval.append( " " ).append( XMLHandler.addTagValue( "ifFileExists", SifFileExists ) ); - - retval.append( " " ).append( XMLHandler.addTagValue( "nr_limit", nr_limit ) ); - retval.append( " " ).append( XMLHandler.addTagValue( "success_condition", success_condition ) ); - - return retval.toString(); - } - - public void loadXML(Node entrynode, List databases, List slaveServers, - Repository rep, IMetaStore metaStore ) throws KettleXMLException { - try { - super.loadXML( entrynode, databases, slaveServers ); - port = XMLHandler.getTagValue( entrynode, "port" ); - serverName = XMLHandler.getTagValue( entrynode, "servername" ); - userName = XMLHandler.getTagValue( entrynode, "username" ); - password = Encr.decryptPasswordOptionallyEncrypted( XMLHandler.getTagValue( entrynode, "password" ) ); - ftpDirectory = XMLHandler.getTagValue( entrynode, "ftpdirectory" ); - targetDirectory = XMLHandler.getTagValue( entrynode, "targetdirectory" ); - wildcard = XMLHandler.getTagValue( entrynode, "wildcard" ); - binaryMode = "Y".equalsIgnoreCase( XMLHandler.getTagValue( entrynode, "binary" ) ); - timeout = Const.toInt( XMLHandler.getTagValue( entrynode, "timeout" ), 10000 ); - remove = "Y".equalsIgnoreCase( XMLHandler.getTagValue( entrynode, "remove" ) ); - onlyGettingNewFiles = "Y".equalsIgnoreCase( XMLHandler.getTagValue( entrynode, "only_new" ) ); - activeConnection = "Y".equalsIgnoreCase( XMLHandler.getTagValue( entrynode, "active" ) ); - controlEncoding = XMLHandler.getTagValue( entrynode, "control_encoding" ); - if ( controlEncoding == null ) { - // if we couldn't retrieve an encoding, assume it's an old instance and - // put in the the encoding used before v 2.4.0 - controlEncoding = LEGACY_CONTROL_ENCODING; - } - movefiles = "Y".equalsIgnoreCase( XMLHandler.getTagValue( entrynode, "movefiles" ) ); - movetodirectory = XMLHandler.getTagValue( entrynode, "movetodirectory" ); - - adddate = "Y".equalsIgnoreCase( XMLHandler.getTagValue( entrynode, "adddate" ) ); - addtime = "Y".equalsIgnoreCase( XMLHandler.getTagValue( entrynode, "addtime" ) ); - SpecifyFormat = "Y".equalsIgnoreCase( XMLHandler.getTagValue( entrynode, "SpecifyFormat" ) ); - date_time_format = XMLHandler.getTagValue( entrynode, "date_time_format" ); - AddDateBeforeExtension = - "Y".equalsIgnoreCase( XMLHandler.getTagValue( entrynode, "AddDateBeforeExtension" ) ); - - String addresult = XMLHandler.getTagValue( entrynode, "isaddresult" ); - if ( Utils.isEmpty( addresult ) ) { - isaddresult = true; - } else { - isaddresult = "Y".equalsIgnoreCase( addresult ); - } + @Override + public Result execute(Result prev_result, int nr) throws KettleException { - createmovefolder = "Y".equalsIgnoreCase( XMLHandler.getTagValue( entrynode, "createmovefolder" ) ); + //TODO ftp业务代码 + log.logBasic("good: " + jobEntryFtpPlusParamsDO.toString()); - proxyHost = XMLHandler.getTagValue( entrynode, "proxy_host" ); - proxyPort = XMLHandler.getTagValue( entrynode, "proxy_port" ); - proxyUsername = XMLHandler.getTagValue( entrynode, "proxy_username" ); - proxyPassword = - Encr.decryptPasswordOptionallyEncrypted( XMLHandler.getTagValue( entrynode, "proxy_password" ) ); - socksProxyHost = XMLHandler.getTagValue( entrynode, "socksproxy_host" ); - socksProxyPort = XMLHandler.getTagValue( entrynode, "socksproxy_port" ); - socksProxyUsername = XMLHandler.getTagValue( entrynode, "socksproxy_username" ); - socksProxyPassword = - Encr.decryptPasswordOptionallyEncrypted( XMLHandler.getTagValue( entrynode, "socksproxy_password" ) ); - SifFileExists = XMLHandler.getTagValue( entrynode, "ifFileExists" ); - if ( Utils.isEmpty( SifFileExists ) ) { - ifFileExists = ifFileExistsSkip; - } else { - if ( SifFileExists.equals( SifFileExistsCreateUniq ) ) { - ifFileExists = ifFileExistsCreateUniq; - } else if ( SifFileExists.equals( SifFileExistsFail ) ) { - ifFileExists = ifFileExistsFail; - } else { - ifFileExists = ifFileExistsSkip; - } - } - nr_limit = XMLHandler.getTagValue( entrynode, "nr_limit" ); - success_condition = - Const.NVL( XMLHandler.getTagValue( entrynode, "success_condition" ), SUCCESS_IF_NO_ERRORS ); - - } catch ( KettleXMLException xe ) { - throw new KettleXMLException( "Unable to load job entry of type 'ftp' from XML node", xe ); - } + // indicate there are no errors + prev_result.setNrErrors( 0 ); + // indicate the result as configured + prev_result.setResult( true ); + return prev_result; } - public void loadRep(Repository rep, IMetaStore metaStore, ObjectId id_jobentry, List databases, - List slaveServers ) throws KettleException { - try { - port = rep.getJobEntryAttributeString( id_jobentry, "port" ); - serverName = rep.getJobEntryAttributeString( id_jobentry, "servername" ); - userName = rep.getJobEntryAttributeString( id_jobentry, "username" ); - password = - Encr.decryptPasswordOptionallyEncrypted( rep.getJobEntryAttributeString( id_jobentry, "password" ) ); - ftpDirectory = rep.getJobEntryAttributeString( id_jobentry, "ftpdirectory" ); - targetDirectory = rep.getJobEntryAttributeString( id_jobentry, "targetdirectory" ); - wildcard = rep.getJobEntryAttributeString( id_jobentry, "wildcard" ); - binaryMode = rep.getJobEntryAttributeBoolean( id_jobentry, "binary" ); - timeout = (int) rep.getJobEntryAttributeInteger( id_jobentry, "timeout" ); - remove = rep.getJobEntryAttributeBoolean( id_jobentry, "remove" ); - onlyGettingNewFiles = rep.getJobEntryAttributeBoolean( id_jobentry, "only_new" ); - activeConnection = rep.getJobEntryAttributeBoolean( id_jobentry, "active" ); - controlEncoding = rep.getJobEntryAttributeString( id_jobentry, "control_encoding" ); - if ( controlEncoding == null ) { - // if we couldn't retrieve an encoding, assume it's an old instance and - // put in the the encoding used before v 2.4.0 - controlEncoding = LEGACY_CONTROL_ENCODING; - } - - movefiles = rep.getJobEntryAttributeBoolean( id_jobentry, "movefiles" ); - movetodirectory = rep.getJobEntryAttributeString( id_jobentry, "movetodirectory" ); - - adddate = rep.getJobEntryAttributeBoolean( id_jobentry, "adddate" ); - addtime = rep.getJobEntryAttributeBoolean( id_jobentry, "addtime" ); - SpecifyFormat = rep.getJobEntryAttributeBoolean( id_jobentry, "SpecifyFormat" ); - date_time_format = rep.getJobEntryAttributeString( id_jobentry, "date_time_format" ); - AddDateBeforeExtension = rep.getJobEntryAttributeBoolean( id_jobentry, "AddDateBeforeExtension" ); - String addToResult = rep.getJobEntryAttributeString( id_jobentry, "isaddresult" ); - if ( Utils.isEmpty( addToResult ) ) { - isaddresult = true; - } else { - isaddresult = rep.getJobEntryAttributeBoolean( id_jobentry, "isaddresult" ); - } + @Override + public String getXML() { + StringBuffer retval = new StringBuffer(); - createmovefolder = rep.getJobEntryAttributeBoolean( id_jobentry, "createmovefolder" ); + retval.append(super.getXML()); + JSONObject.toJSONString(jobEntryFtpPlusParamsDO); - proxyHost = rep.getJobEntryAttributeString( id_jobentry, "proxy_host" ); - proxyPort = rep.getJobEntryAttributeString( id_jobentry, "proxy_port" ); - proxyUsername = rep.getJobEntryAttributeString( id_jobentry, "proxy_username" ); - proxyPassword = - Encr.decryptPasswordOptionallyEncrypted( rep.getJobEntryAttributeString( id_jobentry, "proxy_password" ) ); - socksProxyHost = rep.getJobEntryAttributeString( id_jobentry, "socksproxy_host" ); - socksProxyPort = rep.getJobEntryAttributeString( id_jobentry, "socksproxy_port" ); - socksProxyUsername = rep.getJobEntryAttributeString( id_jobentry, "socksproxy_username" ); - socksProxyPassword = - Encr.decryptPasswordOptionallyEncrypted( rep.getJobEntryAttributeString( - id_jobentry, "socksproxy_password" ) ); - SifFileExists = rep.getJobEntryAttributeString( id_jobentry, "ifFileExists" ); - if ( Utils.isEmpty( SifFileExists ) ) { - ifFileExists = ifFileExistsSkip; - } else { - if ( SifFileExists.equals( SifFileExistsCreateUniq ) ) { - ifFileExists = ifFileExistsCreateUniq; - } else if ( SifFileExists.equals( SifFileExistsFail ) ) { - ifFileExists = ifFileExistsFail; - } else { - ifFileExists = ifFileExistsSkip; - } - } - nr_limit = rep.getJobEntryAttributeString( id_jobentry, "nr_limit" ); - success_condition = - Const.NVL( rep.getJobEntryAttributeString( id_jobentry, "success_condition" ), SUCCESS_IF_NO_ERRORS ); + retval.append(" ").append( + XMLHandler.addTagValue("configInfo", jsonConfStr)); + retval.append(" ").append( + XMLHandler.addTagValue("className", className)); - } catch ( KettleException dbe ) { - throw new KettleException( "Unable to load job entry of type 'ftp' from the repository for id_jobentry=" - + id_jobentry, dbe ); - } + return retval.toString(); } - public void saveRep( Repository rep, IMetaStore metaStore, ObjectId id_job ) throws KettleException { + @Override + public void loadXML(Node entryNode, List databases, + List slaveServers, Repository rep, IMetaStore metaStore) + throws KettleXMLException { try { - rep.saveJobEntryAttribute( id_job, getObjectId(), "port", port ); - rep.saveJobEntryAttribute( id_job, getObjectId(), "servername", serverName ); - rep.saveJobEntryAttribute( id_job, getObjectId(), "username", userName ); - rep.saveJobEntryAttribute( id_job, getObjectId(), "password", Encr - .encryptPasswordIfNotUsingVariables( password ) ); - rep.saveJobEntryAttribute( id_job, getObjectId(), "ftpdirectory", ftpDirectory ); - rep.saveJobEntryAttribute( id_job, getObjectId(), "targetdirectory", targetDirectory ); - rep.saveJobEntryAttribute( id_job, getObjectId(), "wildcard", wildcard ); - rep.saveJobEntryAttribute( id_job, getObjectId(), "binary", binaryMode ); - rep.saveJobEntryAttribute( id_job, getObjectId(), "timeout", timeout ); - rep.saveJobEntryAttribute( id_job, getObjectId(), "remove", remove ); - rep.saveJobEntryAttribute( id_job, getObjectId(), "only_new", onlyGettingNewFiles ); - rep.saveJobEntryAttribute( id_job, getObjectId(), "active", activeConnection ); - rep.saveJobEntryAttribute( id_job, getObjectId(), "control_encoding", controlEncoding ); - - rep.saveJobEntryAttribute( id_job, getObjectId(), "movefiles", movefiles ); - rep.saveJobEntryAttribute( id_job, getObjectId(), "movetodirectory", movetodirectory ); - - rep.saveJobEntryAttribute( id_job, getObjectId(), "addtime", addtime ); - rep.saveJobEntryAttribute( id_job, getObjectId(), "adddate", adddate ); - rep.saveJobEntryAttribute( id_job, getObjectId(), "SpecifyFormat", SpecifyFormat ); - rep.saveJobEntryAttribute( id_job, getObjectId(), "date_time_format", date_time_format ); - - rep.saveJobEntryAttribute( id_job, getObjectId(), "AddDateBeforeExtension", AddDateBeforeExtension ); - rep.saveJobEntryAttribute( id_job, getObjectId(), "isaddresult", isaddresult ); - rep.saveJobEntryAttribute( id_job, getObjectId(), "createmovefolder", createmovefolder ); + super.loadXML(entryNode, databases, slaveServers); + jsonConfStr = XMLHandler.getTagValue(entryNode, "configInfo"); - rep.saveJobEntryAttribute( id_job, getObjectId(), "proxy_host", proxyHost ); - rep.saveJobEntryAttribute( id_job, getObjectId(), "proxy_port", proxyPort ); - rep.saveJobEntryAttribute( id_job, getObjectId(), "proxy_username", proxyUsername ); - rep.saveJobEntryAttribute( id_job, getObjectId(), "proxy_password", Encr - .encryptPasswordIfNotUsingVariables( proxyPassword ) ); - rep.saveJobEntryAttribute( id_job, getObjectId(), "socksproxy_host", socksProxyHost ); - rep.saveJobEntryAttribute( id_job, getObjectId(), "socksproxy_port", socksProxyPort ); - rep.saveJobEntryAttribute( id_job, getObjectId(), "socksproxy_username", socksProxyUsername ); - rep.saveJobEntryAttribute( id_job, getObjectId(), "socksproxy_password", Encr - .encryptPasswordIfNotUsingVariables( socksProxyPassword ) ); - rep.saveJobEntryAttribute( id_job, getObjectId(), "ifFileExists", SifFileExists ); + //解析json到java类 + jobEntryFtpPlusParamsDO= JSON.parseObject(jsonConfStr,JobEntryFtpPlusParamsDO.class); - rep.saveJobEntryAttribute( id_job, getObjectId(), "nr_limit", nr_limit ); - rep.saveJobEntryAttribute( id_job, getObjectId(), "success_condition", success_condition ); - } catch ( KettleDatabaseException dbe ) { - throw new KettleException( - "Unable to save job entry of type 'ftp' to the repository for id_job=" + id_job, dbe ); + className = XMLHandler.getTagValue(entryNode, "className"); + } catch (Exception e) { + throw new KettleXMLException(BaseMessages.getString(PKG, + "JobEntryKettleUtil.UnableToLoadFromXml"), e); } } - public void setLimit( String nr_limitin ) { - this.nr_limit = nr_limitin; - } - - public String getLimit() { - return nr_limit; - } - - public void setSuccessCondition( String success_condition ) { - this.success_condition = success_condition; - } - - public String getSuccessCondition() { - return success_condition; - } - - public void setCreateMoveFolder( boolean createmovefolderin ) { - this.createmovefolder = createmovefolderin; - } - public boolean isCreateMoveFolder() { - return createmovefolder; - } - public void setAddDateBeforeExtension( boolean AddDateBeforeExtension ) { - this.AddDateBeforeExtension = AddDateBeforeExtension; - } - public boolean isAddDateBeforeExtension() { - return AddDateBeforeExtension; - } - - public void setAddToResult( boolean isaddresultin ) { - this.isaddresult = isaddresultin; - } - - public boolean isAddToResult() { - return isaddresult; - } - - public void setDateInFilename( boolean adddate ) { - this.adddate = adddate; - } - - public boolean isDateInFilename() { - return adddate; - } - - public void setTimeInFilename( boolean addtime ) { - this.addtime = addtime; - } - - public boolean isTimeInFilename() { - return addtime; - } - - public boolean isSpecifyFormat() { - return SpecifyFormat; - } - - public void setSpecifyFormat( boolean SpecifyFormat ) { - this.SpecifyFormat = SpecifyFormat; - } - - public String getDateTimeFormat() { - return date_time_format; - } - - public void setDateTimeFormat( String date_time_format ) { - this.date_time_format = date_time_format; - } - - /** - * @return Returns the movefiles. - */ - public boolean isMoveFiles() { - return movefiles; - } - - /** - * @param movefilesin - * The movefiles to set. - */ - public void setMoveFiles( boolean movefilesin ) { - this.movefiles = movefilesin; - } - - /** - * @return Returns the movetodirectory. - */ - public String getMoveToDirectory() { - return movetodirectory; - } - - /** - * @param movetoin - * The movetodirectory to set. - */ - public void setMoveToDirectory( String movetoin ) { - this.movetodirectory = movetoin; - } - - /** - * @return Returns the binaryMode. - */ - public boolean isBinaryMode() { - return binaryMode; - } - - /** - * @param binaryMode - * The binaryMode to set. - */ - public void setBinaryMode( boolean binaryMode ) { - this.binaryMode = binaryMode; - } - - /** - * @return Returns the directory. - */ - public String getFtpDirectory() { - return ftpDirectory; - } - - /** - * @param directory - * The directory to set. - */ - public void setFtpDirectory( String directory ) { - this.ftpDirectory = directory; - } - - /** - * @return Returns the password. - */ - public String getPassword() { - return password; - } - - /** - * @param password - * The password to set. - */ - public void setPassword( String password ) { - this.password = password; - } - - /** - * @return Returns the serverName. - */ - public String getServerName() { - return serverName; - } - - /** - * @param serverName - * The serverName to set. - */ - public void setServerName( String serverName ) { - this.serverName = serverName; - } - - /** - * @return Returns the port. - */ - public String getPort() { - return port; - } - - /** - * @param port - * The port to set. - */ - public void setPort( String port ) { - this.port = port; - } - - /** - * @return Returns the userName. - */ - public String getUserName() { - return userName; - } - - /** - * @param userName - * The userName to set. - */ - public void setUserName( String userName ) { - this.userName = userName; - } - - /** - * @return Returns the wildcard. - */ - public String getWildcard() { - return wildcard; - } - - /** - * @param wildcard - * The wildcard to set. - */ - public void setWildcard( String wildcard ) { - this.wildcard = wildcard; - } - - /** - * @return Returns the targetDirectory. - */ - public String getTargetDirectory() { - return targetDirectory; - } - - /** - * @param targetDirectory - * The targetDirectory to set. - */ - public void setTargetDirectory( String targetDirectory ) { - this.targetDirectory = targetDirectory; - } - - /** - * @param timeout - * The timeout to set. - */ - public void setTimeout( int timeout ) { - this.timeout = timeout; - } - - /** - * @return Returns the timeout. - */ - public int getTimeout() { - return timeout; - } - - /** - * @param remove - * The remove to set. - */ - public void setRemove( boolean remove ) { - this.remove = remove; - } - - /** - * @return Returns the remove. - */ - public boolean getRemove() { - return remove; - } - - /** - * @return Returns the onlyGettingNewFiles. - */ - public boolean isOnlyGettingNewFiles() { - return onlyGettingNewFiles; - } - - /** - * @param onlyGettingNewFiles - * The onlyGettingNewFiles to set. - */ - public void setOnlyGettingNewFiles( boolean onlyGettingNewFilesin ) { - this.onlyGettingNewFiles = onlyGettingNewFilesin; - } - - /** - * Get the control encoding to be used for ftp'ing - * - * @return the used encoding - */ - public String getControlEncoding() { - return controlEncoding; - } - - /** - * Set the encoding to be used for ftp'ing. This determines how names are translated in dir e.g. It does impact the - * contents of the files being ftp'ed. - * - * @param encoding - * The encoding to be used. - */ - public void setControlEncoding( String encoding ) { - this.controlEncoding = encoding; - } - - /** - * @return Returns the hostname of the ftp-proxy. - */ - public String getProxyHost() { - return proxyHost; - } - - /** - * @param proxyHost - * The hostname of the proxy. - */ - public void setProxyHost( String proxyHost ) { - this.proxyHost = proxyHost; - } - - /** - * @param proxyPassword - * The password which is used to authenticate at the socks proxy. - */ - public void setProxyPassword( String proxyPassword ) { - this.proxyPassword = proxyPassword; - } - - /** - * @return Returns the password which is used to authenticate at the proxy. - */ - public String getProxyPassword() { - return proxyPassword; - } - - /** - * @param proxyPassword - * The password which is used to authenticate at the proxy. - */ - public void setSocksProxyPassword( String socksProxyPassword ) { - this.socksProxyPassword = socksProxyPassword; - } - - /** - * @return Returns the password which is used to authenticate at the socks proxy. - */ - public String getSocksProxyPassword() { - return socksProxyPassword; - } - - /** - * @param proxyPort - * The port of the ftp-proxy. - */ - public void setProxyPort( String proxyPort ) { - this.proxyPort = proxyPort; - } - - /** - * @return Returns the port of the ftp-proxy. - */ - public String getProxyPort() { - return proxyPort; - } - - /** - * @return Returns the username which is used to authenticate at the proxy. - */ - public String getProxyUsername() { - return proxyUsername; - } - - /** - * @param proxyUsername - * The username which is used to authenticate at the proxy. - */ - public void setProxyUsername( String proxyUsername ) { - this.proxyUsername = proxyUsername; - } - - /** - * @return Returns the username which is used to authenticate at the socks proxy. - */ - public String getSocksProxyUsername() { - return socksProxyUsername; - } - - /** - * @param proxyUsername - * The username which is used to authenticate at the socks proxy. - */ - public void setSocksProxyUsername( String socksPoxyUsername ) { - this.socksProxyUsername = socksPoxyUsername; - } - - /** - * - * @param socksProxyHost - * The host name of the socks proxy host - */ - public void setSocksProxyHost( String socksProxyHost ) { - this.socksProxyHost = socksProxyHost; - } - - /** - * @return The host name of the socks proxy host - */ - public String getSocksProxyHost() { - return this.socksProxyHost; - } - - /** - * @param socksProxyPort - * The port number the socks proxy host is using - */ - public void setSocksProxyPort( String socksProxyPort ) { - this.socksProxyPort = socksProxyPort; - } - - /** - * @return The port number the socks proxy host is using - */ - public String getSocksProxyPort() { - return this.socksProxyPort; - } - - protected FTPClient initFTPClient() { - return new FTPClient(); - } - - protected InetAddress getInetAddress(String realServername ) throws UnknownHostException { - return InetAddress.getByName( realServername ); - } - - /** - * - * @param previousResult - * @param nr - * @return - */ - public Result execute(Result previousResult, int nr ) { - log.logBasic( BaseMessages.getString( PKG, "JobEntryFTP.Started", serverName ) ); - - Result result = previousResult; - result.setNrErrors( 1 ); - result.setResult( false ); - NrErrors = 0; - NrfilesRetrieved = 0; - successConditionBroken = false; - boolean exitjobentry = false; - limitFiles = Const.toInt( environmentSubstitute( getLimit() ), 10 ); - - - if ( isDetailed() ) { - logDetailed( BaseMessages.getString( PKG, "JobEntryFtpPlus.Start" ) ); - } - - //ftp download - System.out.println("good news"); - - result.setNrErrors( NrErrors ); - result.setNrFilesRetrieved( NrfilesRetrieved ); - if ( getSuccessStatus() ) { - result.setResult( true ); - } - if ( exitjobentry ) { - result.setResult( false ); - } - displayResults(); - return result; - } - - private void downloadFile( FTPClient ftpclient, String filename, String realMoveToFolder, Job parentJob, - Result result ) throws Exception { - String localFilename = filename; - targetFilename = KettleVFS.getFilename( KettleVFS.getFileObject( returnTargetFilename( localFilename ) ) ); - - if ( ( !onlyGettingNewFiles ) || ( onlyGettingNewFiles && needsDownload( targetFilename ) ) ) { - if ( isDetailed() ) { - logDetailed( BaseMessages.getString( - PKG, "JobEntryFTP.GettingFile", filename, environmentSubstitute( targetDirectory ) ) ); - } - ftpclient.get( targetFilename, filename ); - - // Update retrieved files - updateRetrievedFiles(); - if ( isDetailed() ) { - logDetailed( BaseMessages.getString( PKG, "JobEntryFTP.GotFile", filename ) ); - } - - // Add filename to result filenames - addFilenameToResultFilenames( result, parentJob, targetFilename ); - - // Delete the file if this is needed! - if ( remove ) { - ftpclient.delete( filename ); - if ( isDetailed() ) { - if ( isDetailed() ) { - logDetailed( BaseMessages.getString( PKG, "JobEntryFTP.DeletedFile", filename ) ); - } - } - } else { - if ( movefiles ) { - // Try to move file to destination folder ... - ftpclient.rename( filename, realMoveToFolder + FILE_SEPARATOR + filename ); - - if ( isDetailed() ) { - logDetailed( BaseMessages.getString( PKG, "JobEntryFTP.MovedFile", filename, realMoveToFolder ) ); - } - } - } - } - } - - /** - * normalize / to \ and remove trailing slashes from a path - * - * @param path - * @return normalized path - * @throws Exception - */ - public String normalizePath( String path ) throws Exception { - - String normalizedPath = path.replaceAll( "\\\\", FILE_SEPARATOR ); - while ( normalizedPath.endsWith( "\\" ) || normalizedPath.endsWith( FILE_SEPARATOR ) ) { - normalizedPath = normalizedPath.substring( 0, normalizedPath.length() - 1 ); - } - - return normalizedPath; - } - - private void addFilenameToResultFilenames( Result result, Job parentJob, String filename ) throws KettleException { - if ( isaddresult ) { - FileObject targetFile = null; - try { - targetFile = KettleVFS.getFileObject( filename, this ); - - // Add to the result files... - ResultFile resultFile = - new ResultFile( ResultFile.FILE_TYPE_GENERAL, targetFile, parentJob.getJobname(), toString() ); - resultFile.setComment( BaseMessages.getString( PKG, "JobEntryFTP.Downloaded", serverName ) ); - result.getResultFiles().put( resultFile.getFile().toString(), resultFile ); - - if ( isDetailed() ) { - logDetailed( BaseMessages.getString( PKG, "JobEntryFTP.FileAddedToResult", filename ) ); - } - } catch ( Exception e ) { - throw new KettleException( e ); - } finally { - try { - targetFile.close(); - targetFile = null; - } catch ( Exception e ) { - // Ignore close errors - } - } - } - } - - private void displayResults() { - if ( isDetailed() ) { - logDetailed( "=======================================" ); - logDetailed( BaseMessages.getString( PKG, "JobEntryFTP.Log.Info.FilesInError", "" + NrErrors ) ); - logDetailed( BaseMessages.getString( PKG, "JobEntryFTP.Log.Info.FilesRetrieved", "" + NrfilesRetrieved ) ); - logDetailed( "=======================================" ); - } - } - - private boolean getSuccessStatus() { - boolean retval = false; - - if ( ( NrErrors == 0 && getSuccessCondition().equals( SUCCESS_IF_NO_ERRORS ) ) - || ( NrfilesRetrieved >= limitFiles && getSuccessCondition().equals( - SUCCESS_IF_AT_LEAST_X_FILES_DOWNLOADED ) ) - || ( NrErrors <= limitFiles && getSuccessCondition().equals( SUCCESS_IF_ERRORS_LESS ) ) ) { - retval = true; - } - - return retval; - } - - private void updateErrors() { - NrErrors++; - if ( checkIfSuccessConditionBroken() ) { - // Success condition was broken - successConditionBroken = true; - } - } - - private boolean checkIfSuccessConditionBroken() { - boolean retval = false; - if ( ( NrErrors > 0 && getSuccessCondition().equals( SUCCESS_IF_NO_ERRORS ) ) - || ( NrErrors >= limitFiles && getSuccessCondition().equals( SUCCESS_IF_ERRORS_LESS ) ) ) { - retval = true; - } - return retval; - } - - private void updateRetrievedFiles() { - NrfilesRetrieved++; - } - - /** - * @param string - * the filename from the FTP server - * - * @return the calculated target filename - */ - @VisibleForTesting - String returnTargetFilename( String filename ) { - String retval = null; - // Replace possible environment variables... - if ( filename != null ) { - retval = filename; - } else { - return null; - } - - int lenstring = retval.length(); - int lastindexOfDot = retval.lastIndexOf( "." ); - if ( lastindexOfDot == -1 ) { - lastindexOfDot = lenstring; - } - - String fileExtension = retval.substring( lastindexOfDot, lenstring ); - - if ( isAddDateBeforeExtension() ) { - retval = retval.substring( 0, lastindexOfDot ); - } - - SimpleDateFormat daf = new SimpleDateFormat(); - Date now = new Date(); - - if ( SpecifyFormat && !Utils.isEmpty( date_time_format ) ) { - daf.applyPattern( date_time_format ); - String dt = daf.format( now ); - retval += dt; - } else { - if ( adddate ) { - daf.applyPattern( "yyyyMMdd" ); - String d = daf.format( now ); - retval += "_" + d; - } - if ( addtime ) { - daf.applyPattern( "HHmmssSSS" ); - String t = daf.format( now ); - retval += "_" + t; - } - } - - if ( isAddDateBeforeExtension() ) { - retval += fileExtension; - } - - // Add foldername to filename - retval = environmentSubstitute( targetDirectory ) + Const.FILE_SEPARATOR + retval; - return retval; - } - - public boolean evaluates() { - return true; - } - - /** - * See if the filename on the FTP server needs downloading. The default is to check the presence of the file in the - * target directory. If you need other functionality, extend this class and build it into a plugin. - * - * @param filename - * The local filename to check - * @param remoteFileSize - * The size of the remote file - * @return true if the file needs downloading - */ - protected boolean needsDownload( String filename ) { - boolean retval = false; - - File file = new File( filename ); - - if ( !file.exists() ) { - // Local file not exists! - if ( isDebug() ) { - logDebug( BaseMessages.getString( PKG, "JobEntryFTP.LocalFileNotExists" ), filename ); - } - return true; - } else { - - // Local file exists! - if ( ifFileExists == ifFileExistsCreateUniq ) { - if ( isDebug() ) { - logDebug( toString(), BaseMessages.getString( PKG, "JobEntryFTP.LocalFileExists" ), filename ); - // Create file with unique name - } - - int lenstring = targetFilename.length(); - int lastindexOfDot = targetFilename.lastIndexOf( '.' ); - if ( lastindexOfDot == -1 ) { - lastindexOfDot = lenstring; - } - - targetFilename = - targetFilename.substring( 0, lastindexOfDot ) - + StringUtil.getFormattedDateTimeNow( true ) - + targetFilename.substring( lastindexOfDot, lenstring ); - - return true; - } else if ( ifFileExists == ifFileExistsFail ) { - log.logError( BaseMessages.getString( PKG, "JobEntryFTP.LocalFileExists" ), filename ); - updateErrors(); - } else { - if ( isDebug() ) { - logDebug( toString(), BaseMessages.getString( PKG, "JobEntryFTP.LocalFileExists" ), filename ); - } - } - } - - return retval; - } - - /** - * @return the activeConnection - */ - public boolean isActiveConnection() { - return activeConnection; - } - - /** - * @param activeConnection - * the activeConnection to set - */ - public void setActiveConnection( boolean passive ) { - this.activeConnection = passive; - } - - public void check(List remarks, JobMeta jobMeta, VariableSpace space, - Repository repository, IMetaStore metaStore ) { - JobEntryValidatorUtils.andValidator().validate( this, "serverName", remarks, - AndValidator.putValidators( JobEntryValidatorUtils.notBlankValidator() ) ); - JobEntryValidatorUtils.andValidator().validate( - this, "targetDirectory", remarks, AndValidator.putValidators( JobEntryValidatorUtils.notBlankValidator(), - JobEntryValidatorUtils.fileExistsValidator() ) ); - JobEntryValidatorUtils.andValidator().validate( this, "userName", remarks, - AndValidator.putValidators( JobEntryValidatorUtils.notBlankValidator() ) ); - JobEntryValidatorUtils.andValidator().validate( this, "password", remarks, - AndValidator.putValidators( JobEntryValidatorUtils.notNullValidator() ) ); - } - - public List getResourceDependencies(JobMeta jobMeta ) { - List references = super.getResourceDependencies( jobMeta ); - if ( !Utils.isEmpty( serverName ) ) { - String realServername = jobMeta.environmentSubstitute( serverName ); - ResourceReference reference = new ResourceReference( this ); - reference.getEntries().add( new ResourceEntry( realServername, ResourceEntry.ResourceType.SERVER ) ); - references.add( reference ); - } - return references; - } - - /** - * Hook in known parsers, and then those that have been specified in the variable ftp.file.parser.class.names - * - * @param ftpClient - * @throws FTPException - * @throws IOException - */ - protected void hookInOtherParsers( FTPClient ftpClient ) throws FTPException, IOException { - if ( log.isDebug() ) { - logDebug( BaseMessages.getString( PKG, "JobEntryFTP.DEBUG.Hooking.Parsers" ) ); - } - String system = ftpClient.system(); - MVSFileParser parser = new MVSFileParser( log ); - if ( log.isDebug() ) { - logDebug( BaseMessages.getString( PKG, "JobEntryFTP.DEBUG.Created.MVS.Parser" ) ); - } - FTPFileFactory factory = new FTPFileFactory( system ); - if ( log.isDebug() ) { - logDebug( BaseMessages.getString( PKG, "JobEntryFTP.DEBUG.Created.Factory" ) ); - } - factory.addParser( parser ); - ftpClient.setFTPFileFactory( factory ); - if ( log.isDebug() ) { - logDebug( BaseMessages.getString( PKG, "JobEntryFTP.DEBUG.Get.Variable.Space" ) ); - } - VariableSpace vs = this.getVariables(); - if ( vs != null ) { - if ( log.isDebug() ) { - logDebug( BaseMessages.getString( PKG, "JobEntryFTP.DEBUG.Getting.Other.Parsers" ) ); - } - String otherParserNames = vs.getVariable( "ftp.file.parser.class.names" ); - if ( otherParserNames != null ) { - if ( log.isDebug() ) { - logDebug( BaseMessages.getString( PKG, "JobEntryFTP.DEBUG.Creating.Parsers" ) ); - } - String[] parserClasses = otherParserNames.split( "|" ); - String cName = null; - Class clazz = null; - Object parserInstance = null; - for ( int i = 0; i < parserClasses.length; i++ ) { - cName = parserClasses[i].trim(); - if ( cName.length() > 0 ) { - try { - clazz = Class.forName( cName ); - parserInstance = clazz.newInstance(); - if ( parserInstance instanceof FTPFileParser ) { - if ( log.isDetailed() ) { - logDetailed( BaseMessages.getString( PKG, "JobEntryFTP.DEBUG.Created.Other.Parser", cName ) ); - } - factory.addParser( (FTPFileParser) parserInstance ); - } - } catch ( Exception ignored ) { - if ( log.isDebug() ) { - ignored.printStackTrace(); - logError( BaseMessages.getString( PKG, "JobEntryFTP.ERROR.Creating.Parser", cName ) ); - } - } - } - } - } - } - } } diff --git a/kettle-sdk-embedding-samples/src/main/java/org/pentaho/di/sdk/samples/embedding/entries/ftpplus/JobEntryFtpPlusParamsDO.java b/kettle-sdk-embedding-samples/src/main/java/org/pentaho/di/sdk/samples/embedding/entries/ftpplus/JobEntryFtpPlusParamsDO.java new file mode 100644 index 000000000..89134a404 --- /dev/null +++ b/kettle-sdk-embedding-samples/src/main/java/org/pentaho/di/sdk/samples/embedding/entries/ftpplus/JobEntryFtpPlusParamsDO.java @@ -0,0 +1,76 @@ +package org.pentaho.di.sdk.samples.embedding.entries.ftpplus; + + +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; + +/** + * FtpPlus 下载插件参数 + * @author shepf + */ +@Getter +@Setter +@ToString +public class JobEntryFtpPlusParamsDO { + private String serverName; + private String userName; + private String password; + private String ftpDirectory; + private String targetDirectory; + private String wildcard; + private boolean binaryMode; + private int timeout; + private boolean remove; + private boolean onlyGettingNewFiles; /* Don't overwrite files */ + private boolean activeConnection; + private String controlEncoding; /* how to convert list of filenames e.g. */ + /** + * Implicit encoding used before PDI v2.4.1 + */ + private static String LEGACY_CONTROL_ENCODING = "US-ASCII"; + /** + * Default encoding when making a new ftp job entry instance. + */ + private static String DEFAULT_CONTROL_ENCODING = "ISO-8859-1"; + private boolean movefiles; + private String movetodirectory; + private boolean adddate; + private boolean addtime; + private boolean SpecifyFormat; + private String date_time_format; + private boolean AddDateBeforeExtension; + private boolean isaddresult; + private boolean createmovefolder; + private String port; + private String proxyHost; + private String proxyPort; /* string to allow variable substitution */ + private String proxyUsername; + private String proxyPassword; + private String socksProxyHost; + private String socksProxyPort; + private String socksProxyUsername; + private String socksProxyPassword; + public int ifFileExistsSkip = 0; + public String SifFileExistsSkip = "ifFileExistsSkip"; + public int ifFileExistsCreateUniq = 1; + public String SifFileExistsCreateUniq = "ifFileExistsCreateUniq"; + public int ifFileExistsFail = 2; + public String SifFileExistsFail = "ifFileExistsFail"; + public int ifFileExists; + public String SifFileExists; + public String SUCCESS_IF_AT_LEAST_X_FILES_DOWNLOADED = "success_when_at_least"; + public String SUCCESS_IF_ERRORS_LESS = "success_if_errors_less"; + public String SUCCESS_IF_NO_ERRORS = "success_if_no_errors"; + private String nr_limit; + private String success_condition; + long NrErrors = 0; + long NrfilesRetrieved = 0; + boolean successConditionBroken = false; + int limitFiles = 0; + String targetFilename = null; + static String FILE_SEPARATOR = "/"; + + + +} diff --git a/kettle-sdk-jobentry-plugin/src/main/java/org/pentaho/di/sdk/myplugins/jobentries/ftpplus/JobEntryFtpPlus.java b/kettle-sdk-jobentry-plugin/src/main/java/org/pentaho/di/sdk/myplugins/jobentries/ftpplus/JobEntryFtpPlus.java index cd3d7a472..e5fe789bd 100644 --- a/kettle-sdk-jobentry-plugin/src/main/java/org/pentaho/di/sdk/myplugins/jobentries/ftpplus/JobEntryFtpPlus.java +++ b/kettle-sdk-jobentry-plugin/src/main/java/org/pentaho/di/sdk/myplugins/jobentries/ftpplus/JobEntryFtpPlus.java @@ -1,5 +1,7 @@ package org.pentaho.di.sdk.myplugins.jobentries.ftpplus; +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONObject; import org.apache.commons.net.ftp.FTPClient; import org.apache.commons.net.ftp.FTPClientConfig; import org.apache.commons.net.ftp.FTPReply; @@ -38,6 +40,7 @@ public class JobEntryFtpPlus extends JobEntryBase implements Cloneable, JobEntry private String jsonConfStr = "{}"; private String className = "JobEntryFtpPlus"; + private JobEntryFtpPlusParamsDO jobEntryFtpPlusParamsDO; /** * for i18n @@ -57,6 +60,8 @@ public void setJsonConfStr(String jsonConfStr) { public Result execute(Result prev_result, int nr) throws KettleException { //TODO ftp业务代码 + log.logBasic("good: " + jobEntryFtpPlusParamsDO.toString()); + // indicate there are no errors prev_result.setNrErrors( 0 ); @@ -71,6 +76,8 @@ public String getXML() { StringBuffer retval = new StringBuffer(); retval.append(super.getXML()); + JSONObject.toJSONString(jobEntryFtpPlusParamsDO); + retval.append(" ").append( XMLHandler.addTagValue("configInfo", jsonConfStr)); retval.append(" ").append( @@ -80,16 +87,17 @@ public String getXML() { } @Override - public void loadXML(Node entrynode, List databases, + public void loadXML(Node entryNode, List databases, List slaveServers, Repository rep, IMetaStore metaStore) throws KettleXMLException { try { - super.loadXML(entrynode, databases, slaveServers); - jsonConfStr = XMLHandler.getTagValue(entrynode, "configInfo"); - + super.loadXML(entryNode, databases, slaveServers); + jsonConfStr = XMLHandler.getTagValue(entryNode, "configInfo"); + //解析json到java类 + jobEntryFtpPlusParamsDO= JSON.parseObject(jsonConfStr,JobEntryFtpPlusParamsDO.class); - className = XMLHandler.getTagValue(entrynode, "className"); + className = XMLHandler.getTagValue(entryNode, "className"); } catch (Exception e) { throw new KettleXMLException(BaseMessages.getString(PKG, "JobEntryKettleUtil.UnableToLoadFromXml"), e); diff --git a/kettle-sdk-jobentry-plugin/src/main/java/org/pentaho/di/sdk/myplugins/jobentries/ftpplus/JobEntryFtpPlusDialog.java b/kettle-sdk-jobentry-plugin/src/main/java/org/pentaho/di/sdk/myplugins/jobentries/ftpplus/JobEntryFtpPlusDialog.java index 58d4839f4..d8bc175e1 100644 --- a/kettle-sdk-jobentry-plugin/src/main/java/org/pentaho/di/sdk/myplugins/jobentries/ftpplus/JobEntryFtpPlusDialog.java +++ b/kettle-sdk-jobentry-plugin/src/main/java/org/pentaho/di/sdk/myplugins/jobentries/ftpplus/JobEntryFtpPlusDialog.java @@ -169,7 +169,8 @@ public void modifyText(ModifyEvent e) { wlConfigInfo.setLayoutData(fdlConfigInfo); wConfigInfo = new StyledTextComp(jobEntry, shell, SWT.MULTI | SWT.LEFT | SWT.BORDER | SWT.H_SCROLL | SWT.V_SCROLL, ""); - wConfigInfo.setText(BaseMessages.getString(PKG, "JobEntryKettleUtil.Script.Default")); + //默认json配置 + wConfigInfo.setText("{}"); props.setLook(wConfigInfo, Props.WIDGET_STYLE_FIXED); wConfigInfo.addModifyListener(lsMod); fdConfigInfo = new FormData(); diff --git a/kettle-sdk-jobentry-plugin/src/main/java/org/pentaho/di/sdk/myplugins/jobentries/ftpplus/JobEntryFtpPlusParamsDO.java b/kettle-sdk-jobentry-plugin/src/main/java/org/pentaho/di/sdk/myplugins/jobentries/ftpplus/JobEntryFtpPlusParamsDO.java index f695a0965..6d132d2f6 100644 --- a/kettle-sdk-jobentry-plugin/src/main/java/org/pentaho/di/sdk/myplugins/jobentries/ftpplus/JobEntryFtpPlusParamsDO.java +++ b/kettle-sdk-jobentry-plugin/src/main/java/org/pentaho/di/sdk/myplugins/jobentries/ftpplus/JobEntryFtpPlusParamsDO.java @@ -1,11 +1,17 @@ package org.pentaho.di.sdk.myplugins.jobentries.ftpplus; - +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; /** - * + * FtpPlus 下载插件参数 + * @author shepf */ +@Getter +@Setter +@ToString public class JobEntryFtpPlusParamsDO { private String serverName; private String userName; diff --git a/kettle-sdk-jobentry-plugin/src/main/resources/org/pentaho/di/sdk/myplugins/jobentries/ftpplus/messages/messages_en_US.properties b/kettle-sdk-jobentry-plugin/src/main/resources/org/pentaho/di/sdk/myplugins/jobentries/ftpplus/messages/messages_en_US.properties index 16e29f932..de74371fd 100644 --- a/kettle-sdk-jobentry-plugin/src/main/resources/org/pentaho/di/sdk/myplugins/jobentries/ftpplus/messages/messages_en_US.properties +++ b/kettle-sdk-jobentry-plugin/src/main/resources/org/pentaho/di/sdk/myplugins/jobentries/ftpplus/messages/messages_en_US.properties @@ -4,13 +4,9 @@ JobEntryKettleUtil.Jobname.Label=JobName JobEntryKettleUtil.ClassName.Label=JobEntryClassName JobEntryKettleUtil.Script.Label=JsonConfigure -JobEntryKettleUtil.LineNr.Label=LineNr: {0} -JobEntryKettleUtil.Script.Default=please input json String -JobEntryKettleUtil.Position.Label=x {0} y {1} -FtpPlus.Default.Name=FtpPlusDialog # -JobEntryFtpPlus.Name=FtpPlus +JobEntryFtpPlus.Name=FtpPlus Download JobEntryFtpPlus.TooltipDesc=Demo FtpPlusDialog Job Entry JobEntryFtpPlus.DocumentationURL=https://help.pentaho.com/Documentation/6.1/0R0/0V0 JobEntryFtpPlus.CasesURL=http://jira.pentaho.com/browse/PDI/ diff --git a/pom.xml b/pom.xml index 5bdcfcb8d..79ab46938 100644 --- a/pom.xml +++ b/pom.xml @@ -38,11 +38,11 @@ Pentaho Public http://nexus.pentaho.org/content/groups/omni/ - true + false daily - true + false interval:15 From 50eba7c5b3065c7ef02b0f3f1ff381c243387513 Mon Sep 17 00:00:00 2001 From: shepf Date: Fri, 21 Dec 2018 00:08:42 +0800 Subject: [PATCH 4/5] =?UTF-8?q?=20json=E9=85=8D=E7=BD=AE=E9=80=9A=E7=94=A8?= =?UTF-8?q?JobEntry=20kettle=E6=8F=92=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../embedding/GeneratingFtpPlusJobs.java | 5 +- .../jobentries/ftpplus/JobEntryFtpPlus.java | 68 +++++++++++++++---- .../ftpplus/JobEntryFtpPlusDialog.java | 56 +++++++++------ 3 files changed, 94 insertions(+), 35 deletions(-) diff --git a/kettle-sdk-embedding-samples/src/main/java/org/pentaho/di/sdk/samples/embedding/GeneratingFtpPlusJobs.java b/kettle-sdk-embedding-samples/src/main/java/org/pentaho/di/sdk/samples/embedding/GeneratingFtpPlusJobs.java index cb543a907..402512cfa 100644 --- a/kettle-sdk-embedding-samples/src/main/java/org/pentaho/di/sdk/samples/embedding/GeneratingFtpPlusJobs.java +++ b/kettle-sdk-embedding-samples/src/main/java/org/pentaho/di/sdk/samples/embedding/GeneratingFtpPlusJobs.java @@ -127,12 +127,13 @@ public JobMeta generateJob() { jobEntryFtpPlusParamsDO.setServerName("127.0.0.1"); jobEntryFtpPlusParamsDO.setPort("21"); jobEntryFtpPlusParamsDO.setUserName("ftp1"); + jobEntryFtpPlusParamsDO.setPassword("ftp1"); jobEntryFtpPlusParamsDO.setFtpDirectory("/"); jobEntryFtpPlusParamsDO.setWildcard(".*"); jobEntryFtpPlusParamsDO.setTargetDirectory("/tmp"); - ftp.setName("FtpPlus Download"); - //ftp.setPluginId("JobEntryFtpPlus"); + ftp.setName("FtpPlus"); + ftp.setPluginId("JobEntryFtpPlus"); String jsonString = JSONObject.toJSONString(jobEntryFtpPlusParamsDO); ftp.setJsonConfStr(jsonString); diff --git a/kettle-sdk-jobentry-plugin/src/main/java/org/pentaho/di/sdk/myplugins/jobentries/ftpplus/JobEntryFtpPlus.java b/kettle-sdk-jobentry-plugin/src/main/java/org/pentaho/di/sdk/myplugins/jobentries/ftpplus/JobEntryFtpPlus.java index e5fe789bd..1708cf084 100644 --- a/kettle-sdk-jobentry-plugin/src/main/java/org/pentaho/di/sdk/myplugins/jobentries/ftpplus/JobEntryFtpPlus.java +++ b/kettle-sdk-jobentry-plugin/src/main/java/org/pentaho/di/sdk/myplugins/jobentries/ftpplus/JobEntryFtpPlus.java @@ -2,6 +2,9 @@ import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONObject; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; import org.apache.commons.net.ftp.FTPClient; import org.apache.commons.net.ftp.FTPClientConfig; import org.apache.commons.net.ftp.FTPReply; @@ -24,6 +27,8 @@ import java.util.List; /** + * configInfo json字符串格式 + * getDefaultConfigInfo() 格式化 json str * @author shepf */ @JobEntry( @@ -36,10 +41,13 @@ casesUrl = "JobEntryFtpPlus.CasesURL", forumUrl = "JobEntryFtpPlus.ForumURL" ) +@Getter +@Setter +@ToString public class JobEntryFtpPlus extends JobEntryBase implements Cloneable, JobEntryInterface { - private String jsonConfStr = "{}"; - private String className = "JobEntryFtpPlus"; + private String configInfo = "{}"; + private String className = this.getClass().getName(); private JobEntryFtpPlusParamsDO jobEntryFtpPlusParamsDO; /** @@ -47,14 +55,6 @@ public class JobEntryFtpPlus extends JobEntryBase implements Cloneable, JobEntry */ private static Class PKG = JobEntryFtpPlus.class; - public String getJsonConfStr() { - return jsonConfStr; - } - - public void setJsonConfStr(String jsonConfStr) { - this.jsonConfStr = jsonConfStr; - } - @Override public Result execute(Result prev_result, int nr) throws KettleException { @@ -62,6 +62,47 @@ public Result execute(Result prev_result, int nr) throws KettleException { //TODO ftp业务代码 log.logBasic("good: " + jobEntryFtpPlusParamsDO.toString()); + FTPClient ftp = new FTPClient(); + FTPClientConfig config = new FTPClientConfig(); + + //config.setXXX(YYY); // change required options + // for example config.setServerTimeZoneId("Pacific/Pitcairn") + ftp.configure(config ); + boolean error = false; + try { + int reply; + String server = "127.0.0.1"; + ftp.connect(server); + ftp.login("ftp1",null); + + + System.out.println("Connected to " + server + "."); + System.out.print(ftp.getReplyString()); + + // After connection attempt, you should check the reply code to verify + // success. + reply = ftp.getReplyCode(); + + if(!FTPReply.isPositiveCompletion(reply)) { + ftp.disconnect(); + System.err.println("FTP server refused connection."); + System.exit(1); + } + // transfer files + ftp.logout(); + } catch(IOException e) { + error = true; + e.printStackTrace(); + } finally { + if(ftp.isConnected()) { + try { + ftp.disconnect(); + } catch(IOException ioe) { + // do nothing + } + } + System.exit(error ? 1 : 0); + } // indicate there are no errors prev_result.setNrErrors( 0 ); @@ -79,7 +120,7 @@ public String getXML() { JSONObject.toJSONString(jobEntryFtpPlusParamsDO); retval.append(" ").append( - XMLHandler.addTagValue("configInfo", jsonConfStr)); + XMLHandler.addTagValue("configInfo", configInfo)); retval.append(" ").append( XMLHandler.addTagValue("className", className)); @@ -92,10 +133,10 @@ public void loadXML(Node entryNode, List databases, throws KettleXMLException { try { super.loadXML(entryNode, databases, slaveServers); - jsonConfStr = XMLHandler.getTagValue(entryNode, "configInfo"); + configInfo = XMLHandler.getTagValue(entryNode, "configInfo"); //解析json到java类 - jobEntryFtpPlusParamsDO= JSON.parseObject(jsonConfStr,JobEntryFtpPlusParamsDO.class); + jobEntryFtpPlusParamsDO= JSON.parseObject(configInfo,JobEntryFtpPlusParamsDO.class); className = XMLHandler.getTagValue(entryNode, "className"); } catch (Exception e) { @@ -107,4 +148,5 @@ public void loadXML(Node entryNode, List databases, + } diff --git a/kettle-sdk-jobentry-plugin/src/main/java/org/pentaho/di/sdk/myplugins/jobentries/ftpplus/JobEntryFtpPlusDialog.java b/kettle-sdk-jobentry-plugin/src/main/java/org/pentaho/di/sdk/myplugins/jobentries/ftpplus/JobEntryFtpPlusDialog.java index d8bc175e1..e209b0b6a 100644 --- a/kettle-sdk-jobentry-plugin/src/main/java/org/pentaho/di/sdk/myplugins/jobentries/ftpplus/JobEntryFtpPlusDialog.java +++ b/kettle-sdk-jobentry-plugin/src/main/java/org/pentaho/di/sdk/myplugins/jobentries/ftpplus/JobEntryFtpPlusDialog.java @@ -1,5 +1,8 @@ package org.pentaho.di.sdk.myplugins.jobentries.ftpplus; +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONObject; +import com.alibaba.fastjson.serializer.SerializerFeature; import org.apache.commons.lang.StringUtils; import org.eclipse.swt.SWT; import org.eclipse.swt.events.*; @@ -22,6 +25,10 @@ import org.pentaho.di.ui.job.entry.JobEntryDialog; import org.pentaho.di.ui.trans.step.BaseStepDialog; +import javax.swing.*; +import java.awt.event.WindowAdapter; +import java.awt.event.WindowEvent; + /** * @author shepf */ @@ -30,17 +37,24 @@ pluginType = PluginDialog.PluginType.JOBENTRY) public class JobEntryFtpPlusDialog extends JobEntryDialog implements JobEntryDialogInterface { - // for i18n purposes + /** + * for i18n + */ private static Class PKG = JobEntryFtpPlus.class; - // the job entry configuration object + /** + * the job entry configuration object + */ private JobEntryFtpPlus jobEntry; - //配合界面 + /** + * 界面 + */ private Label wlName; private Text wText; private FormData fdlName, fdName; private Label wlConfigInfo; private StyledTextComp wConfigInfo; + private JEditorPane editPane; private FormData fdlConfigInfo, fdConfigInfo; private Label wlPosition; private FormData fdlPosition; @@ -50,7 +64,6 @@ public class JobEntryFtpPlusDialog extends JobEntryDialog implements JobEntryDia private SelectionAdapter lsDef; private boolean changed; - /** * 配置名称 */ @@ -115,7 +128,7 @@ public void modifyText(ModifyEvent e) { // at the bottom BaseStepDialog.positionBottomButtons(shell, new Button[]{wOK, wCancel, wGet}, margin, null); - // Filename line + // Label组件 wlName = new Label(shell, SWT.RIGHT); wlName.setText(BaseMessages.getString(PKG, "JobEntryKettleUtil.Jobname.Label")); props.setLook(wlName); @@ -242,23 +255,27 @@ public void handleEvent(Event e) { lsGet = new Listener() { @Override public void handleEvent(Event e) { -// jobEntry.setClassName(wClassName.getText()); -// jobEntry.setConfigInfo(wConfigInfo.getText()); - String conf = null; - String msg = "获取默认配置失败，请输入正确的类名称"; + String prettyConf = null; + String msg = "获取默认配置失败"; try { - //conf = jobEntry.getDefaultConfigInfo(); + String confStr = jobEntry.getConfigInfo(); + //TODO json格式化显示 + //prettyConf = JSON.toJSONString(confStr,SerializerFeature.PrettyFormat); + //JSONObject jsonObject = JSON.parseObject(jobEntry.environmentSubstitute(confStr)); + //prettyConf = jsonObject.toJSONString(); + prettyConf = JSON.toJSONString(jobEntry.environmentSubstitute(confStr),SerializerFeature.PrettyFormat); + } catch (Exception e1) { msg = e1.getMessage(); } - if (StringUtils.isBlank(conf)) { + if (StringUtils.isBlank(prettyConf)) { wConfigInfo.setText("{}"); MessageBox mb = new MessageBox(shell, SWT.OK | SWT.ICON_ERROR); mb.setMessage(msg); mb.setText("错误"); mb.open(); } else { - wConfigInfo.setText(conf); + wConfigInfo.setText(prettyConf); } } }; @@ -327,12 +344,12 @@ public void getData() { if (jobEntry.getName() != null) { wText.setText(jobEntry.getName()); } -// if ( jobEntry.getClassName() != null ) { -// wClassName.setText( jobEntry.getClassName() ); -// } -// if ( jobEntry.getConfigInfo() != null ) { -// wConfigInfo.setText( jobEntry.getConfigInfo() ); -// } + if ( jobEntry.getClassName() != null ) { + wClassName.setText( jobEntry.getClassName() ); + } + if ( jobEntry.getConfigInfo() != null ) { + wConfigInfo.setText( jobEntry.getConfigInfo() ); + } wText.selectAll(); wText.setFocus(); @@ -353,8 +370,7 @@ private void ok() { return; } - jobEntry.setJsonConfStr(wText.getText()); - //jobEntry.setConfigInfo( wConfigInfo.getText() ); + jobEntry.setConfigInfo( wConfigInfo.getText() ); //jobEntry.setClassName( wClassName.getText() ); dispose(); } From aa411fec227b5bd1e63377e17ef62b1ff5f7ef5a Mon Sep 17 00:00:00 2001 From: shepf Date: Fri, 21 Dec 2018 11:11:29 +0800 Subject: [PATCH 5/5] =?UTF-8?q?=20json=E9=85=8D=E7=BD=AE=E9=80=9A=E7=94=A8?= =?UTF-8?q?JobEntry=20kettle=E6=8F=92=E4=BB=B6=EF=BC=9Ajson=E6=A0=BC?= =?UTF-8?q?=E5=BC=8F=E5=8C=96=E6=98=BE=E7=A4=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../di/sdk/samples/embedding/GeneratingFtpPlusJobs.java | 4 +++- .../myplugins/jobentries/ftpplus/JobEntryFtpPlus.java | 5 ----- .../jobentries/ftpplus/JobEntryFtpPlusDialog.java | 9 ++++----- 3 files changed, 7 insertions(+), 11 deletions(-) diff --git a/kettle-sdk-embedding-samples/src/main/java/org/pentaho/di/sdk/samples/embedding/GeneratingFtpPlusJobs.java b/kettle-sdk-embedding-samples/src/main/java/org/pentaho/di/sdk/samples/embedding/GeneratingFtpPlusJobs.java index 402512cfa..cce3ebce7 100644 --- a/kettle-sdk-embedding-samples/src/main/java/org/pentaho/di/sdk/samples/embedding/GeneratingFtpPlusJobs.java +++ b/kettle-sdk-embedding-samples/src/main/java/org/pentaho/di/sdk/samples/embedding/GeneratingFtpPlusJobs.java @@ -1,5 +1,6 @@ package org.pentaho.di.sdk.samples.embedding; +import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONObject; import org.apache.commons.io.FileUtils; import org.pentaho.di.core.KettleEnvironment; @@ -134,7 +135,8 @@ public JobMeta generateJob() { ftp.setName("FtpPlus"); ftp.setPluginId("JobEntryFtpPlus"); - String jsonString = JSONObject.toJSONString(jobEntryFtpPlusParamsDO); + String jsonString = JSON.toJSONString(jobEntryFtpPlusParamsDO); + //String jsonString = JSONObject.toJSONString(jobEntryFtpPlusParamsDO); ftp.setJsonConfStr(jsonString); diff --git a/kettle-sdk-jobentry-plugin/src/main/java/org/pentaho/di/sdk/myplugins/jobentries/ftpplus/JobEntryFtpPlus.java b/kettle-sdk-jobentry-plugin/src/main/java/org/pentaho/di/sdk/myplugins/jobentries/ftpplus/JobEntryFtpPlus.java index 1708cf084..a0bd8526f 100644 --- a/kettle-sdk-jobentry-plugin/src/main/java/org/pentaho/di/sdk/myplugins/jobentries/ftpplus/JobEntryFtpPlus.java +++ b/kettle-sdk-jobentry-plugin/src/main/java/org/pentaho/di/sdk/myplugins/jobentries/ftpplus/JobEntryFtpPlus.java @@ -23,7 +23,6 @@ import org.w3c.dom.Node; import java.io.IOException; -import java.net.SocketException; import java.util.List; /** @@ -145,8 +144,4 @@ public void loadXML(Node entryNode, List databases, } } - - - - } diff --git a/kettle-sdk-jobentry-plugin/src/main/java/org/pentaho/di/sdk/myplugins/jobentries/ftpplus/JobEntryFtpPlusDialog.java b/kettle-sdk-jobentry-plugin/src/main/java/org/pentaho/di/sdk/myplugins/jobentries/ftpplus/JobEntryFtpPlusDialog.java index e209b0b6a..dc88af96d 100644 --- a/kettle-sdk-jobentry-plugin/src/main/java/org/pentaho/di/sdk/myplugins/jobentries/ftpplus/JobEntryFtpPlusDialog.java +++ b/kettle-sdk-jobentry-plugin/src/main/java/org/pentaho/di/sdk/myplugins/jobentries/ftpplus/JobEntryFtpPlusDialog.java @@ -1,6 +1,7 @@ package org.pentaho.di.sdk.myplugins.jobentries.ftpplus; import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONObject; import com.alibaba.fastjson.serializer.SerializerFeature; import org.apache.commons.lang.StringUtils; @@ -259,11 +260,9 @@ public void handleEvent(Event e) { String msg = "获取默认配置失败"; try { String confStr = jobEntry.getConfigInfo(); - //TODO json格式化显示 - //prettyConf = JSON.toJSONString(confStr,SerializerFeature.PrettyFormat); - //JSONObject jsonObject = JSON.parseObject(jobEntry.environmentSubstitute(confStr)); - //prettyConf = jsonObject.toJSONString(); - prettyConf = JSON.toJSONString(jobEntry.environmentSubstitute(confStr),SerializerFeature.PrettyFormat); + + //格式化显示json + prettyConf = JSON.toJSONString(jobEntry.getJobEntryFtpPlusParamsDO(),SerializerFeature.PrettyFormat); } catch (Exception e1) { msg = e1.getMessage();