From 39b0fef9a278ca5112f5565973abb52a5f8ede6f Mon Sep 17 00:00:00 2001 From: NShani Date: Wed, 19 Jun 2019 15:25:04 +0530 Subject: [PATCH 1/5] Add Callback Log Appender --- internal/scan-manager/scanners/common/pom.xml | 12 +++- .../common/logging/CallbackLogAppender.java | 54 ++++++++++++++ .../scanners/common/model/CallbackLog.java | 71 +++++++++++++++++++ .../src/main/resources/log4j.properties | 33 --------- .../common/src/main/resources/log4j2.xml | 44 ++++++++++++ .../scan-manager/scanners/veracode/pom.xml | 12 +++- .../src/main/resources/log4j.properties | 33 --------- 7 files changed, 187 insertions(+), 72 deletions(-) create mode 100644 internal/scan-manager/scanners/common/src/main/java/org/wso2/security/tools/scanmanager/scanners/common/logging/CallbackLogAppender.java create mode 100644 internal/scan-manager/scanners/common/src/main/java/org/wso2/security/tools/scanmanager/scanners/common/model/CallbackLog.java delete mode 100644 internal/scan-manager/scanners/common/src/main/resources/log4j.properties create mode 100644 internal/scan-manager/scanners/common/src/main/resources/log4j2.xml delete mode 100644 internal/scan-manager/scanners/veracode/src/main/resources/log4j.properties diff --git a/internal/scan-manager/scanners/common/pom.xml b/internal/scan-manager/scanners/common/pom.xml index 18141ba6..b60f03a9 100644 --- a/internal/scan-manager/scanners/common/pom.xml +++ b/internal/scan-manager/scanners/common/pom.xml @@ -78,9 +78,14 @@ ${org.apache.commons.version} - log4j - log4j - ${log4j.version} + org.apache.logging.log4j + log4j-core + ${org.apache.logging.log4j.version} + + + org.apache.logging.log4j + log4j-api + ${org.apache.logging.log4j.version} com.google.code.gson @@ -116,6 +121,7 @@ 0.1.54 2.12.0.wso2v1 1.16 + 2.11.0 4.5.3 2.8.2 1.0-SNAPSHOT diff --git a/internal/scan-manager/scanners/common/src/main/java/org/wso2/security/tools/scanmanager/scanners/common/logging/CallbackLogAppender.java b/internal/scan-manager/scanners/common/src/main/java/org/wso2/security/tools/scanmanager/scanners/common/logging/CallbackLogAppender.java new file mode 100644 index 00000000..2325c658 --- /dev/null +++ b/internal/scan-manager/scanners/common/src/main/java/org/wso2/security/tools/scanmanager/scanners/common/logging/CallbackLogAppender.java @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2019, WSO2 Inc., WSO2 Inc. (http://www.wso2.org) All Rights Reserved. + * + * WSO2 Inc. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.wso2.security.tools.scanmanager.scanners.common.logging; + +import org.apache.logging.log4j.core.LogEvent; +import org.apache.logging.log4j.core.appender.AbstractAppender; +import org.apache.logging.log4j.core.config.plugins.Plugin; +import org.apache.logging.log4j.core.config.plugins.PluginAttribute; +import org.apache.logging.log4j.core.config.plugins.PluginFactory; +import org.apache.logging.log4j.message.Message; +import org.wso2.security.tools.scanmanager.scanners.common.model.CallbackLog; +import org.wso2.security.tools.scanmanager.scanners.common.util.CallbackUtil; + +/** + * Log appender to persist the logs in the scan manager. + */ +@Plugin(category = "Core", name = "CallbackLogAppender") +public class CallbackLogAppender extends AbstractAppender { + + private CallbackLogAppender(String name) { + super(name, null, null); + } + + @PluginFactory + public static CallbackLogAppender createAppender(@PluginAttribute("name") String name) { + return new CallbackLogAppender(name); + } + + @Override + public void append(LogEvent event) { + Message message = event.getMessage(); + event.getLevel(); + if (message instanceof CallbackLog) { + CallbackLog callbackLog = (CallbackLog) event.getMessage(); + + CallbackUtil.persistScanLog(callbackLog.getJobId(), callbackLog.getMessage(), event.getLevel()); + } + } +} diff --git a/internal/scan-manager/scanners/common/src/main/java/org/wso2/security/tools/scanmanager/scanners/common/model/CallbackLog.java b/internal/scan-manager/scanners/common/src/main/java/org/wso2/security/tools/scanmanager/scanners/common/model/CallbackLog.java new file mode 100644 index 00000000..48e7cc16 --- /dev/null +++ b/internal/scan-manager/scanners/common/src/main/java/org/wso2/security/tools/scanmanager/scanners/common/model/CallbackLog.java @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2019, WSO2 Inc., WSO2 Inc. (http://www.wso2.org) All Rights Reserved. + * + * WSO2 Inc. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.wso2.security.tools.scanmanager.scanners.common.model; + +import org.apache.logging.log4j.message.Message; + +/** + * Model to represent the logging event message. + */ +public class CallbackLog implements Message { + private String jobId; + private String msg; + private static final Object[] NULL_OBJECT = {}; + private static final long serialVersionUID = 2L; + + public CallbackLog(String jobId, String msg) { + this.msg = msg; + this.jobId = jobId; + } + + public String getJobId() { + return jobId; + } + + public void setJobId(String jobId) { + this.jobId = jobId; + } + + public String getMessage() { + return msg; + } + + public void setMessage(String msg) { + this.msg = msg; + } + + @Override + public String getFormattedMessage() { + return msg; + } + + @Override + public String getFormat() { + return null; + } + + @Override + public Object[] getParameters() { + return NULL_OBJECT; + } + + @Override + public Throwable getThrowable() { + return null; + } +} diff --git a/internal/scan-manager/scanners/common/src/main/resources/log4j.properties b/internal/scan-manager/scanners/common/src/main/resources/log4j.properties deleted file mode 100644 index 55382f22..00000000 --- a/internal/scan-manager/scanners/common/src/main/resources/log4j.properties +++ /dev/null @@ -1,33 +0,0 @@ -# -# Copyright 2019 WSO2, Inc. (http://wso2.com) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Root logger option -log4j.rootLogger=INFO, stdout, file - -# Redirect log messages to console -log4j.appender.stdout=org.apache.log4j.ConsoleAppender -log4j.appender.stdout.Target=System.out -log4j.appender.stdout.layout=org.apache.log4j.PatternLayout -log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n - -# Redirect log messages to a log file -log4j.appender.file=org.apache.log4j.RollingFileAppender -#outputs to Tomcat home -log4j.appender.file.File=scan-manager-scanners-common.log -log4j.appender.file.MaxFileSize=5MB -log4j.appender.file.MaxBackupIndex=10 -log4j.appender.file.layout=org.apache.log4j.PatternLayout -log4j.appender.file.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n diff --git a/internal/scan-manager/scanners/common/src/main/resources/log4j2.xml b/internal/scan-manager/scanners/common/src/main/resources/log4j2.xml new file mode 100644 index 00000000..2caa86b3 --- /dev/null +++ b/internal/scan-manager/scanners/common/src/main/resources/log4j2.xml @@ -0,0 +1,44 @@ + + + + + + + + %d{yyyy-MMM-dd HH:mm:ss a} [%t] %-5level %logger{36} - %msg%n + + + + scanner.log + + + + + + + + + + + + + + + diff --git a/internal/scan-manager/scanners/veracode/pom.xml b/internal/scan-manager/scanners/veracode/pom.xml index de7a29b6..39aff97c 100644 --- a/internal/scan-manager/scanners/veracode/pom.xml +++ b/internal/scan-manager/scanners/veracode/pom.xml @@ -43,9 +43,14 @@ ${vosp.api.wrappers.version} - log4j - log4j - ${log4j.version} + org.apache.logging.log4j + log4j-core + ${org.apache.logging.log4j.version} + + + org.apache.logging.log4j + log4j-api + ${org.apache.logging.log4j.version} org.springframework @@ -133,6 +138,7 @@ 1.2.17 1.0-SNAPSHOT 1.0-SNAPSHOT + 2.11.0 1.0-SNAPSHOT 1.5.6.RELEASE 2.4.1 diff --git a/internal/scan-manager/scanners/veracode/src/main/resources/log4j.properties b/internal/scan-manager/scanners/veracode/src/main/resources/log4j.properties deleted file mode 100644 index 014503aa..00000000 --- a/internal/scan-manager/scanners/veracode/src/main/resources/log4j.properties +++ /dev/null @@ -1,33 +0,0 @@ -# -# Copyright 2019 WSO2, Inc. (http://wso2.com) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Root logger option -log4j.rootLogger=INFO, stdout, file - -# Redirect log messages to console -log4j.appender.stdout=org.apache.log4j.ConsoleAppender -log4j.appender.stdout.Target=System.out -log4j.appender.stdout.layout=org.apache.log4j.PatternLayout -log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n - -# Redirect log messages to a log file -log4j.appender.file=org.apache.log4j.RollingFileAppender -#outputs to Tomcat home -log4j.appender.file.File=scan-manager-scanners-veracode.log -log4j.appender.file.MaxFileSize=5MB -log4j.appender.file.MaxBackupIndex=10 -log4j.appender.file.layout=org.apache.log4j.PatternLayout -log4j.appender.file.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n From 69a3597d642ac58d6358b367c47bd15c47539861 Mon Sep 17 00:00:00 2001 From: NShani Date: Wed, 19 Jun 2019 16:05:13 +0530 Subject: [PATCH 2/5] Improve cancel scan process --- .../scanners/common/ScannerConstants.java | 12 +- .../scanners/common/ScannerController.java | 86 ++- .../scanners/common/util/CallbackUtil.java | 31 +- .../scanners/common/util/FileUtil.java | 10 +- .../veracode/VeracodeScannerConstants.java | 1 - .../scanners/veracode/handler/ScanTask.java | 552 +++++++++--------- .../veracode/src/main/resources/jarFilter.xml | 8 +- .../src/main/resources/scanner-config.yaml | 34 +- 8 files changed, 408 insertions(+), 326 deletions(-) diff --git a/internal/scan-manager/scanners/common/src/main/java/org/wso2/security/tools/scanmanager/scanners/common/ScannerConstants.java b/internal/scan-manager/scanners/common/src/main/java/org/wso2/security/tools/scanmanager/scanners/common/ScannerConstants.java index 332a1270..d1a80b50 100644 --- a/internal/scan-manager/scanners/common/src/main/java/org/wso2/security/tools/scanmanager/scanners/common/ScannerConstants.java +++ b/internal/scan-manager/scanners/common/src/main/java/org/wso2/security/tools/scanmanager/scanners/common/ScannerConstants.java @@ -27,7 +27,6 @@ public final class ScannerConstants { public static final String POM_FILE = "pom.xml"; public static final String PDF_FILE_EXTENSION = ".pdf"; public static final String XML_FILE_EXTENSION = ".xml"; - public static final String CALLBACK_RETRY_INTERVAL_SECONDS = "callback_retry_interval_seconds"; public static final String CALLBACK_RETRY_INCREASE_SECONDS = "callback_retry_interval_seconds"; public static final String CONFIGURTION_FILE_NAME = "scanner-config.yaml"; public static final String RESOURCE_FILE_PATH = "src/main/resources"; @@ -44,19 +43,12 @@ public final class ScannerConstants { // Scan manager config. public static final String SCAN_MANAGER_CALLBACK_URL_ENDPOINT = "scan_manager_callback_url_endpoint"; - public static final String SCAN_MANAGER_CALLBACK_URL = "scan_manager_callback_url"; public static final String SCAN_MANAGER_CALLBACK_STATUS = "scan_manager_callback_status"; public static final String SCAN_MANAGER_CALLBACK_LOG = "scan_manager_callback_log"; - public static final String SCAN_MANAGER_HOST = "scan.manager.host"; - public static final String SCAN_MANAGER_PORT = "scan.manager.port"; + public static final String SCAN_MANAGER_HOST = "SCAN_MANAGER_HOST"; + public static final String SCAN_MANAGER_PORT = "SCAN_MANAGER_PORT"; public static final String HTTP_PROTOCOL = "http://"; - // Log types. - public static final String INFO = "info"; - public static final String WARN = "warn"; - public static final String DEBUG = "debug"; - public static final String ERROR = "error"; - private ScannerConstants() { } } diff --git a/internal/scan-manager/scanners/common/src/main/java/org/wso2/security/tools/scanmanager/scanners/common/ScannerController.java b/internal/scan-manager/scanners/common/src/main/java/org/wso2/security/tools/scanmanager/scanners/common/ScannerController.java index a926b44f..001cf385 100644 --- a/internal/scan-manager/scanners/common/src/main/java/org/wso2/security/tools/scanmanager/scanners/common/ScannerController.java +++ b/internal/scan-manager/scanners/common/src/main/java/org/wso2/security/tools/scanmanager/scanners/common/ScannerController.java @@ -18,11 +18,13 @@ package org.wso2.security.tools.scanmanager.scanners.common; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Controller; +import org.springframework.util.StringUtils; import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; @@ -30,9 +32,8 @@ import org.springframework.web.bind.annotation.ResponseBody; import org.wso2.security.tools.scanmanager.common.internal.model.ScannerScanRequest; import org.wso2.security.tools.scanmanager.common.model.ErrorMessage; -import org.wso2.security.tools.scanmanager.common.model.LogType; +import org.wso2.security.tools.scanmanager.scanners.common.model.CallbackLog; import org.wso2.security.tools.scanmanager.scanners.common.service.Scanner; -import org.wso2.security.tools.scanmanager.scanners.common.util.CallbackUtil; import java.io.IOException; @@ -43,8 +44,15 @@ @RequestMapping("scanner") public class ScannerController { - private static final Logger log = Logger.getLogger(ScannerController.class); + private static final Logger log = LogManager.getLogger(ScannerController.class); Scanner scanner; + + // Scan task thread. + Thread startScanThread; + + // Cancel scan task thread. + Thread cancelScanThread; + // This represents if a scan is started. private boolean hasScanStarted = false; @@ -57,26 +65,50 @@ public ScannerController(Scanner scanner) throws IOException { /** * Start a new scan. * - * @param scanRequest Object that represent the required information for the scanner operation + * @param scannerScanRequest Object that represent the required information for the scanner operation * @return whether the start scan request is accepted */ @PostMapping("scan") @ResponseBody - public ResponseEntity startScan(@RequestBody ScannerScanRequest scanRequest) { + public ResponseEntity startScan(@RequestBody ScannerScanRequest scannerScanRequest) { ResponseEntity responseEntity; + responseEntity = validateStartScanReq(scannerScanRequest); + if (responseEntity.getStatusCode().equals(HttpStatus.ACCEPTED)) { + if (scanner.validateStartScan(scannerScanRequest)) { + log.info("Invoking start scan API."); + startScanThread = new Thread(() -> scanner.startScan(scannerScanRequest), "StartScanThread"); - if (!hasScanStarted) { - log.info("Invoking start scan API."); - responseEntity = scanner.startScan(scanRequest); - if (responseEntity.getStatusCode().equals(HttpStatus.ACCEPTED)) { + startScanThread.start(); hasScanStarted = true; + } else { + String message = "Start scan request validation is failed."; + responseEntity = new ResponseEntity<>(new ErrorMessage(HttpStatus.BAD_REQUEST.value(), + message), HttpStatus.BAD_REQUEST); + } + } + return responseEntity; + } + + private ResponseEntity validateStartScanReq(ScannerScanRequest scannerScanRequest) { + ResponseEntity responseEntity; + if (!hasScanStarted) { + if (!StringUtils.isEmpty(scannerScanRequest.getAppId())) { + if (!StringUtils.isEmpty(scannerScanRequest.getJobId())) { + responseEntity = new ResponseEntity<>(HttpStatus.ACCEPTED); + } else { + String message = "Job Id is missing in the request."; + responseEntity = new ResponseEntity<>(new ErrorMessage(HttpStatus.BAD_REQUEST.value(), + message), HttpStatus.BAD_REQUEST); + } + } else { + String message = "Application Id is missing in the request."; + responseEntity = new ResponseEntity<>(new ErrorMessage(HttpStatus.BAD_REQUEST.value(), + message), HttpStatus.BAD_REQUEST); } } else { String message = "Cannot start a new scan since another scan is in progress."; responseEntity = new ResponseEntity<>(new ErrorMessage(HttpStatus.BAD_REQUEST.value(), message), HttpStatus.BAD_REQUEST); - log.error(message); - CallbackUtil.persistScanLog(scanRequest.getJobId(), message, LogType.ERROR); } return responseEntity; } @@ -96,12 +128,36 @@ public ResponseEntity cancelScan(@RequestBody ScannerScanRequest scanRequest) { String message = "No scan running to perform cancellation."; responseEntity = new ResponseEntity<>(new ErrorMessage(HttpStatus.NOT_ACCEPTABLE.value(), message), HttpStatus.BAD_REQUEST); - log.error(message); - CallbackUtil.persistScanLog(scanRequest.getJobId(), message, LogType.ERROR); } else { log.info("Invoking cancel scan API."); - responseEntity = scanner.cancelScan(scanRequest); + if (scanner.validateCancelScan(scanRequest)) { + if (startScanThread != null) { + startScanThread.interrupt(); + } else { + log.info("There is no running scan thread to cancel."); + } + + cancelScanThread = new Thread(() -> { + stopStartScanThread(); + scanner.cancelScan(scanRequest); + }, "CancelScanThread"); + cancelScanThread.start(); + + responseEntity = new ResponseEntity<>(HttpStatus.ACCEPTED); + } else { + String message = "Cancel scan request validation is failed."; + responseEntity = new ResponseEntity<>(new ErrorMessage(HttpStatus.BAD_REQUEST.value(), + message), HttpStatus.BAD_REQUEST); + } } return responseEntity; } + + private boolean stopStartScanThread() { + + while (startScanThread.isAlive()) { + // run until the start scan thread is dead. + } + return true; + } } diff --git a/internal/scan-manager/scanners/common/src/main/java/org/wso2/security/tools/scanmanager/scanners/common/util/CallbackUtil.java b/internal/scan-manager/scanners/common/src/main/java/org/wso2/security/tools/scanmanager/scanners/common/util/CallbackUtil.java index bcbbb41e..031dcb1c 100644 --- a/internal/scan-manager/scanners/common/src/main/java/org/wso2/security/tools/scanmanager/scanners/common/util/CallbackUtil.java +++ b/internal/scan-manager/scanners/common/src/main/java/org/wso2/security/tools/scanmanager/scanners/common/util/CallbackUtil.java @@ -17,7 +17,9 @@ */ package org.wso2.security.tools.scanmanager.scanners.common.util; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.web.client.RestClientException; @@ -35,7 +37,7 @@ */ public class CallbackUtil { - private static final Logger log = Logger.getLogger(CallbackUtil.class); + private static final Logger log = LogManager.getLogger(CallbackUtil.class); private static String scanManagerLogCallbackURL; private static String scanManagerStatusCallbackURL; private static Long retryTimeInterval = Long.valueOf(0); @@ -97,7 +99,7 @@ public static void updateScanStatus(String jobId, ScanStatus scanStatus, String try { log.info("Callback log endpoint is not currently available and will retry after " + statusUpdateRetryTimeInterval + " Seconds"); - TimeUnit.MINUTES.sleep(statusUpdateRetryTimeInterval); + TimeUnit.SECONDS.sleep(statusUpdateRetryTimeInterval); } catch (InterruptedException e) { log.error(e); } @@ -113,9 +115,26 @@ public static void updateScanStatus(String jobId, ScanStatus scanStatus, String * * @param jobId id of the scan manager for the current scan * @param message log message - * @param type log type + * @param level log level */ - public static void persistScanLog(String jobId, String message, LogType type) { + public static void persistScanLog(String jobId, String message, Level level) { + LogType type; + switch (level.toString()) { + case "INFO": + type = LogType.INFO; + break; + case "DEBUG": + type = LogType.DEBUG; + break; + case "WARN": + type = LogType.WARN; + break; + case "ERROR": + type = LogType.ERROR; + break; + default: + type = null; + } persistScanLog(jobId, message, type, Long.valueOf(0)); } @@ -151,7 +170,7 @@ public static void persistScanLog(String jobId, String message, LogType type, Lo log.info("Callback log endpoint is not currently available and will retry after " + retryTimeInterval + " Seconds"); - TimeUnit.MINUTES.sleep(retryTimeInterval); + TimeUnit.SECONDS.sleep(retryTimeInterval); } catch (InterruptedException e) { log.error(e); } diff --git a/internal/scan-manager/scanners/common/src/main/java/org/wso2/security/tools/scanmanager/scanners/common/util/FileUtil.java b/internal/scan-manager/scanners/common/src/main/java/org/wso2/security/tools/scanmanager/scanners/common/util/FileUtil.java index 7936ab0d..686d854d 100644 --- a/internal/scan-manager/scanners/common/src/main/java/org/wso2/security/tools/scanmanager/scanners/common/util/FileUtil.java +++ b/internal/scan-manager/scanners/common/src/main/java/org/wso2/security/tools/scanmanager/scanners/common/util/FileUtil.java @@ -155,7 +155,7 @@ public static void zipFiles(String source, String destination) throws ArchiveExc * @throws SftpException when unable to connect to the FTP server */ public static void downloadFromFtp(String filePathInFtp, String fileName, File outputFile, String ftpUsername - , char[] ftpPassword, String ftpHost, int ftpPort) throws IOException, JSchException, SftpException { + , char[] ftpPassword, String ftpHost, int ftpPort) throws JSchException, SftpException, IOException { ChannelSftp sftp = openFtpLocation(filePathInFtp, ftpUsername, ftpPassword, ftpHost, ftpPort); downloadFromFtp(sftp.get(fileName), outputFile); @@ -215,10 +215,10 @@ public static boolean saveReport(byte[] bytesResult, String filePath) throws Fil * Open the FTP location of the file and return the created channel. * * @param filePathInFtp path to the file - * @param ftpUsername username of the ftp location where file is located - * @param ftpPassword password of the ftp location where file is located - * @param ftpHost host of the ftp location where file is located - * @param ftpPort port of the ftp location where file is located + * @param ftpUsername username of the ftp location where file is located + * @param ftpPassword password of the ftp location where file is located + * @param ftpHost host of the ftp location where file is located + * @param ftpPort port of the ftp location where file is located * @return SFTP channel to access the FTP location * @throws JSchException when unable to create the session for connecting the FTP server * @throws SftpException when unable to connect to the FTP server diff --git a/internal/scan-manager/scanners/veracode/src/main/java/org/wso2/security/tools/scanmanager/scanners/veracode/VeracodeScannerConstants.java b/internal/scan-manager/scanners/veracode/src/main/java/org/wso2/security/tools/scanmanager/scanners/veracode/VeracodeScannerConstants.java index 8a66bfef..17d58e90 100644 --- a/internal/scan-manager/scanners/veracode/src/main/java/org/wso2/security/tools/scanmanager/scanners/veracode/VeracodeScannerConstants.java +++ b/internal/scan-manager/scanners/veracode/src/main/java/org/wso2/security/tools/scanmanager/scanners/veracode/VeracodeScannerConstants.java @@ -30,7 +30,6 @@ public final class VeracodeScannerConstants { public static final String VERACODE_LOG_FILE_PATH = "log_filepath"; public static final String SCANNER_BEAN_CLASS_NAME = "scanner_bean_class"; public static final String DEFAULT_PRODUCT_PATH = "default_product_path"; - public static final String DEFAULT_GIT_PRODUCT_PATH = "default_git_product_path"; public static final String JAR_FILTER_PATTERN_FILE_PATH = "jar_filter_pattern_file_path"; public static final String PREFIX = "prefix"; public static final String SUFFIX = "suffix"; diff --git a/internal/scan-manager/scanners/veracode/src/main/java/org/wso2/security/tools/scanmanager/scanners/veracode/handler/ScanTask.java b/internal/scan-manager/scanners/veracode/src/main/java/org/wso2/security/tools/scanmanager/scanners/veracode/handler/ScanTask.java index 8eb7b6e9..d47cb119 100644 --- a/internal/scan-manager/scanners/veracode/src/main/java/org/wso2/security/tools/scanmanager/scanners/veracode/handler/ScanTask.java +++ b/internal/scan-manager/scanners/veracode/src/main/java/org/wso2/security/tools/scanmanager/scanners/veracode/handler/ScanTask.java @@ -22,15 +22,16 @@ import com.veracode.apiwrapper.wrappers.ResultsAPIWrapper; import com.veracode.apiwrapper.wrappers.UploadAPIWrapper; import org.apache.commons.compress.archivers.ArchiveException; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.w3c.dom.Document; import org.w3c.dom.Element; import org.w3c.dom.Node; import org.w3c.dom.NodeList; -import org.wso2.security.tools.scanmanager.common.model.LogType; import org.wso2.security.tools.scanmanager.common.model.ScanStatus; import org.wso2.security.tools.scanmanager.scanners.common.ScannerConstants; import org.wso2.security.tools.scanmanager.scanners.common.exception.ScannerException; +import org.wso2.security.tools.scanmanager.scanners.common.model.CallbackLog; import org.wso2.security.tools.scanmanager.scanners.common.util.CallbackUtil; import org.wso2.security.tools.scanmanager.scanners.common.util.ErrorProcessingUtil; import org.wso2.security.tools.scanmanager.scanners.common.util.FileUtil; @@ -45,6 +46,7 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; +import java.io.InterruptedIOException; import java.io.OutputStream; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -59,9 +61,9 @@ /** * Represents the scan handling tasks. */ -public class ScanTask implements Runnable { +public class ScanTask { - private static final Logger log = Logger.getLogger(ScanTask.class); + private static final Logger log = LogManager.getLogger(ScanTask.class); // Scan request coming to the scan micro-service API. private ScanContext scanContext; @@ -73,33 +75,29 @@ public ScanTask(ScanContext scanContext) { if (log.isDebugEnabled()) { String logMessage = "Upload Artifact Handler thread is being initialized for the application:" + scanContext.getAppId(); - log.debug(logMessage); - CallbackUtil.persistScanLog(scanContext.getJobId(), logMessage, LogType.DEBUG); + log.debug(new CallbackLog(scanContext.getJobId(), logMessage)); } this.scanContext = scanContext; } - @Override public void run() { + if (isScanRunning()) { // If another scans is running on the actual Veracode cloud scanner, then the start scan request // would be failed and Scan status is updated as 'ERROR'. - CallbackUtil.updateScanStatus(scanContext.getJobId(), ScanStatus.ERROR, null, null); String logMessage = "Currently another scan is running on the application id : " + scanContext - .getAppId() + ". So please check this scan before proceed " + "with another scan."; - log.error(logMessage); - CallbackUtil.persistScanLog(scanContext.getJobId(), logMessage, LogType.ERROR); + .getAppId() + ". So please check this scan before proceed with another scan."; + log.error(new CallbackLog(scanContext.getJobId(), logMessage)); - Thread.currentThread().interrupt(); + CallbackUtil.updateScanStatus(scanContext.getJobId(), ScanStatus.ERROR, null, null); } if (handleUploadingTask()) { if (handleResultProcessTask()) { String logMessage = "Start scan process is successfully completed for the application: " + scanContext.getAppId(); - log.info(logMessage); - CallbackUtil.persistScanLog(scanContext.getJobId(), logMessage, LogType.INFO); + log.info(new CallbackLog(scanContext.getJobId(), logMessage)); } else { Thread.currentThread().interrupt(); } @@ -131,9 +129,7 @@ private boolean handleUploadingTask() { } if (!isUploadSuccess) { logMessage = logMessage.concat(" Terminating scan for app: " + scanContext.getAppId()); - log.error(logMessage); - CallbackUtil.persistScanLog(scanContext.getJobId(), logMessage, LogType.ERROR); - CallbackUtil.updateScanStatus(scanContext.getJobId(), ScanStatus.ERROR, null, null); + log.error(new CallbackLog(scanContext.getJobId(), logMessage)); } return isUploadSuccess; } @@ -161,9 +157,7 @@ private boolean handleResultProcessTask() { } if (!isResultsUploaded) { logMessage = logMessage + (" Terminating scan for app: " + scanContext.getAppId()); - log.error(logMessage); - CallbackUtil.persistScanLog(scanContext.getJobId(), logMessage, LogType.ERROR); - CallbackUtil.updateScanStatus(scanContext.getJobId(), ScanStatus.ERROR, null, null); + log.error(new CallbackLog(scanContext.getJobId(), logMessage)); } return isResultsUploaded; } @@ -178,20 +172,24 @@ private boolean isScanRunning() { String result; ScanStatus currentScanStatus; - try { - UploadAPIWrapper uploadAPIWrapper = VeracodeAPIUtil.getUploadAPIWrapper(); - result = uploadAPIWrapper.getBuildInfo(scanContext.getAppId()); - currentScanStatus = VeracodeResultProcessor.getScanStatus(result); - if (ScanStatus.RUNNING.equals(currentScanStatus)) { - isScanRunning = true; - } - } catch (IOException | ParserConfigurationException | XPathExpressionException | SAXException e) { - String logMessage = "Error occured while retrieving the scan status for application : " + scanContext - .getAppId() + " " + ErrorProcessingUtil.getFullErrorMessage(e); - log.error(logMessage); - CallbackUtil.persistScanLog(scanContext.getJobId(), logMessage, LogType.ERROR); + if (!Thread.currentThread().isInterrupted()) { + try { + UploadAPIWrapper uploadAPIWrapper = VeracodeAPIUtil.getUploadAPIWrapper(); + result = uploadAPIWrapper.getBuildInfo(scanContext.getAppId()); + currentScanStatus = VeracodeResultProcessor.getScanStatus(result); + if (ScanStatus.RUNNING.equals(currentScanStatus)) { + isScanRunning = true; + } + } catch (IOException | ParserConfigurationException | XPathExpressionException | SAXException e) { + String logMessage = "Error occured while retrieving the scan status for application : " + scanContext + .getAppId() + " " + ErrorProcessingUtil.getFullErrorMessage(e); + log.error(new CallbackLog(scanContext.getJobId(), logMessage)); - CallbackUtil.updateScanStatus(scanContext.getJobId(), ScanStatus.ERROR, null, null); + CallbackUtil.updateScanStatus(scanContext.getJobId(), ScanStatus.ERROR, null, null); + } + } else { + String logMessage = "Current thread is interrupted for application : " + scanContext.getAppId(); + log.error(new CallbackLog(scanContext.getJobId(), logMessage)); } return isScanRunning; } @@ -204,24 +202,28 @@ private boolean isScanRunning() { private boolean cleanPreviousScans() { boolean isScannerCleaned = false; String result; - try { - UploadAPIWrapper uploadAPIWrapper = VeracodeAPIUtil.getUploadAPIWrapper(); - result = uploadAPIWrapper.deleteBuild(scanContext.getAppId()); - if (log.isDebugEnabled()) { - String logMessage = "Deleted the last scan of the application :" + scanContext.getAppId(); - log.debug(logMessage); - CallbackUtil.persistScanLog(scanContext.getJobId(), logMessage, LogType.DEBUG); - } + if (!Thread.currentThread().isInterrupted()) { + try { + UploadAPIWrapper uploadAPIWrapper = VeracodeAPIUtil.getUploadAPIWrapper(); + result = uploadAPIWrapper.deleteBuild(scanContext.getAppId()); - isScannerCleaned = VeracodeResultProcessor.isOperationProceedWithoutError(result); - } catch (IOException e) { - String logMessage = "Error occured while deleting the scan status for application : " + scanContext - .getAppId() + " " + ErrorProcessingUtil.getFullErrorMessage(e); - log.error(logMessage); - CallbackUtil.persistScanLog(scanContext.getJobId(), logMessage, LogType.ERROR); + if (log.isDebugEnabled()) { + String logMessage = "Deleted the last scan of the application :" + scanContext.getAppId(); + log.debug(new CallbackLog(scanContext.getJobId(), logMessage)); + } - CallbackUtil.updateScanStatus(scanContext.getJobId(), ScanStatus.ERROR, null, null); + isScannerCleaned = VeracodeResultProcessor.isOperationProceedWithoutError(result); + } catch (IOException e) { + String logMessage = "Error occured while deleting the scan status for application : " + scanContext + .getAppId() + " " + ErrorProcessingUtil.getFullErrorMessage(e); + log.error(new CallbackLog(scanContext.getJobId(), logMessage)); + + CallbackUtil.updateScanStatus(scanContext.getJobId(), ScanStatus.ERROR, null, null); + } + } else { + String logMessage = "Current thread is interrupted for application : " + scanContext.getAppId(); + log.error(new CallbackLog(scanContext.getJobId(), logMessage)); } return isScannerCleaned; } @@ -242,58 +244,58 @@ private boolean creatingScanArtifactZip() { File productFile = new File(VeracodeScannerConfiguration.getInstance().getConfigProperty( ScannerConstants.DEFAULT_FTP_PRODUCT_PATH) + productPackName); + if (!Thread.currentThread().isInterrupted()) { + try { + String logMessage = "Product pack is downloading for the application: " + scanContext.getAppId(); + log.info(new CallbackLog(scanContext.getJobId(), logMessage)); + + FileUtil.downloadFromFtp(productPath, productPackName, productFile, VeracodeScannerConfiguration + .getInstance().getConfigProperty(ScannerConstants.FTP_USERNAME), + (VeracodeScannerConfiguration.getInstance().getConfigProperty(ScannerConstants.FTP_PASSWORD)) + .toCharArray(), VeracodeScannerConfiguration.getInstance().getConfigProperty( + ScannerConstants.FTP_HOST), Integer.parseInt(VeracodeScannerConfiguration.getInstance() + .getConfigProperty(ScannerConstants.FTP_PORT))); + + logMessage = "Product downloading completed for the application: " + scanContext.getAppId() + " into " + + productFile; + log.info(new CallbackLog(scanContext.getJobId(), logMessage)); + + extractedFilePath = FileUtil.extractArchive(productFile, productFile.getParent()); + workingDirectory = new File(extractedFilePath + VeracodeScannerConstants.WORK_DIRECTORY_SUFIX); + + if (workingDirectory.mkdirs()) { + logMessage = "Filtering the artifacts for the scan for the application: " + scanContext.getAppId() + + " into " + workingDirectory; + log.info(new CallbackLog(scanContext.getJobId(), logMessage)); + + copyRequiredScanArtifact(extractedFilePath); + FileUtil.zipFiles(workingDirectory.getAbsolutePath(), workingDirectory.getAbsolutePath() + + ScannerConstants.ZIP_FILE_EXTENSION); + isZipCreated = true; + + logMessage = "Created the zip artifact for the scan for the application: " + scanContext.getAppId() + + " as " + workingDirectory + ScannerConstants.ZIP_FILE_EXTENSION; + log.info(new CallbackLog(scanContext.getJobId(), logMessage)); + } else { + logMessage = "Error occured while creating the working directory for application : " + scanContext + .getAppId(); + log.error(new CallbackLog(scanContext.getJobId(), logMessage)); - try { - String logMessage = "Product pack is downloading for the application: " + scanContext.getAppId(); - log.info(logMessage); - CallbackUtil.persistScanLog(scanContext.getJobId(), logMessage, LogType.INFO); - - FileUtil.downloadFromFtp(productPath, productPackName, productFile, VeracodeScannerConfiguration - .getInstance().getConfigProperty(ScannerConstants.FTP_USERNAME), - (VeracodeScannerConfiguration.getInstance().getConfigProperty(ScannerConstants.FTP_PASSWORD)) - .toCharArray(), VeracodeScannerConfiguration.getInstance().getConfigProperty( - ScannerConstants.FTP_HOST), Integer.parseInt(VeracodeScannerConfiguration.getInstance() - .getConfigProperty(ScannerConstants.FTP_PORT))); - - logMessage = "Product downloading completed for the application: " + scanContext.getAppId() + " into " - + productFile; - log.info(logMessage); - CallbackUtil.persistScanLog(scanContext.getJobId(), logMessage, LogType.INFO); - - extractedFilePath = FileUtil.extractArchive(productFile, productFile.getParent()); - workingDirectory = new File(extractedFilePath + VeracodeScannerConstants.WORK_DIRECTORY_SUFIX); - - if (workingDirectory.mkdirs()) { - logMessage = "Filtering the artifacts for the scan for the application: " + scanContext.getAppId() + - " into " + workingDirectory; - log.info(logMessage); - CallbackUtil.persistScanLog(scanContext.getJobId(), logMessage, LogType.INFO); - - copyRequiredScanArtifact(extractedFilePath); - FileUtil.zipFiles(workingDirectory.getAbsolutePath(), workingDirectory.getAbsolutePath() - + ScannerConstants.ZIP_FILE_EXTENSION); - isZipCreated = true; - - logMessage = "Created the zip artifact for the scan for the application: " + scanContext.getAppId() + - " as " + workingDirectory + ScannerConstants.ZIP_FILE_EXTENSION; - log.info(logMessage); - CallbackUtil.persistScanLog(scanContext.getJobId(), logMessage, LogType.INFO); - } else { - logMessage = "Error occured while creating the working directory for application : " + scanContext - .getAppId(); - log.error(logMessage); - CallbackUtil.persistScanLog(scanContext.getJobId(), logMessage, LogType.ERROR); + CallbackUtil.updateScanStatus(scanContext.getJobId(), ScanStatus.ERROR, null, null); + } + } catch (IOException | JSchException | SftpException | SAXException | ScannerException | ArchiveException | + ParserConfigurationException e) { + String logMessage = "Error occured while creating the scan zip artifact for application : " + + scanContext.getAppId() + "\n" + ErrorProcessingUtil.getFullErrorMessage(e); + log.error(new CallbackLog(scanContext.getJobId(), logMessage)); - CallbackUtil.updateScanStatus(scanContext.getJobId(), ScanStatus.ERROR, null, null); + if (!e.getClass().isInstance(InterruptedIOException.class)) { + CallbackUtil.updateScanStatus(scanContext.getJobId(), ScanStatus.ERROR, null, null); + } } - } catch (IOException | JSchException | SftpException | SAXException | ScannerException | ArchiveException | - ParserConfigurationException e) { - String logMessage = "Error occured while creating the scan zip artifact for application : " + scanContext - .getAppId() + " " + ErrorProcessingUtil.getFullErrorMessage(e); - log.error(e); - CallbackUtil.persistScanLog(scanContext.getJobId(), logMessage, LogType.ERROR); - - CallbackUtil.updateScanStatus(scanContext.getJobId(), ScanStatus.ERROR, null, null); + } else { + String logMessage = "Current thread is interrupted for application : " + scanContext.getAppId(); + log.error(new CallbackLog(scanContext.getJobId(), logMessage)); } return isZipCreated; } @@ -307,35 +309,37 @@ private boolean uploadScanArtifact() { boolean isUploadSuccess = false; String buildId; - try { - UploadAPIWrapper uploadAPIWrapper = VeracodeAPIUtil.getUploadAPIWrapper(); - String result = uploadAPIWrapper.uploadFile(scanContext.getAppId(), workingDirectory - + ScannerConstants.ZIP_FILE_EXTENSION); - isUploadSuccess = VeracodeResultProcessor.isOperationProceedWithoutError(result); - if (isUploadSuccess) { - result = uploadAPIWrapper.getBuildInfo(scanContext.getAppId()); - buildId = VeracodeResultProcessor.getBuildIdByResponse(result); + if (!Thread.currentThread().isInterrupted()) { + try { + UploadAPIWrapper uploadAPIWrapper = VeracodeAPIUtil.getUploadAPIWrapper(); + String result = uploadAPIWrapper.uploadFile(scanContext.getAppId(), workingDirectory + + ScannerConstants.ZIP_FILE_EXTENSION); + isUploadSuccess = VeracodeResultProcessor.isOperationProceedWithoutError(result); + if (isUploadSuccess) { + result = uploadAPIWrapper.getBuildInfo(scanContext.getAppId()); + buildId = VeracodeResultProcessor.getBuildIdByResponse(result); + + CallbackUtil.updateScanStatus(scanContext.getJobId(), ScanStatus.RUNNING, null, buildId); + String logMessage = "Product scan artifacts were uploaded to Veracode scanner for the application: " + + scanContext.getAppId(); + log.info(new CallbackLog(scanContext.getJobId(), logMessage)); + } else { + String logMessage = "Product scan artifacts uploading was failed to Veracode scanner for the " + + "application : " + scanContext.getAppId(); + log.error(new CallbackLog(scanContext.getJobId(), logMessage)); - CallbackUtil.updateScanStatus(scanContext.getJobId(), ScanStatus.RUNNING, null, buildId); - String logMessage = "Product scan artifacts were uploaded to Veracode scanner for the application: " - + scanContext.getAppId(); - log.info(logMessage); - CallbackUtil.persistScanLog(scanContext.getJobId(), logMessage, LogType.INFO); - } else { + CallbackUtil.updateScanStatus(scanContext.getJobId(), ScanStatus.ERROR, null, null); + } + } catch (IOException | ParserConfigurationException | XPathExpressionException | SAXException e) { String logMessage = "Product scan artifacts uploading was failed to Veracode scanner for the " + - "application : " + scanContext.getAppId(); - log.error(logMessage); - CallbackUtil.persistScanLog(scanContext.getJobId(), logMessage, LogType.ERROR); + "application : " + scanContext.getAppId() + " " + ErrorProcessingUtil.getFullErrorMessage(e); + log.error(new CallbackLog(scanContext.getJobId(), logMessage)); CallbackUtil.updateScanStatus(scanContext.getJobId(), ScanStatus.ERROR, null, null); } - } catch (IOException | ParserConfigurationException | XPathExpressionException | SAXException e) { - String logMessage = "Product scan artifacts uploading was failed to Veracode scanner for the " + - "application : " + scanContext.getAppId() + " " + ErrorProcessingUtil.getFullErrorMessage(e); - log.error(logMessage); - CallbackUtil.persistScanLog(scanContext.getJobId(), logMessage, LogType.ERROR); - - CallbackUtil.updateScanStatus(scanContext.getJobId(), ScanStatus.ERROR, null, null); + } else { + String logMessage = "Current thread is interrupted for application : " + scanContext.getAppId(); + log.error(new CallbackLog(scanContext.getJobId(), logMessage)); } return isUploadSuccess; } @@ -354,35 +358,40 @@ private void copyRequiredScanArtifact(String filePath) throws IOException, SAXEx File[] files = dir.listFiles(); File patternXmlFile = new File(JAR_FILTER_FILE); - try (InputStream input = VeracodeScannerConfiguration.class.getClassLoader() - .getResourceAsStream(JAR_FILTER_FILE); - OutputStream out = new FileOutputStream(patternXmlFile)) { - int read; - byte[] bytes = new byte[1024]; + if (!Thread.currentThread().isInterrupted()) { + try (InputStream input = VeracodeScannerConfiguration.class.getClassLoader() + .getResourceAsStream(JAR_FILTER_FILE); + OutputStream out = new FileOutputStream(patternXmlFile)) { + int read; + byte[] bytes = new byte[1024]; - while ((read = input.read(bytes)) != -1) { - out.write(bytes, 0, read); + while ((read = input.read(bytes)) != -1) { + out.write(bytes, 0, read); + } + } catch (IOException e) { + throw new IOException(e); } - } catch (IOException e) { - throw new IOException(e); - } - NodeList nodeList = getScanArtifactPatternList(patternXmlFile); - - if (files != null) { - for (File file : files) { - if (file.isFile()) { - for (int i = 0; i < nodeList.getLength(); i++) { - Node node = nodeList.item(i); - Element element = (Element) node; - - checkFileNamePattern(element, file); + NodeList nodeList = getScanArtifactPatternList(patternXmlFile); + + if (files != null) { + for (File file : files) { + if (file.isFile()) { + for (int i = 0; i < nodeList.getLength(); i++) { + Node node = nodeList.item(i); + Element element = (Element) node; + + checkFileNamePattern(element, file); + } + } else if (file.isDirectory()) { + copyRequiredScanArtifact(file.getAbsolutePath()); } - } else if (file.isDirectory()) { - copyRequiredScanArtifact(file.getAbsolutePath()); } + } else { + log.warn("File list that needs to be archived cannot be null."); } } else { - log.warn("File list that needs to be archived cannot be null."); + String logMessage = "Current thread is interrupted for application : " + scanContext.getAppId(); + log.error(new CallbackLog(scanContext.getJobId(), logMessage)); } } @@ -435,8 +444,7 @@ private void checkFileNamePattern(Element element, File file) throws IOException if ((e).toString().startsWith("java.nio.file.FileAlreadyExistsException")) { String logMessage = "Error occured while copying file. \nWarning Message : " + ErrorProcessingUtil.getFullErrorMessage(e); - log.warn(logMessage); - CallbackUtil.persistScanLog(scanContext.getJobId(), logMessage, LogType.WARN); + log.warn(new CallbackLog(scanContext.getJobId(), logMessage)); } else { throw e; } @@ -454,34 +462,36 @@ private boolean beginPreScan() { String buildId = null; boolean isPreScanStarted = false; - try { - UploadAPIWrapper uploadAPIWrapper = VeracodeAPIUtil.getUploadAPIWrapper(); - result = uploadAPIWrapper.beginPreScan(scanContext.getAppId(), null, "true", "true"); - isPreScanStarted = VeracodeResultProcessor.isOperationProceedWithoutError(result); + if (!Thread.currentThread().isInterrupted()) { + try { + UploadAPIWrapper uploadAPIWrapper = VeracodeAPIUtil.getUploadAPIWrapper(); + result = uploadAPIWrapper.beginPreScan(scanContext.getAppId(), null, "true", "true"); + isPreScanStarted = VeracodeResultProcessor.isOperationProceedWithoutError(result); - if (isPreScanStarted) { - buildId = VeracodeResultProcessor.getBuildIdByResponse(result); - String logMessage = "Pre-Scan is started in Veracode scanner for the application: " + - scanContext.getAppId(); - log.info(logMessage); - CallbackUtil.persistScanLog(scanContext.getJobId(), logMessage, LogType.INFO); + if (isPreScanStarted) { + buildId = VeracodeResultProcessor.getBuildIdByResponse(result); + String logMessage = "Pre-Scan is started in Veracode scanner for the application: " + + scanContext.getAppId(); + log.info(new CallbackLog(scanContext.getJobId(), logMessage)); - CallbackUtil.updateScanStatus(scanContext.getJobId(), ScanStatus.RUNNING, null, buildId); - } else { - String logMessage = "Pre-Scan is failed in Veracode scanner for the application: " + - scanContext.getAppId(); - log.error(logMessage); - CallbackUtil.persistScanLog(scanContext.getJobId(), logMessage, LogType.ERROR); + CallbackUtil.updateScanStatus(scanContext.getJobId(), ScanStatus.RUNNING, null, buildId); + } else { + String logMessage = "Pre-Scan is failed in Veracode scanner for the application: " + + scanContext.getAppId(); + log.error(new CallbackLog(scanContext.getJobId(), logMessage)); + + CallbackUtil.updateScanStatus(scanContext.getJobId(), ScanStatus.ERROR, null, buildId); + } + } catch (IOException | ParserConfigurationException | XPathExpressionException | SAXException e) { + String logMessage = "Pre-Scan is failed in Veracode scanner for the application: " + + scanContext.getAppId() + " " + ErrorProcessingUtil.getFullErrorMessage(e); + log.error(new CallbackLog(scanContext.getJobId(), logMessage)); CallbackUtil.updateScanStatus(scanContext.getJobId(), ScanStatus.ERROR, null, buildId); } - } catch (IOException | ParserConfigurationException | XPathExpressionException | SAXException e) { - String logMessage = "Pre-Scan is failed in Veracode scanner for the application: " - + scanContext.getAppId() + " " + ErrorProcessingUtil.getFullErrorMessage(e); - log.error(logMessage); - CallbackUtil.persistScanLog(scanContext.getJobId(), logMessage, LogType.ERROR); - - CallbackUtil.updateScanStatus(scanContext.getJobId(), ScanStatus.ERROR, null, buildId); + } else { + String logMessage = "Current thread is interrupted for application : " + scanContext.getAppId(); + log.error(new CallbackLog(scanContext.getJobId(), logMessage)); } return isPreScanStarted; } @@ -495,28 +505,32 @@ private boolean beginScan() { String result; boolean isScanStarted = false; - try { - UploadAPIWrapper uploadAPIWrapper = VeracodeAPIUtil.getUploadAPIWrapper(); - result = uploadAPIWrapper.beginScan(scanContext.getAppId(), "all", "true"); - isScanStarted = VeracodeResultProcessor.isOperationProceedWithoutError(result); - - if (isScanStarted) { - String logMessage = "Scan is started on the Veracode for the application: " + scanContext.getAppId(); - log.info(logMessage); - CallbackUtil.persistScanLog(scanContext.getJobId(), logMessage, LogType.INFO); - } else { - String logMessage = "Starting Scan failed in Veracode scanner for the application: " + scanContext - .getAppId(); - log.error(logMessage); - CallbackUtil.persistScanLog(scanContext.getJobId(), logMessage, LogType.ERROR); + if (!Thread.currentThread().isInterrupted()) { + try { + UploadAPIWrapper uploadAPIWrapper = VeracodeAPIUtil.getUploadAPIWrapper(); + result = uploadAPIWrapper.beginScan(scanContext.getAppId(), "all", "true"); + isScanStarted = VeracodeResultProcessor.isOperationProceedWithoutError(result); + + if (isScanStarted) { + String logMessage = "Scan is started on Veracode for the application: " + scanContext.getAppId(); + log.info(new CallbackLog(scanContext.getJobId(), logMessage)); + } else { + String logMessage = "Starting Scan failed in Veracode scanner for the application: " + scanContext + .getAppId(); + log.error(new CallbackLog(scanContext.getJobId(), logMessage)); + + CallbackUtil.updateScanStatus(scanContext.getJobId(), ScanStatus.ERROR, null, null); + } + } catch (IOException e) { + String logMessage = "Starting scan failed in Veracode scanner for the application: " + + scanContext.getAppId() + " " + ErrorProcessingUtil.getFullErrorMessage(e); + log.error(new CallbackLog(scanContext.getJobId(), logMessage)); + CallbackUtil.updateScanStatus(scanContext.getJobId(), ScanStatus.ERROR, null, null); } - } catch (IOException e) { - String logMessage = "Starting scan failed in Veracode scanner for the application: " - + scanContext.getAppId() + " " + ErrorProcessingUtil.getFullErrorMessage(e); - log.error(logMessage); - CallbackUtil.persistScanLog(scanContext.getJobId(), logMessage, LogType.ERROR); - CallbackUtil.updateScanStatus(scanContext.getJobId(), ScanStatus.ERROR, null, null); + } else { + String logMessage = "Current thread is interrupted for application : " + scanContext.getAppId(); + log.error(new CallbackLog(scanContext.getJobId(), logMessage)); } return isScanStarted; } @@ -532,21 +546,19 @@ private boolean getScanReport() { if (log.isDebugEnabled()) { String logMessage = "Scan started in Veracode for the application: " + scanContext.getAppId(); - log.debug(logMessage); - CallbackUtil.persistScanLog(scanContext.getJobId(), logMessage, LogType.DEBUG); + log.debug(new CallbackLog(scanContext.getJobId(), logMessage)); } try { scanStatus = getScanStatus(); - while (!scanStatus.equals(ScanStatus.COMPLETED)) { + while (!scanStatus.equals(ScanStatus.COMPLETED) && !Thread.currentThread().isInterrupted()) { if (scanStatus.equals(ScanStatus.ERROR)) { break; } String logMessage = "Waiting for " + VeracodeScannerConfiguration.getInstance().getConfigProperty( VeracodeScannerConstants.SCAN_RESULT_RETRY_MINS) + " mins until the scan is completed" + " for the application: " + scanContext.getAppId(); - log.info(logMessage); - CallbackUtil.persistScanLog(scanContext.getJobId(), logMessage, LogType.INFO); + log.info(new CallbackLog(scanContext.getJobId(), logMessage)); TimeUnit.MINUTES.sleep(Integer.parseInt(VeracodeScannerConfiguration.getInstance().getConfigProperty( VeracodeScannerConstants.SCAN_RESULT_RETRY_MINS))); @@ -554,14 +566,12 @@ private boolean getScanReport() { logMessage = "Scan result status is : " + scanStatus + " for the application:" + scanContext.getAppId(); - log.info(logMessage); - CallbackUtil.persistScanLog(scanContext.getJobId(), logMessage, LogType.INFO); + log.info(new CallbackLog(scanContext.getJobId(), logMessage)); } if (scanStatus.equals(ScanStatus.COMPLETED)) { String logMessage = "Scan results are ready for the application: " + scanContext.getAppId(); - log.info(logMessage); - CallbackUtil.persistScanLog(scanContext.getJobId(), logMessage, LogType.INFO); + log.info(new CallbackLog(scanContext.getJobId(), logMessage)); String reportPath = VeracodeScannerConfiguration.getInstance().getConfigProperty( VeracodeScannerConstants.VERACODE_OUTPUT_FOLDER_PATH); @@ -575,13 +585,13 @@ private boolean getScanReport() { } } else { logMessage = "Downloading scan report is failed for the application: " + scanContext.getAppId(); - log.error(logMessage); - CallbackUtil.persistScanLog(scanContext.getJobId(), logMessage, LogType.ERROR); + log.error(new CallbackLog(scanContext.getJobId(), logMessage)); + CallbackUtil.updateScanStatus(scanContext.getJobId(), ScanStatus.ERROR, null, null); } } - } catch (InterruptedException | XPathExpressionException | ParserConfigurationException | SAXException | - IOException | ArchiveException | ScannerException e) { + } catch (XPathExpressionException | ParserConfigurationException | SAXException | IOException | + ArchiveException | ScannerException e) { String logMessage; if (e.getClass().isInstance(ScannerException.class)) { logMessage = "Extracting scan report zip is failed for the application: " + scanContext.getAppId() @@ -590,9 +600,12 @@ private boolean getScanReport() { logMessage = "Downloading scan report is failed for the application: " + scanContext.getAppId() + " " + ErrorProcessingUtil.getFullErrorMessage(e); } - log.error(logMessage); - CallbackUtil.persistScanLog(scanContext.getJobId(), logMessage, LogType.ERROR); + log.error(new CallbackLog(scanContext.getJobId(), logMessage)); + CallbackUtil.updateScanStatus(scanContext.getJobId(), ScanStatus.ERROR, null, null); + } catch (InterruptedException e) { + String logMessage = "Current thread is interrupted for application : " + scanContext.getAppId(); + log.error(new CallbackLog(scanContext.getJobId(), logMessage)); } return isReportUploaded; } @@ -609,38 +622,41 @@ private boolean uploadReportToFtp(String scanArtifact, String reportPath) { String scanReportFtpLocation = scanArtifact.substring(0, scanArtifact.lastIndexOf(File.separator)); File reports = new File(reportPath); - try { - FileUtil.uploadReport(scanReportFtpLocation, reports, - VeracodeScannerConfiguration.getInstance().getConfigProperty(ScannerConstants.FTP_USERNAME), - (VeracodeScannerConfiguration.getInstance().getConfigProperty(ScannerConstants.FTP_PASSWORD)) - .toCharArray(), VeracodeScannerConfiguration.getInstance().getConfigProperty( - ScannerConstants.FTP_HOST), Integer.parseInt(VeracodeScannerConfiguration.getInstance() - .getConfigProperty(ScannerConstants.FTP_PORT))); - - isReportUploaded = true; - CallbackUtil.updateScanStatus(scanContext.getJobId(), ScanStatus.COMPLETED, - scanReportFtpLocation + File.separator + reports.getName(), - null); - - String logMessage = "Scan report is uploaded to the FTP server for the application: " + - scanContext.getAppId(); - log.info(logMessage); - CallbackUtil.persistScanLog(scanContext.getJobId(), logMessage, LogType.INFO); - } catch (SftpException | JSchException e) { - int retryInterval = Integer.parseInt(VeracodeScannerConfiguration.getInstance().getConfigProperty( - VeracodeScannerConstants.SCAN_REPORT_UPLOAD_RETRY_SECONDS)); - log.info("Report upload will retry after " + retryInterval + " seconds since that operation was failed " + - "due to FTP server issue. \n" + ErrorProcessingUtil.getFullErrorMessage(e)); + + if (!Thread.currentThread().isInterrupted()) { try { - TimeUnit.SECONDS.sleep(retryInterval); - } catch (InterruptedException e1) { - log.error(e1); + FileUtil.uploadReport(scanReportFtpLocation, reports, + VeracodeScannerConfiguration.getInstance().getConfigProperty(ScannerConstants.FTP_USERNAME), + (VeracodeScannerConfiguration.getInstance().getConfigProperty(ScannerConstants.FTP_PASSWORD)) + .toCharArray(), VeracodeScannerConfiguration.getInstance().getConfigProperty( + ScannerConstants.FTP_HOST), Integer.parseInt(VeracodeScannerConfiguration.getInstance() + .getConfigProperty(ScannerConstants.FTP_PORT))); + + isReportUploaded = true; + CallbackUtil.updateScanStatus(scanContext.getJobId(), ScanStatus.COMPLETED, scanReportFtpLocation + + File.separator + reports.getName(), null); + + String logMessage = "Scan report is uploaded to the FTP server for the application: " + + scanContext.getAppId(); + log.info(new CallbackLog(scanContext.getJobId(), logMessage)); + } catch (SftpException | JSchException e) { + int retryInterval = Integer.parseInt(VeracodeScannerConfiguration.getInstance().getConfigProperty( + VeracodeScannerConstants.SCAN_REPORT_UPLOAD_RETRY_SECONDS)); + log.info("Report upload will retry after " + retryInterval + " seconds since that operation was failed" + + "due to FTP server issue. \n" + ErrorProcessingUtil.getFullErrorMessage(e)); + try { + TimeUnit.SECONDS.sleep(retryInterval); + } catch (InterruptedException e1) { + log.error(e1); + } + uploadReportToFtp(scanArtifact, reportPath); + } catch (ScannerException | IOException e) { + log.error(new CallbackLog(scanContext.getJobId(), e.getMessage())); + CallbackUtil.updateScanStatus(scanContext.getJobId(), ScanStatus.ERROR, scanReportFtpLocation, null); } - uploadReportToFtp(scanArtifact, reportPath); - } catch (ScannerException | IOException e) { - log.error(e); - CallbackUtil.persistScanLog(scanContext.getJobId(), e.getMessage(), LogType.ERROR); - CallbackUtil.updateScanStatus(scanContext.getJobId(), ScanStatus.ERROR, scanReportFtpLocation, null); + } else { + String logMessage = "Current thread is interrupted for application : " + scanContext.getAppId(); + log.error(new CallbackLog(scanContext.getJobId(), logMessage)); } return isReportUploaded; } @@ -670,38 +686,42 @@ public ScanStatus getScanStatus() throws IOException, XPathExpressionException, public boolean getReports() { boolean isReportPrinted = false; - try { - ResultsAPIWrapper resultsAPIWrapper = VeracodeAPIUtil.getResultAPIWrapper(); - String buildId = getBuildIDByAppId(scanContext.getAppId()); - byte[] resultPdfDetailed = resultsAPIWrapper.detailedReportPdf(buildId); - String resultXMLDetailed = resultsAPIWrapper.detailedReport(buildId); - byte[] resultPdfSummary = resultsAPIWrapper.summaryReportPdf(buildId); - String resultXMLSummary = resultsAPIWrapper.summaryReport(buildId); - byte[] resultXMLThridParty = resultsAPIWrapper.thirdPartyReportPdf(buildId); - - String filePath = VeracodeScannerConfiguration.getInstance().getConfigProperty( - VeracodeScannerConstants.VERACODE_OUTPUT_FOLDER_PATH) + File.separator + scanContext.getAppId(); - FileUtil.saveReport(resultPdfDetailed, filePath + ScannerConstants.PDF_FILE_EXTENSION); - FileUtil.saveReport(resultXMLDetailed.getBytes(StandardCharsets.UTF_8.name()), filePath + - VeracodeScannerConstants.SUMMARY + ScannerConstants.XML_FILE_EXTENSION); - FileUtil.saveReport(resultPdfSummary, filePath + ScannerConstants.PDF_FILE_EXTENSION); - FileUtil.saveReport(resultXMLSummary.getBytes(StandardCharsets.UTF_8.name()), filePath + - VeracodeScannerConstants.SUMMARY + ScannerConstants.XML_FILE_EXTENSION); - FileUtil.saveReport(resultXMLThridParty, filePath + VeracodeScannerConstants.THIRD_PARTY - + ScannerConstants.PDF_FILE_EXTENSION); - isReportPrinted = true; - - String logMessage = "Scan reports are completed and downloaded to the location : " + - VeracodeScannerConfiguration.getInstance().getConfigProperty(VeracodeScannerConstants. - VERACODE_OUTPUT_FOLDER_PATH) + " for the application " + scanContext.getAppId(); - log.info(logMessage); - CallbackUtil.persistScanLog(scanContext.getJobId(), logMessage, LogType.INFO); - } catch (IOException | ParserConfigurationException | SAXException | XPathExpressionException | - ScannerException e) { - String logMessage = "Error occured while downloading the sca reports for the application " - + scanContext.getAppId(); - log.error(logMessage); - CallbackUtil.persistScanLog(scanContext.getJobId(), logMessage, LogType.ERROR); + if (!Thread.currentThread().isInterrupted()) { + try { + ResultsAPIWrapper resultsAPIWrapper = VeracodeAPIUtil.getResultAPIWrapper(); + String buildId = getBuildIDByAppId(scanContext.getAppId()); + byte[] resultPdfDetailed = resultsAPIWrapper.detailedReportPdf(buildId); + String resultXMLDetailed = resultsAPIWrapper.detailedReport(buildId); + byte[] resultPdfSummary = resultsAPIWrapper.summaryReportPdf(buildId); + String resultXMLSummary = resultsAPIWrapper.summaryReport(buildId); + byte[] resultXMLThridParty = resultsAPIWrapper.thirdPartyReportPdf(buildId); + + String filePath = VeracodeScannerConfiguration.getInstance().getConfigProperty( + VeracodeScannerConstants.VERACODE_OUTPUT_FOLDER_PATH) + File.separator + scanContext.getAppId(); + FileUtil.saveReport(resultPdfDetailed, filePath + ScannerConstants.PDF_FILE_EXTENSION); + FileUtil.saveReport(resultXMLDetailed.getBytes(StandardCharsets.UTF_8.name()), filePath + + ScannerConstants.XML_FILE_EXTENSION); + FileUtil.saveReport(resultPdfSummary, filePath + VeracodeScannerConstants.SUMMARY + + ScannerConstants.PDF_FILE_EXTENSION); + FileUtil.saveReport(resultXMLSummary.getBytes(StandardCharsets.UTF_8.name()), filePath + + VeracodeScannerConstants.SUMMARY + ScannerConstants.XML_FILE_EXTENSION); + FileUtil.saveReport(resultXMLThridParty, filePath + VeracodeScannerConstants.THIRD_PARTY + + ScannerConstants.PDF_FILE_EXTENSION); + isReportPrinted = true; + + String logMessage = "Scan reports are completed and downloaded to the location : " + + VeracodeScannerConfiguration.getInstance().getConfigProperty(VeracodeScannerConstants. + VERACODE_OUTPUT_FOLDER_PATH) + " for the application " + scanContext.getAppId(); + log.info(new CallbackLog(scanContext.getJobId(), logMessage)); + } catch (IOException | ParserConfigurationException | SAXException | XPathExpressionException | + ScannerException e) { + String logMessage = "Error occured while downloading the sca reports for the application " + + scanContext.getAppId(); + log.error(new CallbackLog(scanContext.getJobId(), logMessage)); + } + } else { + String logMessage = "Current thread is interrupted for application : " + scanContext.getAppId(); + log.error(new CallbackLog(scanContext.getJobId(), logMessage)); } return isReportPrinted; } diff --git a/internal/scan-manager/scanners/veracode/src/main/resources/jarFilter.xml b/internal/scan-manager/scanners/veracode/src/main/resources/jarFilter.xml index 8391424b..4872ec4c 100644 --- a/internal/scan-manager/scanners/veracode/src/main/resources/jarFilter.xml +++ b/internal/scan-manager/scanners/veracode/src/main/resources/jarFilter.xml @@ -19,17 +19,17 @@ under the License. org.jaggeryjs. - .jar + .jar org.wso2. - .jar + .jar siddhi- - .jar + .jar - .war + .war diff --git a/internal/scan-manager/scanners/veracode/src/main/resources/scanner-config.yaml b/internal/scan-manager/scanners/veracode/src/main/resources/scanner-config.yaml index 81029561..8392edc3 100644 --- a/internal/scan-manager/scanners/veracode/src/main/resources/scanner-config.yaml +++ b/internal/scan-manager/scanners/veracode/src/main/resources/scanner-config.yaml @@ -18,31 +18,27 @@ under the License. # Veracode Scanner Configuration data --- -vuser: -vpassword: -veracode_apiID: -veracode_apiKey: -output_folderpath: -output_filename: -log_filepath: -git_username: -git_password: -default_ftp_product_path: -default_product_path: -report_folder_path: +vuser: +vpassword: +veracode_apiID: +veracode_apiKey: +output_folderpath: out +log_filepath: scanArtifacts/ +default_ftp_product_path: scanArtifacts/ +default_product_path: scanArtifacts/ +report_folder_path: out jar_filter_pattern_file_path: src/main/resources/jarFilter.xml -scan_manager_callback_url_endpoint: -scan_manager_callback_status: -scan_manager_callback_log: +scan_manager_callback_url_endpoint: /callback/ +scan_manager_callback_status: update-scan +scan_manager_callback_log: persist-scan-log build_id_xpath: //veracode:buildinfo/veracode:build[@build_id] scan_status_xpath: //veracode:buildinfo/veracode:build/veracode:analysis_unit[@status] build_id_attribute: build_id scan_status_attribute: status scan_response_namespace: https://analysiscenter.veracode.com/schema/4.0/buildinfo callback_retry_interval_seconds: 5 -scan_result_waiting_interval_hours : 4 scan_result_retry_interval_mins : 10 -ftp_username: +ftp_username: ftp_password: -ftp_host: -ftp_port: 22 +ftp_host: +ftp_port: From 53add91e68cca7672581c4789d8996fbdac58c29 Mon Sep 17 00:00:00 2001 From: NShani Date: Wed, 19 Jun 2019 16:07:05 +0530 Subject: [PATCH 3/5] Add scan request validations --- .../scanners/common/service/Scanner.java | 22 ++++- .../veracode/service/VeracodeScanner.java | 94 +++++++++---------- 2 files changed, 62 insertions(+), 54 deletions(-) diff --git a/internal/scan-manager/scanners/common/src/main/java/org/wso2/security/tools/scanmanager/scanners/common/service/Scanner.java b/internal/scan-manager/scanners/common/src/main/java/org/wso2/security/tools/scanmanager/scanners/common/service/Scanner.java index 2ae15d49..d66eff49 100644 --- a/internal/scan-manager/scanners/common/src/main/java/org/wso2/security/tools/scanmanager/scanners/common/service/Scanner.java +++ b/internal/scan-manager/scanners/common/src/main/java/org/wso2/security/tools/scanmanager/scanners/common/service/Scanner.java @@ -20,7 +20,6 @@ package org.wso2.security.tools.scanmanager.scanners.common.service; -import org.springframework.http.ResponseEntity; import org.wso2.security.tools.scanmanager.common.internal.model.ScannerScanRequest; /** @@ -32,16 +31,29 @@ public interface Scanner { * Run scan. * * @param scanRequest Object that represent the required information for tha scanner operation - * @return details of the start scan response from the scanner service */ - public ResponseEntity startScan(ScannerScanRequest scanRequest); + public void startScan(ScannerScanRequest scanRequest); + + /** + * Validate the start scan request. + * + * @param scannerScanRequest start scan request + * @return whether start scan request is a valid one + */ + public boolean validateStartScan(ScannerScanRequest scannerScanRequest); /** * Stop the last scan for a given application. * * @param scanRequest Object that represent the required information for tha scanner operation - * @return details of the cancel scan response from the scanner service */ - public ResponseEntity cancelScan(ScannerScanRequest scanRequest); + public void cancelScan(ScannerScanRequest scanRequest); + /** + * Validate the cancel scan request. + * + * @param scannerScanRequest cancel scan request + * @return whether cancel scan request is a valid one + */ + public boolean validateCancelScan(ScannerScanRequest scannerScanRequest); } diff --git a/internal/scan-manager/scanners/veracode/src/main/java/org/wso2/security/tools/scanmanager/scanners/veracode/service/VeracodeScanner.java b/internal/scan-manager/scanners/veracode/src/main/java/org/wso2/security/tools/scanmanager/scanners/veracode/service/VeracodeScanner.java index 4dad1074..74cf44a1 100644 --- a/internal/scan-manager/scanners/veracode/src/main/java/org/wso2/security/tools/scanmanager/scanners/veracode/service/VeracodeScanner.java +++ b/internal/scan-manager/scanners/veracode/src/main/java/org/wso2/security/tools/scanmanager/scanners/veracode/service/VeracodeScanner.java @@ -19,17 +19,16 @@ import com.veracode.apiwrapper.cli.VeracodeCommand; import com.veracode.apiwrapper.wrappers.UploadAPIWrapper; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.springframework.beans.factory.config.ConfigurableBeanFactory; import org.springframework.context.annotation.Scope; -import org.springframework.http.HttpStatus; -import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Component; import org.springframework.util.StringUtils; import org.wso2.security.tools.scanmanager.common.internal.model.ScannerScanRequest; -import org.wso2.security.tools.scanmanager.common.model.ErrorMessage; -import org.wso2.security.tools.scanmanager.common.model.LogType; import org.wso2.security.tools.scanmanager.common.model.ScanStatus; import org.wso2.security.tools.scanmanager.scanners.common.ScannerConstants; +import org.wso2.security.tools.scanmanager.scanners.common.model.CallbackLog; import org.wso2.security.tools.scanmanager.scanners.common.service.Scanner; import org.wso2.security.tools.scanmanager.scanners.common.util.CallbackUtil; import org.wso2.security.tools.scanmanager.scanners.common.util.ErrorProcessingUtil; @@ -41,7 +40,6 @@ import org.wso2.security.tools.scanmanager.scanners.veracode.util.VeracodeAPIUtil; import org.xml.sax.SAXException; -import java.io.File; import java.io.IOException; import javax.xml.parsers.ParserConfigurationException; import javax.xml.xpath.XPathExpressionException; @@ -53,8 +51,7 @@ @Scope(value = ConfigurableBeanFactory.SCOPE_SINGLETON) public class VeracodeScanner implements Scanner { - // Scan task thread. - Thread scanTaskThread; + private static final Logger log = LogManager.getLogger(VeracodeScanner.class); // Scan context object for a particular container. private ScanContext scanContext; @@ -72,9 +69,6 @@ public VeracodeScanner() throws IOException { options = new VeracodeCommand.Options(); options._output_folderpath = VeracodeScannerConfiguration.getInstance().getConfigProperty( VeracodeScannerConstants.VERACODE_OUTPUT_FOLDER_PATH); - options._output_filepath = VeracodeScannerConfiguration.getInstance().getConfigProperty( - VeracodeScannerConstants.VERACODE_OUTPUT_FOLDER_PATH) + File.separator + VeracodeScannerConfiguration - .getInstance().getConfigProperty(VeracodeScannerConstants.VERACODE_OUTPUT_FILE_NAME); options._log_filepath = VeracodeScannerConfiguration.getInstance().getConfigProperty(VeracodeScannerConstants .VERACODE_LOG_FILE_PATH); options._vid = VeracodeScannerConfiguration.getInstance().getConfigProperty(VeracodeScannerConstants @@ -83,10 +77,14 @@ public VeracodeScanner() throws IOException { VeracodeScannerConstants.VERACODE_API_KEY)); VeracodeAPIUtil.setCredentials(options); - CallbackUtil.setCallbackUrls(VeracodeScannerConfiguration.getInstance().getConfigProperty(ScannerConstants - .SCAN_MANAGER_CALLBACK_URL) + VeracodeScannerConfiguration.getInstance().getConfigProperty( - ScannerConstants.SCAN_MANAGER_CALLBACK_LOG), VeracodeScannerConfiguration.getInstance() - .getConfigProperty(ScannerConstants.SCAN_MANAGER_CALLBACK_URL) + VeracodeScannerConfiguration + CallbackUtil.setCallbackUrls(ScannerConstants.HTTP_PROTOCOL + System.getenv(ScannerConstants.SCAN_MANAGER_HOST) + + ":" + System.getenv(ScannerConstants.SCAN_MANAGER_PORT) + VeracodeScannerConfiguration + .getInstance().getConfigProperty(ScannerConstants.SCAN_MANAGER_CALLBACK_URL_ENDPOINT) + + VeracodeScannerConfiguration.getInstance().getConfigProperty(ScannerConstants + .SCAN_MANAGER_CALLBACK_LOG), ScannerConstants.HTTP_PROTOCOL + System.getenv( + ScannerConstants.SCAN_MANAGER_HOST) + ":" + System.getenv(ScannerConstants + .SCAN_MANAGER_PORT) + VeracodeScannerConfiguration.getInstance().getConfigProperty( + ScannerConstants.SCAN_MANAGER_CALLBACK_URL_ENDPOINT) + VeracodeScannerConfiguration .getInstance().getConfigProperty(ScannerConstants.SCAN_MANAGER_CALLBACK_STATUS), Long.parseLong(VeracodeScannerConfiguration.getInstance().getConfigProperty(ScannerConstants .CALLBACK_RETRY_INCREASE_SECONDS))); @@ -107,41 +105,51 @@ private static void loadConfiguration() throws IOException { * Run the scan using product zip file. * * @param scanRequest Object that represent the required information for the scanner operation - * @return ResponseEntity with status of the request */ @Override - public ResponseEntity startScan(ScannerScanRequest scanRequest) { + public void startScan(ScannerScanRequest scanRequest) { scanContext.setJobId(scanRequest.getJobId()); scanContext.setAppId(scanRequest.getAppId()); scanContext.setArtifactLocation(scanRequest.getFileMap().get(VeracodeScannerConstants.SCAN_ARTIFACT).get(0)); - ResponseEntity responseEntity; if (scanRequest.getFileMap().get(VeracodeScannerConstants.SCAN_ARTIFACT) != null) { if (!StringUtils.isEmpty(scanContext.getAppId())) { - responseEntity = new ResponseEntity<>(HttpStatus.ACCEPTED); - startVeracodeScan(); + if (!Thread.currentThread().isInterrupted()) { + ScanTask scanTask = new ScanTask(scanContext); + scanTask.run(); + } else { + String message = "Current thread is interrupted. "; + log.error(new CallbackLog(scanContext.getJobId(), message)); + } } else { String message = "Error occured while submitting the start scan request since the application " + "is empty in the request. "; - responseEntity = callbackErrorReport(message, HttpStatus.BAD_REQUEST); + callbackErrorReport(message); } } else { String message = "Error occured while submitting the start scan request since the scan artifacts " + "are empty in the request. "; - responseEntity = callbackErrorReport(message, HttpStatus.BAD_REQUEST); + callbackErrorReport(message); + } + } + + @Override + public boolean validateStartScan(ScannerScanRequest scannerScanRequest) { + + if (!StringUtils.isEmpty(scannerScanRequest.getFileMap().get(VeracodeScannerConstants.SCAN_ARTIFACT))) { + return true; + } else { + return false; } - return responseEntity; } /** * Stop the last scan for a given application. * * @param scanRequest Object that represent the required information for tha scanner operation - * @return ResponseEntity with status of the request */ @Override - public ResponseEntity cancelScan(ScannerScanRequest scanRequest) { - ResponseEntity responseEntity; + public void cancelScan(ScannerScanRequest scanRequest) { String scanInfoResult; String deleteApiResult; ScanStatus currentScanStatus; @@ -155,53 +163,41 @@ public ResponseEntity cancelScan(ScannerScanRequest scanRequest) { deleteApiResult = uploadAPIWrapper.deleteBuild(scanContext.getAppId()); if (VeracodeResultProcessor.isOperationProceedWithoutError(deleteApiResult)) { String message = "Successfully cancelled the scan of the application : " + scanRequest.getAppId(); + log.info(new CallbackLog(scanRequest.getJobId(), message)); + CallbackUtil.updateScanStatus(scanContext.getJobId(), ScanStatus.CANCELED, null, null); - CallbackUtil.persistScanLog(scanContext.getJobId(), message, LogType.INFO); - responseEntity = new ResponseEntity<>(HttpStatus.ACCEPTED); } else { String message = "Error occured while deleting the last scan of the application : " + scanRequest.getAppId(); - responseEntity = callbackErrorReport(message, HttpStatus.BAD_REQUEST); + callbackErrorReport(message); } } else { String message = "Successfully cancelled the scan of the application : " + scanRequest.getAppId(); + log.info(new CallbackLog(scanContext.getJobId(), message)); + CallbackUtil.updateScanStatus(scanContext.getJobId(), ScanStatus.CANCELED, null, null); - CallbackUtil.persistScanLog(scanContext.getJobId(), message, LogType.INFO); - responseEntity = new ResponseEntity<>(HttpStatus.ACCEPTED); - scanTaskThread.interrupt(); } } catch (IOException | ParserConfigurationException | XPathExpressionException | SAXException e) { String message = "Error occured while deleting the last scan of the application : " + scanRequest.getAppId() + " " + ErrorProcessingUtil.getFullErrorMessage(e); - responseEntity = callbackErrorReport(message, HttpStatus.INTERNAL_SERVER_ERROR); + callbackErrorReport(message); } - return responseEntity; } - /** - * Initiate the scan in the Veracode. - */ - private void startVeracodeScan() { - ScanTask scanTask = new ScanTask(scanContext); - - scanTaskThread = new Thread(scanTask); - scanTaskThread.start(); + @Override + public boolean validateCancelScan(ScannerScanRequest scannerScanRequest) { + return true; } /** * Update the call back endpoint when error happens at the service layer. * - * @param message error message - * @param httpStatus http status code of the error + * @param message error message * @return ResponseEntity with status of the updating the call back endpoint */ - private ResponseEntity callbackErrorReport(String message, HttpStatus httpStatus) { - ResponseEntity responseEntity = new ResponseEntity<>(new ErrorMessage(httpStatus.value(), message), - HttpStatus.BAD_REQUEST); + private void callbackErrorReport(String message) { + log.error(new CallbackLog(scanContext.getJobId(), message)); CallbackUtil.updateScanStatus(scanContext.getJobId(), ScanStatus.ERROR, null, null); - CallbackUtil.persistScanLog(scanContext.getJobId(), message, LogType.ERROR); - - return responseEntity; } } From 8ff87d0710b0d7039256d4948bab9753f501adf2 Mon Sep 17 00:00:00 2001 From: NShani Date: Mon, 24 Jun 2019 14:17:26 +0530 Subject: [PATCH 4/5] Update internal/scan-manager/scanners/common/src/main/java/org/wso2/security/tools/scanmanager/scanners/common/ScannerController.java Co-Authored-By: Ayoma Wijethunga --- .../tools/scanmanager/scanners/common/ScannerController.java | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/scan-manager/scanners/common/src/main/java/org/wso2/security/tools/scanmanager/scanners/common/ScannerController.java b/internal/scan-manager/scanners/common/src/main/java/org/wso2/security/tools/scanmanager/scanners/common/ScannerController.java index fd328c35..f33d8c7d 100644 --- a/internal/scan-manager/scanners/common/src/main/java/org/wso2/security/tools/scanmanager/scanners/common/ScannerController.java +++ b/internal/scan-manager/scanners/common/src/main/java/org/wso2/security/tools/scanmanager/scanners/common/ScannerController.java @@ -153,7 +153,6 @@ public ResponseEntity cancelScan(@RequestBody ScannerScanRequest scanRequest) { } private boolean stopStartScanThread() { - while (startScanThread.isAlive()) { // run until the start scan thread is dead. } From 6a93b9d5e798005259e5621bdbbcae99e074fa92 Mon Sep 17 00:00:00 2001 From: NShani Date: Tue, 25 Jun 2019 10:49:34 +0530 Subject: [PATCH 5/5] Resolve PR comments --- .../scanners/common/ScannerController.java | 59 ++--- .../scanners/common/util/FileUtil.java | 24 +- .../scanners/veracode/handler/ScanTask.java | 217 ++++++++++-------- .../veracode/service/VeracodeScanner.java | 47 ++-- 4 files changed, 195 insertions(+), 152 deletions(-) diff --git a/internal/scan-manager/scanners/common/src/main/java/org/wso2/security/tools/scanmanager/scanners/common/ScannerController.java b/internal/scan-manager/scanners/common/src/main/java/org/wso2/security/tools/scanmanager/scanners/common/ScannerController.java index fd328c35..51716b2d 100644 --- a/internal/scan-manager/scanners/common/src/main/java/org/wso2/security/tools/scanmanager/scanners/common/ScannerController.java +++ b/internal/scan-manager/scanners/common/src/main/java/org/wso2/security/tools/scanmanager/scanners/common/ScannerController.java @@ -35,6 +35,7 @@ import org.wso2.security.tools.scanmanager.scanners.common.service.Scanner; import java.io.IOException; +import java.util.concurrent.TimeUnit; /** * Web controller which defines the routines for initiating scanner operations. @@ -49,9 +50,6 @@ public class ScannerController { // Scan task thread. Thread startScanThread; - // Cancel scan task thread. - Thread cancelScanThread; - // This represents if a scan is started. private boolean hasScanStarted = false; @@ -76,11 +74,11 @@ public ResponseEntity startScan(@RequestBody ScannerScanRequest scannerScanReque if (scanner.validateStartScan(scannerScanRequest)) { log.info("Invoking start scan API."); startScanThread = new Thread(() -> scanner.startScan(scannerScanRequest), "StartScanThread"); - startScanThread.start(); hasScanStarted = true; } else { String message = "Start scan request validation is failed."; + log.error(message); responseEntity = new ResponseEntity<>(new ErrorMessage(HttpStatus.BAD_REQUEST.value(), message), HttpStatus.BAD_REQUEST); } @@ -90,24 +88,27 @@ public ResponseEntity startScan(@RequestBody ScannerScanRequest scannerScanReque private ResponseEntity validateStartScanReq(ScannerScanRequest scannerScanRequest) { ResponseEntity responseEntity; - if (!hasScanStarted) { - if (!StringUtils.isEmpty(scannerScanRequest.getAppId())) { - if (!StringUtils.isEmpty(scannerScanRequest.getJobId())) { - responseEntity = new ResponseEntity<>(HttpStatus.ACCEPTED); - } else { + if (hasScanStarted) { + String message = "Cannot start a new scan since another scan is in progress."; + log.error(message); + responseEntity = new ResponseEntity<>(new ErrorMessage(HttpStatus.BAD_REQUEST.value(), + message), HttpStatus.BAD_REQUEST); + } else { + if (StringUtils.isEmpty(scannerScanRequest.getAppId())) { + String message = "Application Id is missing in the request."; + log.error(message); + responseEntity = new ResponseEntity<>(new ErrorMessage(HttpStatus.BAD_REQUEST.value(), + message), HttpStatus.BAD_REQUEST); + } else { + if (StringUtils.isEmpty(scannerScanRequest.getJobId())) { String message = "Job Id is missing in the request."; + log.error(message); responseEntity = new ResponseEntity<>(new ErrorMessage(HttpStatus.BAD_REQUEST.value(), message), HttpStatus.BAD_REQUEST); + } else { + responseEntity = new ResponseEntity<>(HttpStatus.ACCEPTED); } - } else { - String message = "Application Id is missing in the request."; - responseEntity = new ResponseEntity<>(new ErrorMessage(HttpStatus.BAD_REQUEST.value(), - message), HttpStatus.BAD_REQUEST); } - } else { - String message = "Cannot start a new scan since another scan is in progress."; - responseEntity = new ResponseEntity<>(new ErrorMessage(HttpStatus.BAD_REQUEST.value(), - message), HttpStatus.BAD_REQUEST); } return responseEntity; } @@ -123,11 +124,7 @@ private ResponseEntity validateStartScanReq(ScannerScanRequest scannerScanReques public ResponseEntity cancelScan(@RequestBody ScannerScanRequest scanRequest) { ResponseEntity responseEntity; - if (!hasScanStarted) { - String message = "No scan running to perform cancellation."; - responseEntity = new ResponseEntity<>(new ErrorMessage(HttpStatus.NOT_ACCEPTABLE.value(), - message), HttpStatus.BAD_REQUEST); - } else { + if (hasScanStarted) { log.info("Invoking cancel scan API."); if (scanner.validateCancelScan(scanRequest)) { if (startScanThread != null) { @@ -136,26 +133,36 @@ public ResponseEntity cancelScan(@RequestBody ScannerScanRequest scanRequest) { log.info("There is no running scan thread to cancel."); } - cancelScanThread = new Thread(() -> { + new Thread(() -> { stopStartScanThread(); scanner.cancelScan(scanRequest); - }, "CancelScanThread"); - cancelScanThread.start(); + }, "CancelScanThread").start(); responseEntity = new ResponseEntity<>(HttpStatus.ACCEPTED); } else { String message = "Cancel scan request validation is failed."; + log.error(message); responseEntity = new ResponseEntity<>(new ErrorMessage(HttpStatus.BAD_REQUEST.value(), message), HttpStatus.BAD_REQUEST); } + } else { + String message = "No scan running to perform cancellation."; + log.error(message); + responseEntity = new ResponseEntity<>(new ErrorMessage(HttpStatus.NOT_ACCEPTABLE.value(), + message), HttpStatus.BAD_REQUEST); } return responseEntity; } private boolean stopStartScanThread() { - while (startScanThread.isAlive()) { // run until the start scan thread is dead. + try { + TimeUnit.SECONDS.sleep(10); + } catch (InterruptedException e) { + log.error("Interrupted exception occured while waiting till the start scan thread is dead. \n" + + e.getMessage()); + } } return true; } diff --git a/internal/scan-manager/scanners/common/src/main/java/org/wso2/security/tools/scanmanager/scanners/common/util/FileUtil.java b/internal/scan-manager/scanners/common/src/main/java/org/wso2/security/tools/scanmanager/scanners/common/util/FileUtil.java index 686d854d..7a317519 100644 --- a/internal/scan-manager/scanners/common/src/main/java/org/wso2/security/tools/scanmanager/scanners/common/util/FileUtil.java +++ b/internal/scan-manager/scanners/common/src/main/java/org/wso2/security/tools/scanmanager/scanners/common/util/FileUtil.java @@ -84,13 +84,17 @@ public static String extractArchive(File sourceFile, String destination) throws // Handing for the Zip Slip Vulnerability File destinationFile = new File(destination, currentEntry); String canonicalizedDestinationFilePath = destinationFile.getCanonicalPath(); - if (!canonicalizedDestinationFilePath.startsWith(new File(destination).getCanonicalPath())) { + if (canonicalizedDestinationFilePath.startsWith(new File(destination).getCanonicalPath())) { + // if a valid zip file uploaded + } else { String errorMessage = "Attempt to upload invalid zip archive with file at " + currentEntry + ". File path is outside target directory"; log.error(errorMessage); } - if (!entry.isDirectory()) { + if (entry.isDirectory()) { + // if the entry is a directory + } else { zipInputStream = zip.getInputStream(entry); inputStream = new BufferedInputStream(zipInputStream); if (destinationFile.getParentFile().mkdirs()) { @@ -197,14 +201,14 @@ public static boolean saveReport(byte[] bytesResult, String filePath) throws Fil UnsupportedEncodingException, ScannerException { if (bytesResult != null) { - if (!(filePath.isEmpty())) { + if (filePath.isEmpty()) { + throw new ScannerException("Output file path is missing."); + } else { try (PrintStream writer = new PrintStream(new FileOutputStream(filePath), true, StandardCharsets .UTF_8.name())) { writer.write(bytesResult, 0, bytesResult.length); return true; } - } else { - throw new ScannerException("Output file path is missing."); } } else { throw new ScannerException("Unable to retrieve data from byte stream."); @@ -240,7 +244,7 @@ private static ChannelSftp openFtpLocation(String filePathInFtp, String ftpUsern sftp.connect(); sftp.cd(filePathInFtp); - cleanFtpPassword(ftpPassword); + cleanPassword(ftpPassword); return sftp; } @@ -267,11 +271,11 @@ private static void downloadFromFtp(InputStream input, File outputFile) throws I /** * Clean the FTP password from the variable. * - * @param ftpPassword password that needs to be cleared + * @param password password that needs to be cleared */ - private static void cleanFtpPassword(char[] ftpPassword) { - for (int i = 0; i < ftpPassword.length; i++) { - ftpPassword[i] = '\0'; + public static void cleanPassword(char[] password) { + for (int i = 0; i < password.length; i++) { + password[i] = '\0'; } } diff --git a/internal/scan-manager/scanners/veracode/src/main/java/org/wso2/security/tools/scanmanager/scanners/veracode/handler/ScanTask.java b/internal/scan-manager/scanners/veracode/src/main/java/org/wso2/security/tools/scanmanager/scanners/veracode/handler/ScanTask.java index d47cb119..fe23f696 100644 --- a/internal/scan-manager/scanners/veracode/src/main/java/org/wso2/security/tools/scanmanager/scanners/veracode/handler/ScanTask.java +++ b/internal/scan-manager/scanners/veracode/src/main/java/org/wso2/security/tools/scanmanager/scanners/veracode/handler/ScanTask.java @@ -51,6 +51,8 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.util.concurrent.TimeUnit; +import java.util.stream.IntStream; +import java.util.stream.Stream; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; @@ -127,7 +129,9 @@ private boolean handleUploadingTask() { } else { logMessage = "Cleaning previous scans is failed."; } - if (!isUploadSuccess) { + if (isUploadSuccess) { + // return to parent method. + } else { logMessage = logMessage.concat(" Terminating scan for app: " + scanContext.getAppId()); log.error(new CallbackLog(scanContext.getJobId(), logMessage)); } @@ -155,7 +159,9 @@ private boolean handleResultProcessTask() { } else { logMessage = "Pre-Scan is failed."; } - if (!isResultsUploaded) { + if (isResultsUploaded) { + // return to parent method. + } else { logMessage = logMessage + (" Terminating scan for app: " + scanContext.getAppId()); log.error(new CallbackLog(scanContext.getJobId(), logMessage)); } @@ -172,7 +178,10 @@ private boolean isScanRunning() { String result; ScanStatus currentScanStatus; - if (!Thread.currentThread().isInterrupted()) { + if (Thread.currentThread().isInterrupted()) { + String logMessage = "Current thread is interrupted for application : " + scanContext.getAppId(); + log.error(new CallbackLog(scanContext.getJobId(), logMessage)); + } else { try { UploadAPIWrapper uploadAPIWrapper = VeracodeAPIUtil.getUploadAPIWrapper(); result = uploadAPIWrapper.getBuildInfo(scanContext.getAppId()); @@ -187,9 +196,6 @@ private boolean isScanRunning() { CallbackUtil.updateScanStatus(scanContext.getJobId(), ScanStatus.ERROR, null, null); } - } else { - String logMessage = "Current thread is interrupted for application : " + scanContext.getAppId(); - log.error(new CallbackLog(scanContext.getJobId(), logMessage)); } return isScanRunning; } @@ -203,7 +209,10 @@ private boolean cleanPreviousScans() { boolean isScannerCleaned = false; String result; - if (!Thread.currentThread().isInterrupted()) { + if (Thread.currentThread().isInterrupted()) { + String logMessage = "Current thread is interrupted for application : " + scanContext.getAppId(); + log.error(new CallbackLog(scanContext.getJobId(), logMessage)); + } else { try { UploadAPIWrapper uploadAPIWrapper = VeracodeAPIUtil.getUploadAPIWrapper(); result = uploadAPIWrapper.deleteBuild(scanContext.getAppId()); @@ -221,9 +230,6 @@ private boolean cleanPreviousScans() { CallbackUtil.updateScanStatus(scanContext.getJobId(), ScanStatus.ERROR, null, null); } - } else { - String logMessage = "Current thread is interrupted for application : " + scanContext.getAppId(); - log.error(new CallbackLog(scanContext.getJobId(), logMessage)); } return isScannerCleaned; } @@ -244,17 +250,26 @@ private boolean creatingScanArtifactZip() { File productFile = new File(VeracodeScannerConfiguration.getInstance().getConfigProperty( ScannerConstants.DEFAULT_FTP_PRODUCT_PATH) + productPackName); - if (!Thread.currentThread().isInterrupted()) { + + String ftpUsername = VeracodeScannerConfiguration.getInstance().getConfigProperty(ScannerConstants + .FTP_USERNAME); + char[] ftpPassword = VeracodeScannerConfiguration.getInstance().getConfigProperty(ScannerConstants + .FTP_PASSWORD).toCharArray(); + String ftpHost = VeracodeScannerConfiguration.getInstance().getConfigProperty(ScannerConstants.FTP_HOST); + int ftpPort = Integer.parseInt(VeracodeScannerConfiguration.getInstance().getConfigProperty(ScannerConstants + .FTP_PORT)); + + if (Thread.currentThread().isInterrupted()) { + String logMessage = "Current thread is interrupted for application : " + scanContext.getAppId(); + log.error(new CallbackLog(scanContext.getJobId(), logMessage)); + } else { try { String logMessage = "Product pack is downloading for the application: " + scanContext.getAppId(); log.info(new CallbackLog(scanContext.getJobId(), logMessage)); - FileUtil.downloadFromFtp(productPath, productPackName, productFile, VeracodeScannerConfiguration - .getInstance().getConfigProperty(ScannerConstants.FTP_USERNAME), - (VeracodeScannerConfiguration.getInstance().getConfigProperty(ScannerConstants.FTP_PASSWORD)) - .toCharArray(), VeracodeScannerConfiguration.getInstance().getConfigProperty( - ScannerConstants.FTP_HOST), Integer.parseInt(VeracodeScannerConfiguration.getInstance() - .getConfigProperty(ScannerConstants.FTP_PORT))); + FileUtil.downloadFromFtp(productPath, productPackName, productFile, ftpUsername, ftpPassword, ftpHost, + ftpPort); + FileUtil.cleanPassword(ftpPassword); logMessage = "Product downloading completed for the application: " + scanContext.getAppId() + " into " + productFile; @@ -289,13 +304,12 @@ private boolean creatingScanArtifactZip() { scanContext.getAppId() + "\n" + ErrorProcessingUtil.getFullErrorMessage(e); log.error(new CallbackLog(scanContext.getJobId(), logMessage)); - if (!e.getClass().isInstance(InterruptedIOException.class)) { + if (e.getClass().isInstance(InterruptedIOException.class)) { + // if the error is an interrupted exception, will return to the parent method. + } else { CallbackUtil.updateScanStatus(scanContext.getJobId(), ScanStatus.ERROR, null, null); } } - } else { - String logMessage = "Current thread is interrupted for application : " + scanContext.getAppId(); - log.error(new CallbackLog(scanContext.getJobId(), logMessage)); } return isZipCreated; } @@ -309,7 +323,10 @@ private boolean uploadScanArtifact() { boolean isUploadSuccess = false; String buildId; - if (!Thread.currentThread().isInterrupted()) { + if (Thread.currentThread().isInterrupted()) { + String logMessage = "Current thread is interrupted for application : " + scanContext.getAppId(); + log.error(new CallbackLog(scanContext.getJobId(), logMessage)); + } else { try { UploadAPIWrapper uploadAPIWrapper = VeracodeAPIUtil.getUploadAPIWrapper(); String result = uploadAPIWrapper.uploadFile(scanContext.getAppId(), workingDirectory @@ -337,9 +354,6 @@ private boolean uploadScanArtifact() { CallbackUtil.updateScanStatus(scanContext.getJobId(), ScanStatus.ERROR, null, null); } - } else { - String logMessage = "Current thread is interrupted for application : " + scanContext.getAppId(); - log.error(new CallbackLog(scanContext.getJobId(), logMessage)); } return isUploadSuccess; } @@ -358,7 +372,10 @@ private void copyRequiredScanArtifact(String filePath) throws IOException, SAXEx File[] files = dir.listFiles(); File patternXmlFile = new File(JAR_FILTER_FILE); - if (!Thread.currentThread().isInterrupted()) { + if (Thread.currentThread().isInterrupted()) { + String logMessage = "Current thread is interrupted for application : " + scanContext.getAppId(); + log.error(new CallbackLog(scanContext.getJobId(), logMessage)); + } else { try (InputStream input = VeracodeScannerConfiguration.class.getClassLoader() .getResourceAsStream(JAR_FILTER_FILE); OutputStream out = new FileOutputStream(patternXmlFile)) { @@ -373,25 +390,21 @@ private void copyRequiredScanArtifact(String filePath) throws IOException, SAXEx } NodeList nodeList = getScanArtifactPatternList(patternXmlFile); - if (files != null) { - for (File file : files) { - if (file.isFile()) { - for (int i = 0; i < nodeList.getLength(); i++) { - Node node = nodeList.item(i); - Element element = (Element) node; - + Stream nodeStream = IntStream.range(0, nodeList.getLength()).mapToObj(nodeList::item); + for (File file : files) { + if (file.isFile()) { + nodeStream.forEach(node -> { + Element element = (Element) node; + try { checkFileNamePattern(element, file); + } catch (IOException e) { + throw new RuntimeException(e); } - } else if (file.isDirectory()) { - copyRequiredScanArtifact(file.getAbsolutePath()); - } + }); + } else { + copyRequiredScanArtifact(file.getAbsolutePath()); } - } else { - log.warn("File list that needs to be archived cannot be null."); } - } else { - String logMessage = "Current thread is interrupted for application : " + scanContext.getAppId(); - log.error(new CallbackLog(scanContext.getJobId(), logMessage)); } } @@ -462,7 +475,10 @@ private boolean beginPreScan() { String buildId = null; boolean isPreScanStarted = false; - if (!Thread.currentThread().isInterrupted()) { + if (Thread.currentThread().isInterrupted()) { + String logMessage = "Current thread is interrupted for application : " + scanContext.getAppId(); + log.error(new CallbackLog(scanContext.getJobId(), logMessage)); + } else { try { UploadAPIWrapper uploadAPIWrapper = VeracodeAPIUtil.getUploadAPIWrapper(); result = uploadAPIWrapper.beginPreScan(scanContext.getAppId(), null, "true", "true"); @@ -489,9 +505,6 @@ private boolean beginPreScan() { CallbackUtil.updateScanStatus(scanContext.getJobId(), ScanStatus.ERROR, null, buildId); } - } else { - String logMessage = "Current thread is interrupted for application : " + scanContext.getAppId(); - log.error(new CallbackLog(scanContext.getJobId(), logMessage)); } return isPreScanStarted; } @@ -505,7 +518,10 @@ private boolean beginScan() { String result; boolean isScanStarted = false; - if (!Thread.currentThread().isInterrupted()) { + if (Thread.currentThread().isInterrupted()) { + String logMessage = "Current thread is interrupted for application : " + scanContext.getAppId(); + log.error(new CallbackLog(scanContext.getJobId(), logMessage)); + } else { try { UploadAPIWrapper uploadAPIWrapper = VeracodeAPIUtil.getUploadAPIWrapper(); result = uploadAPIWrapper.beginScan(scanContext.getAppId(), "all", "true"); @@ -528,9 +544,6 @@ private boolean beginScan() { CallbackUtil.updateScanStatus(scanContext.getJobId(), ScanStatus.ERROR, null, null); } - } else { - String logMessage = "Current thread is interrupted for application : " + scanContext.getAppId(); - log.error(new CallbackLog(scanContext.getJobId(), logMessage)); } return isScanStarted; } @@ -555,13 +568,13 @@ private boolean getScanReport() { if (scanStatus.equals(ScanStatus.ERROR)) { break; } - String logMessage = "Waiting for " + VeracodeScannerConfiguration.getInstance().getConfigProperty( - VeracodeScannerConstants.SCAN_RESULT_RETRY_MINS) + " mins until the scan is completed" + + String waitingTime = VeracodeScannerConfiguration.getInstance().getConfigProperty( + VeracodeScannerConstants.SCAN_RESULT_RETRY_MINS); + String logMessage = "Waiting for " + waitingTime + " minutes until the scan is completed" + " for the application: " + scanContext.getAppId(); log.info(new CallbackLog(scanContext.getJobId(), logMessage)); - TimeUnit.MINUTES.sleep(Integer.parseInt(VeracodeScannerConfiguration.getInstance().getConfigProperty( - VeracodeScannerConstants.SCAN_RESULT_RETRY_MINS))); + TimeUnit.MINUTES.sleep(Integer.parseInt(waitingTime)); scanStatus = getScanStatus(); logMessage = "Scan result status is : " + scanStatus + " for the application:" @@ -569,25 +582,34 @@ private boolean getScanReport() { log.info(new CallbackLog(scanContext.getJobId(), logMessage)); } - if (scanStatus.equals(ScanStatus.COMPLETED)) { - String logMessage = "Scan results are ready for the application: " + scanContext.getAppId(); - log.info(new CallbackLog(scanContext.getJobId(), logMessage)); + if (Thread.currentThread().isInterrupted()) { + String logMessage = "Current thread is interrupted for application : " + scanContext.getAppId(); + log.error(new CallbackLog(scanContext.getJobId(), logMessage)); + } else { + if (scanStatus.equals(ScanStatus.COMPLETED)) { + String logMessage = "Scan results are ready for the application: " + scanContext.getAppId(); + log.info(new CallbackLog(scanContext.getJobId(), logMessage)); - String reportPath = VeracodeScannerConfiguration.getInstance().getConfigProperty( - VeracodeScannerConstants.VERACODE_OUTPUT_FOLDER_PATH); + String reportPath = VeracodeScannerConfiguration.getInstance().getConfigProperty( + VeracodeScannerConstants.VERACODE_OUTPUT_FOLDER_PATH); + + String scanArtifact = scanContext.getArtifactLocation(); + boolean isReportDownloaded = getReports(); + if (isReportDownloaded) { + FileUtil.zipFiles(reportPath, reportPath + ScannerConstants.ZIP_FILE_EXTENSION); + if (uploadReportToFtp(scanArtifact, reportPath + ScannerConstants.ZIP_FILE_EXTENSION)) { + isReportUploaded = true; + } + } else { + logMessage = "Downloading scan report is failed for the application: " + scanContext.getAppId(); + log.error(new CallbackLog(scanContext.getJobId(), logMessage)); - String scanArtifact = scanContext.getArtifactLocation(); - boolean isReportDownloaded = getReports(); - if (isReportDownloaded) { - FileUtil.zipFiles(reportPath, reportPath + ScannerConstants.ZIP_FILE_EXTENSION); - if (uploadReportToFtp(scanArtifact, reportPath + ScannerConstants.ZIP_FILE_EXTENSION)) { - isReportUploaded = true; + CallbackUtil.updateScanStatus(scanContext.getJobId(), ScanStatus.ERROR, null, null); } } else { - logMessage = "Downloading scan report is failed for the application: " + scanContext.getAppId(); + String logMessage = "Error occcured in the Veracode while retrieving the scan results for " + + "application : " + scanContext.getAppId(); log.error(new CallbackLog(scanContext.getJobId(), logMessage)); - - CallbackUtil.updateScanStatus(scanContext.getJobId(), ScanStatus.ERROR, null, null); } } } catch (XPathExpressionException | ParserConfigurationException | SAXException | IOException | @@ -622,15 +644,21 @@ private boolean uploadReportToFtp(String scanArtifact, String reportPath) { String scanReportFtpLocation = scanArtifact.substring(0, scanArtifact.lastIndexOf(File.separator)); File reports = new File(reportPath); - - if (!Thread.currentThread().isInterrupted()) { + String ftpUsername = VeracodeScannerConfiguration.getInstance().getConfigProperty(ScannerConstants + .FTP_USERNAME); + char[] ftpPassword = VeracodeScannerConfiguration.getInstance().getConfigProperty(ScannerConstants + .FTP_PASSWORD).toCharArray(); + String ftpHost = VeracodeScannerConfiguration.getInstance().getConfigProperty(ScannerConstants.FTP_HOST); + int ftpPort = Integer.parseInt(VeracodeScannerConfiguration.getInstance().getConfigProperty(ScannerConstants + .FTP_PORT)); + + if (Thread.currentThread().isInterrupted()) { + String logMessage = "Current thread is interrupted for application : " + scanContext.getAppId(); + log.error(new CallbackLog(scanContext.getJobId(), logMessage)); + } else { try { - FileUtil.uploadReport(scanReportFtpLocation, reports, - VeracodeScannerConfiguration.getInstance().getConfigProperty(ScannerConstants.FTP_USERNAME), - (VeracodeScannerConfiguration.getInstance().getConfigProperty(ScannerConstants.FTP_PASSWORD)) - .toCharArray(), VeracodeScannerConfiguration.getInstance().getConfigProperty( - ScannerConstants.FTP_HOST), Integer.parseInt(VeracodeScannerConfiguration.getInstance() - .getConfigProperty(ScannerConstants.FTP_PORT))); + FileUtil.uploadReport(scanReportFtpLocation, reports, ftpUsername, ftpPassword, ftpHost, ftpPort); + FileUtil.cleanPassword(ftpPassword); isReportUploaded = true; CallbackUtil.updateScanStatus(scanContext.getJobId(), ScanStatus.COMPLETED, scanReportFtpLocation + @@ -654,9 +682,6 @@ private boolean uploadReportToFtp(String scanArtifact, String reportPath) { log.error(new CallbackLog(scanContext.getJobId(), e.getMessage())); CallbackUtil.updateScanStatus(scanContext.getJobId(), ScanStatus.ERROR, scanReportFtpLocation, null); } - } else { - String logMessage = "Current thread is interrupted for application : " + scanContext.getAppId(); - log.error(new CallbackLog(scanContext.getJobId(), logMessage)); } return isReportUploaded; } @@ -686,7 +711,10 @@ public ScanStatus getScanStatus() throws IOException, XPathExpressionException, public boolean getReports() { boolean isReportPrinted = false; - if (!Thread.currentThread().isInterrupted()) { + if (Thread.currentThread().isInterrupted()) { + String logMessage = "Current thread is interrupted for application : " + scanContext.getAppId(); + log.error(new CallbackLog(scanContext.getJobId(), logMessage)); + } else { try { ResultsAPIWrapper resultsAPIWrapper = VeracodeAPIUtil.getResultAPIWrapper(); String buildId = getBuildIDByAppId(scanContext.getAppId()); @@ -694,21 +722,27 @@ public boolean getReports() { String resultXMLDetailed = resultsAPIWrapper.detailedReport(buildId); byte[] resultPdfSummary = resultsAPIWrapper.summaryReportPdf(buildId); String resultXMLSummary = resultsAPIWrapper.summaryReport(buildId); - byte[] resultXMLThridParty = resultsAPIWrapper.thirdPartyReportPdf(buildId); + byte[] resultXMLThirdParty = resultsAPIWrapper.thirdPartyReportPdf(buildId); String filePath = VeracodeScannerConfiguration.getInstance().getConfigProperty( VeracodeScannerConstants.VERACODE_OUTPUT_FOLDER_PATH) + File.separator + scanContext.getAppId(); - FileUtil.saveReport(resultPdfDetailed, filePath + ScannerConstants.PDF_FILE_EXTENSION); - FileUtil.saveReport(resultXMLDetailed.getBytes(StandardCharsets.UTF_8.name()), filePath + - ScannerConstants.XML_FILE_EXTENSION); - FileUtil.saveReport(resultPdfSummary, filePath + VeracodeScannerConstants.SUMMARY + - ScannerConstants.PDF_FILE_EXTENSION); - FileUtil.saveReport(resultXMLSummary.getBytes(StandardCharsets.UTF_8.name()), filePath + - VeracodeScannerConstants.SUMMARY + ScannerConstants.XML_FILE_EXTENSION); - FileUtil.saveReport(resultXMLThridParty, filePath + VeracodeScannerConstants.THIRD_PARTY - + ScannerConstants.PDF_FILE_EXTENSION); - isReportPrinted = true; + String pdfDetailReport = filePath + ScannerConstants.PDF_FILE_EXTENSION; + String xmlDetailReport = filePath + ScannerConstants.XML_FILE_EXTENSION; + String pdfSummaryReport = filePath + VeracodeScannerConstants.SUMMARY + + ScannerConstants.PDF_FILE_EXTENSION; + String xmlSummaryReport = filePath + VeracodeScannerConstants.SUMMARY + + ScannerConstants.XML_FILE_EXTENSION; + String xmlThirdPartyReport = filePath + VeracodeScannerConstants.THIRD_PARTY + + ScannerConstants.PDF_FILE_EXTENSION; + + FileUtil.saveReport(resultPdfDetailed, pdfDetailReport); + FileUtil.saveReport(resultXMLDetailed.getBytes(StandardCharsets.UTF_8.name()), xmlDetailReport); + FileUtil.saveReport(resultPdfSummary, pdfSummaryReport); + FileUtil.saveReport(resultXMLSummary.getBytes(StandardCharsets.UTF_8.name()), xmlSummaryReport); + FileUtil.saveReport(resultXMLThirdParty, xmlThirdPartyReport); + + isReportPrinted = true; String logMessage = "Scan reports are completed and downloaded to the location : " + VeracodeScannerConfiguration.getInstance().getConfigProperty(VeracodeScannerConstants. VERACODE_OUTPUT_FOLDER_PATH) + " for the application " + scanContext.getAppId(); @@ -719,9 +753,6 @@ public boolean getReports() { + scanContext.getAppId(); log.error(new CallbackLog(scanContext.getJobId(), logMessage)); } - } else { - String logMessage = "Current thread is interrupted for application : " + scanContext.getAppId(); - log.error(new CallbackLog(scanContext.getJobId(), logMessage)); } return isReportPrinted; } diff --git a/internal/scan-manager/scanners/veracode/src/main/java/org/wso2/security/tools/scanmanager/scanners/veracode/service/VeracodeScanner.java b/internal/scan-manager/scanners/veracode/src/main/java/org/wso2/security/tools/scanmanager/scanners/veracode/service/VeracodeScanner.java index 74cf44a1..fbc9e83c 100644 --- a/internal/scan-manager/scanners/veracode/src/main/java/org/wso2/security/tools/scanmanager/scanners/veracode/service/VeracodeScanner.java +++ b/internal/scan-manager/scanners/veracode/src/main/java/org/wso2/security/tools/scanmanager/scanners/veracode/service/VeracodeScanner.java @@ -77,17 +77,18 @@ public VeracodeScanner() throws IOException { VeracodeScannerConstants.VERACODE_API_KEY)); VeracodeAPIUtil.setCredentials(options); - CallbackUtil.setCallbackUrls(ScannerConstants.HTTP_PROTOCOL + System.getenv(ScannerConstants.SCAN_MANAGER_HOST) - + ":" + System.getenv(ScannerConstants.SCAN_MANAGER_PORT) + VeracodeScannerConfiguration - .getInstance().getConfigProperty(ScannerConstants.SCAN_MANAGER_CALLBACK_URL_ENDPOINT) + - VeracodeScannerConfiguration.getInstance().getConfigProperty(ScannerConstants - .SCAN_MANAGER_CALLBACK_LOG), ScannerConstants.HTTP_PROTOCOL + System.getenv( - ScannerConstants.SCAN_MANAGER_HOST) + ":" + System.getenv(ScannerConstants - .SCAN_MANAGER_PORT) + VeracodeScannerConfiguration.getInstance().getConfigProperty( - ScannerConstants.SCAN_MANAGER_CALLBACK_URL_ENDPOINT) + VeracodeScannerConfiguration - .getInstance().getConfigProperty(ScannerConstants.SCAN_MANAGER_CALLBACK_STATUS), - Long.parseLong(VeracodeScannerConfiguration.getInstance().getConfigProperty(ScannerConstants - .CALLBACK_RETRY_INCREASE_SECONDS))); + + String callbackUrl = ScannerConstants.HTTP_PROTOCOL + System.getenv(ScannerConstants.SCAN_MANAGER_HOST) + ":" + + System.getenv(ScannerConstants.SCAN_MANAGER_PORT) + VeracodeScannerConfiguration.getInstance() + .getConfigProperty(ScannerConstants.SCAN_MANAGER_CALLBACK_URL_ENDPOINT); + String logCallbackUrl = callbackUrl + VeracodeScannerConfiguration.getInstance().getConfigProperty( + ScannerConstants.SCAN_MANAGER_CALLBACK_LOG); + String statusCallbackUrl = callbackUrl + VeracodeScannerConfiguration.getInstance().getConfigProperty( + ScannerConstants.SCAN_MANAGER_CALLBACK_STATUS); + Long callbackRetryInterval = Long.parseLong(VeracodeScannerConfiguration.getInstance().getConfigProperty( + ScannerConstants.CALLBACK_RETRY_INCREASE_SECONDS)); + + CallbackUtil.setCallbackUrls(logCallbackUrl, statusCallbackUrl, callbackRetryInterval); } /** @@ -113,18 +114,18 @@ public void startScan(ScannerScanRequest scanRequest) { scanContext.setArtifactLocation(scanRequest.getFileMap().get(VeracodeScannerConstants.SCAN_ARTIFACT).get(0)); if (scanRequest.getFileMap().get(VeracodeScannerConstants.SCAN_ARTIFACT) != null) { - if (!StringUtils.isEmpty(scanContext.getAppId())) { - if (!Thread.currentThread().isInterrupted()) { - ScanTask scanTask = new ScanTask(scanContext); - scanTask.run(); - } else { - String message = "Current thread is interrupted. "; - log.error(new CallbackLog(scanContext.getJobId(), message)); - } - } else { + if (StringUtils.isEmpty(scanContext.getAppId())) { String message = "Error occured while submitting the start scan request since the application " + "is empty in the request. "; callbackErrorReport(message); + } else { + if (Thread.currentThread().isInterrupted()) { + String message = "Current thread is interrupted. "; + log.error(new CallbackLog(scanContext.getJobId(), message)); + } else { + ScanTask scanTask = new ScanTask(scanContext); + scanTask.run(); + } } } else { String message = "Error occured while submitting the start scan request since the scan artifacts " + @@ -136,10 +137,10 @@ public void startScan(ScannerScanRequest scanRequest) { @Override public boolean validateStartScan(ScannerScanRequest scannerScanRequest) { - if (!StringUtils.isEmpty(scannerScanRequest.getFileMap().get(VeracodeScannerConstants.SCAN_ARTIFACT))) { - return true; - } else { + if (StringUtils.isEmpty(scannerScanRequest.getFileMap().get(VeracodeScannerConstants.SCAN_ARTIFACT))) { return false; + } else { + return true; } }