diff --git a/README.md b/README.md index 0a2b836f475..705913547c8 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,8 @@ Gobblin is a universal data ingestion framework for extracting, transforming, and loading large volume of data from a variety of data sources, e.g., databases, rest APIs, FTP/SFTP servers, filers, etc., onto Hadoop. Gobblin handles the common routine tasks required for all data ingestion ETLs, including job/task scheduling, task partitioning, error handling, state management, data quality checking, data publishing, etc. Gobblin ingests data from different data sources in the same execution framework, and manages metadata of different sources all in one place. This, combined with other features such as auto scalability, fault tolerance, data quality assurance, extensibility, and the ability of handling data model evolution, makes Gobblin an easy-to-use, self-serving, and efficient data ingestion framework. ## Documentation -Check out the Gobblin documentation at [https://github.com/linkedin/gobblin/wiki](https://github.com/linkedin/gobblin/wiki) + +Check out the Gobblin documentation at [https://github.com/linkedin/gobblin/wiki](https://github.com/linkedin/gobblin/wiki). ## Getting Started @@ -42,7 +43,6 @@ Out of the box, Gobblin can run either in standalone mode on a single box or on Please refer to the page [Getting Started](https://github.com/linkedin/gobblin/wiki/Getting%20Started) in the documentation on how to run the examples. - ## Configuration Please refer to the page [Configuration Glossary](https://github.com/linkedin/gobblin/wiki/Configuration%20Properties%20Glossary) diff --git a/bin/gobblin-standalone.sh b/bin/gobblin-standalone.sh index f47111ba498..f5e0ed486c6 100755 --- a/bin/gobblin-standalone.sh +++ b/bin/gobblin-standalone.sh @@ -6,7 +6,7 @@ FWDIR_CONF=$FWDIR/conf function print_usage(){ echo "gobblin-standalone.sh [OPTION]" - echo "Where:" + echo "Where OPTION can be:" echo " --workdir Gobblin's base work directory: if not set, taken from \${GOBBLIN_WORK_DIR}" echo " --jars Job jar(s): if not set, "$FWDIR_LIB" is examined" echo " --conf Directory of job configuration files: if not set, taken from ${GOBBLIN_JOB_CONFIG_DIR}" @@ -53,12 +53,12 @@ if [ -z "$JAVA_HOME" ]; then fi check=false -if [ $ACTION == "start" ] || [ $ACTION == "restart" ]; then +if [ $ACTION == "start" ] || [ $ACTION == "restart" ]; then check=true fi # User defined job configuration directory overrides $GOBBLIN_JOB_CONFIG_DIR -if [ -n "$JOB_CONFIG_DIR" ]; then +if [ -n "$JOB_CONFIG_DIR" ]; then export GOBBLIN_JOB_CONFIG_DIR="$JOB_CONFIG_DIR" fi @@ -67,11 +67,11 @@ if [ -z "$GOBBLIN_JOB_CONFIG_DIR" ] && [ "$check" == true ]; then fi # User defined work directory overrides $GOBBLIN_WORK_DIR -if [ -n "$WORK_DIR" ]; then +if [ -n "$WORK_DIR" ]; then export GOBBLIN_WORK_DIR="$WORK_DIR" fi -if [ -z "$GOBBLIN_WORK_DIR" ] && [ "$check" == true ]; then +if [ -z "$GOBBLIN_WORK_DIR" ] && [ "$check" == true ]; then die "GOBBLIN_WORK_DIR is not set!" fi @@ -93,7 +93,7 @@ fi set_user_jars(){ local separator='' - if [ -n "$1" ]; then + if [ -n "$1" ]; then IFS=',' read -ra userjars <<< "$1" for userjar in ${userjars[@]}; do @@ -122,7 +122,7 @@ start() { GOBBLIN_JARS=$jar fi done - + CLASSPATH="$GOBBLIN_JARS:$FWDIR_CONF" echo "Starting Gobblin standalone daemon" diff --git a/gobblin-test/build.gradle b/gobblin-test/build.gradle deleted file mode 100644 index 5548b481b92..00000000000 --- a/gobblin-test/build.gradle +++ /dev/null @@ -1,59 +0,0 @@ -// (c) 2014 LinkedIn Corp. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use -// this file except in compliance with the License. You may obtain a copy of the -// License at http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. -// - -apply plugin: 'java' - -dependencies { - compile project(":gobblin-api") - - compile externalDependency.commonsCli - compile externalDependency.avro - compile externalDependency.avroMapred - compile externalDependency.gson - compile externalDependency.guava - compile externalDependency.jacksonCore - compile externalDependency.jacksonMapper - compile externalDependency.slf4j - - testCompile externalDependency.testng -} - -configurations { - compile { - transitive = false - } -} - -test { - useTestNG () { - excludeGroups 'ignore' - } - workingDir rootProject.rootDir -} - -classification="library" - -buildscript { - repositories { - jcenter() - } - dependencies { - classpath 'com.github.jengelman.gradle.plugins:shadow:0.8' - } -} - -apply plugin: 'shadow' - -shadow { - artifactSet { - exclude 'org.apache.hadoop' - } -} diff --git a/settings.gradle b/settings.gradle index 917ee8f2d80..8d97e86a381 100644 --- a/settings.gradle +++ b/settings.gradle @@ -9,7 +9,7 @@ // CONDITIONS OF ANY KIND, either express or implied. // -def modules = ['gobblin-api','gobblin-azkaban','gobblin-compaction','gobblin-core','gobblin-distribution','gobblin-example','gobblin-metrics','gobblin-metastore','gobblin-rest-service','gobblin-runtime','gobblin-scheduler','gobblin-test','gobblin-utility','gobblin-salesforce'] +def modules = ['gobblin-api','gobblin-azkaban','gobblin-compaction','gobblin-core','gobblin-distribution','gobblin-example','gobblin-metrics','gobblin-metastore','gobblin-rest-service','gobblin-runtime','gobblin-scheduler','gobblin-utility','gobblin-salesforce'] modules.each { module -> include "${module}"