Skip to content

Commit

Permalink
Simplifying changes
Browse files Browse the repository at this point in the history
Signed-off-by: Jonathan Moraes <jonathan.pinto@cs.unicid.edu.br>
  • Loading branch information
jonathanlbt1 committed Sep 9, 2024
1 parent 12f081f commit 0c5997d
Show file tree
Hide file tree
Showing 28 changed files with 38 additions and 37 deletions.
2 changes: 1 addition & 1 deletion .circleci/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -196,4 +196,4 @@ workflows:
- determine_changed_modules:
filters:
tags:
only: /^[0-9]+(\.[0-9]+){2}(-rc\.[0-9]+)?$/
only: /^[0-9]+(\.[0-9]+){2}(-rc\.[0-9]+)?$/
8 changes: 4 additions & 4 deletions .circleci/continue_config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -534,7 +534,7 @@ jobs:
export ORG_GRADLE_PROJECT_signingKey=$(echo $GPG_SIGNING_KEY | base64 -d)
export RELEASE_PASSWORD=$(echo $OSSRH_TOKEN_PASSWORD)
export RELEASE_USERNAME=$(echo $OSSRH_TOKEN_USERNAME)
./gradlew --console=plain javadoc sourceJar shadowJar -Pscala.binary.version=<< parameters.scala >> -Pjava.compile.home=/usr/lib/jvm/java-17-openjdk-amd64
./gradlew --console=plain publishToMavenLocal -Pscala.binary.version=<< parameters.scala >> -Pjava.compile.home=/usr/lib/jvm/java-17-openjdk-amd64
- persist_to_workspace:
Expand Down Expand Up @@ -1010,8 +1010,8 @@ jobs:
paths:
- ~/.gradle

build-proxy-fluent-plugin-openlineage:
working_directory: ~/openlineage/proxy/fluent-plugin-openlineage
build-proxy-fluentd:
working_directory: ~/openlineage/proxy/fluentd
docker:
- image: cimg/ruby:3.2
steps:
Expand Down Expand Up @@ -1168,4 +1168,4 @@ jobs:
machine:
image: ubuntu-2404:current
steps:
- run: echo "Complete"
- run: echo "Complete"
11 changes: 6 additions & 5 deletions .circleci/workflows/openlineage-proxy-fluentd.yml
Original file line number Diff line number Diff line change
@@ -1,22 +1,23 @@
workflows:
openlineage-proxy-fluent-plugin-openlineage:
openlineage-proxy-fluentd:
jobs:
- build-proxy-fluent-plugin-openlineage:
- build-proxy-fluentd:
filters:
tags:
only: /^[0-9]+(\.[0-9]+){2}(-rc\.[0-9]+)?$/
- build-proxy-fluent-plugin-openlineage:
- release-docker-proxy-backend:
filters:
tags:
only: /^[0-9]+(\.[0-9]+){2}(-rc\.[0-9]+)?$/
branches:
only: main
context: release
requires:
- build-proxy-fluent-plugin-openlineage
- build-proxy-fluentd
- workflow_complete:
filters:
tags:
only: /^[0-9]+(\.[0-9]+){2}(-rc\.[0-9]+)?$/
requires:
# - build-image-proxy-fluent-plugin-openlineage
- build-proxy-fluentd
- release-docker-proxy-backend
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ RUN buildDeps="sudo make gcc g++ libc-dev" \
&& rm -rf /var/lib/apt/lists/* \
&& rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem

COPY proxy/fluent-plugin-openlineage/docker/entrypoint.sh /bin/entrypoint.sh
COPY proxy/fluentd/docker/entrypoint.sh /bin/entrypoint.sh
COPY /spec /etc/spec

USER fluent
File renamed without changes.
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ services:
proxy:
build:
context: ../../..
dockerfile: proxy/fluent-plugin-openlineage/docker/Dockerfile
dockerfile: proxy/fluentd/docker/Dockerfile
container_name: openlineage-fluentd
volumes:
- ./conf:/fluentd/etc/
Expand Down
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
require_relative '../helper'
require_relative '../../lib/fluent/plugin/parser_openlineage.rb'
require "helper"
require "fluent/plugin/parser_openlineage.rb"

class OpenlineageParserTest < Test::Unit::TestCase
setup do
Expand All @@ -10,14 +10,14 @@ class OpenlineageParserTest < Test::Unit::TestCase
)
end

test "test event with no runId" do
ol_event = File.read("events/event_no_run_id.json")
test "test event with no runId" do
ol_event = File.read("events/event_no_run_id.json")

err = assert_raise Fluent::ParserError do
@parser.instance.parse(ol_event)
end
assert_match(/Openlineage validation failed: (.+) path "\/run": "runId" is a required property/, err.message)
end
err = assert_raise Fluent::ParserError do
@parser.instance.parse(ol_event)
end
assert_match(/Openlineage validation failed: (.+) path "\/run": "runId" is a required property/, err.message)
end

test "test invalid json" do
assert_raise Fluent::ParserError do
Expand All @@ -28,8 +28,8 @@ class OpenlineageParserTest < Test::Unit::TestCase
test "event full test" do
ol_event = File.read("events/event_full.json")
@parser.instance.parse(ol_event) { | time, json |
assert_equal("ea041791-68bc-4ae1-bd89-4c8106a157e4", json['run']['runId'])
assert_equal(2000, json['outputs'][0]['outputFacets']['outputStatistics']['rowCount'])
assert_equal("ea041791-68bc-4ae1-bd89-4c8106a157e4", json['run']['runId'])
assert_equal(2000, json['outputs'][0]['outputFacets']['outputStatistics']['rowCount'])
}
end

Expand All @@ -40,13 +40,13 @@ class OpenlineageParserTest < Test::Unit::TestCase
}
end

test "invalid spec_directory test" do
assert_raise Fluent::ParserError do
@parser.configure(
'spec_directory' => './non-existent-spec/'
)
end
end
test "invalid spec_directory test" do
assert_raise Fluent::ParserError do
@parser.configure(
'spec_directory' => './non-existent-spec/'
)
end
end

test "valid spec_directory without slash" do
@parser.configure(
Expand All @@ -71,7 +71,7 @@ class OpenlineageParserTest < Test::Unit::TestCase
@parser.configure(
'spec_directory' => '../../spec/',
'validate_run_facets' => false,
)
)
@parser.instance.parse(ol_event) { | time, json |
assert_equal("41fb5137-f0fd-4ee5-ba5c-56f8571d1bd7", json['run']['runId'])
}
Expand All @@ -90,7 +90,7 @@ class OpenlineageParserTest < Test::Unit::TestCase
@parser.configure(
'spec_directory' => '../../spec/',
'validate_job_facets' => false,
)
)
@parser.instance.parse(ol_event) { | time, json |
assert_equal("41fb5137-f0fd-4ee5-ba5c-56f8571d1bd7", json['run']['runId'])
}
Expand All @@ -101,7 +101,7 @@ class OpenlineageParserTest < Test::Unit::TestCase
@parser.configure(
'spec_directory' => '../../spec/',
'validate_dataset_facets' => true,
)
)
err = assert_raise Fluent::ParserError do
@parser.instance.parse(ol_event)
end
Expand All @@ -110,11 +110,11 @@ class OpenlineageParserTest < Test::Unit::TestCase

test "input dataset facet validation" do
ol_event = File.read("events/event_invalid_input_dataset_facet.json")
@parser.configure(
@parser.configure(
'json_parser' => 'yajl',
'spec_directory' => '../../spec/',
'validate_input_dataset_facets' => true,
)
)
err = assert_raise Fluent::ParserError do
@parser.instance.parse(ol_event)
end
Expand All @@ -126,7 +126,7 @@ class OpenlineageParserTest < Test::Unit::TestCase
@parser.configure(
'spec_directory' => '../../spec/',
'validate_output_dataset_facets' => true,
)
)
err = assert_raise Fluent::ParserError do
@parser.instance.parse(ol_event)
end
Expand All @@ -138,4 +138,4 @@ class OpenlineageParserTest < Test::Unit::TestCase
def create_driver(conf)
Fluent::Test::Driver::Parser.new(Fluent::Plugin::OpenlineageParser).configure(conf)
end
end
end

0 comments on commit 0c5997d

Please sign in to comment.