Merge branch 'main' into ui-feedback-ch-3

.github/workflows/docker-openmetadata-server.yml

@@ -113,6 +113,9 @@ jobs:
   push_to_docker_hub:
     runs-on: ubuntu-latest
     env:
+      - name: Set build arguments
+        run: |
+          echo "BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_ENV
       DOCKER_RELEASE_TAG: ${{ github.event.inputs.DOCKER_RELEASE_TAG }}
     if: ${{ always() && contains(join(needs.*.result, ','), 'success') }}
     needs: [release-project-event-workflow_dispatch, release-project-event-release]  
@@ -156,3 +159,6 @@ jobs:
           push: ${{ github.event_name == 'release' || github.event_name == 'workflow_dispatch' }}
           tags: ${{ steps.input_check.outputs.tags }}
           file: ./docker/docker-compose-quickstart/Dockerfile
+          build-args: |
+            BUILD_DATE=${{ env.BUILD_DATE }}
+            COMMIT_ID=${{ env.github.sha }}

bootstrap/sql/org.postgresql.Driver/v013__create_db_connection_info.sql

@@ -5,20 +5,20 @@ where name = 'OpenMetadata';
 
 -- Rename githubCredentials to gitCredentials
 UPDATE dashboard_service_entity
-SET json = jsonb_set(json, '{connection,config,gitCredentials}', json#>'{connection,config,githubCredentials}')
+SET json = jsonb_set(json::jsonb #- '{connection,config,githubCredentials}', '{connection,config,gitCredentials}', json#>'{connection,config,githubCredentials}')
     where serviceType = 'Looker'
   and json#>'{connection,config,githubCredentials}' is not null;
 
 -- Rename gcsConfig in BigQuery to gcpConfig
 UPDATE dbservice_entity
-SET json = jsonb_set(json, '{connection,config,credentials,gcpConfig}',
+SET json = jsonb_set(json::jsonb #- '{connection,config,credentials,gcsConfig}', '{connection,config,credentials,gcpConfig}',
 json#>'{connection,config,credentials,gcsConfig}')
 where serviceType in ('BigQuery')
   and json#>'{connection,config,credentials,gcsConfig}' is not null;
 
 -- Rename gcsConfig in Datalake to gcpConfig
 UPDATE dbservice_entity
-SET json = jsonb_set(json, '{connection,config,configSource,securityConfig,gcpConfig}',
+SET json = jsonb_set(json::jsonb #- '{connection,config,configSource,securityConfig,gcsConfig}', '{connection,config,configSource,securityConfig,gcpConfig}',
 json#>'{connection,config,configSource,securityConfig,gcsConfig}')
 where serviceType in ('Datalake')
   and json#>'{connection,config,configSource,securityConfig,gcsConfig}' is not null;

docker/development/Dockerfile

@@ -9,17 +9,27 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-FROM alpine:3
+# Build stage
+FROM alpine:3.15 AS build
 
-EXPOSE 8585
-
-RUN  apk update \
-  && apk upgrade \
-  && apk add --update wget curl bash openjdk11 \
-  && rm -rf /var/cache/apk/*
-COPY docker/openmetadata-start.sh ./
 COPY openmetadata-dist/target/openmetadata-*.tar.gz /
-RUN tar zxvf openmetadata-*.tar.gz && \
+
+RUN mkdir -p /opt/openmetadata && \
+    tar zxvf openmetadata-*.tar.gz -C /opt/openmetadata --strip-components 1 && \
     rm openmetadata-*.tar.gz
+
+# Final stage
+FROM alpine:3.15
+
+EXPOSE 8585
+
+RUN apk add --update --no-cache bash openjdk11-jre
+
+COPY --from=build /opt/openmetadata /opt/openmetadata
+COPY docker/openmetadata-start.sh /
+
 RUN chmod 777 openmetadata-start.sh
-CMD ["./openmetadata-start.sh"]
+
+WORKDIR /opt/openmetadata
+ENTRYPOINT [ "/bin/bash" ]
+CMD ["/openmetadata-start.sh"]

docker/development/docker-compose-postgres.yml

@@ -31,6 +31,8 @@ services:
       - 5432
     ports:
       - "5432:5432"
+    volumes:
+     - ./docker-volume/db-data-postgres:/var/lib/postgresql/data
     networks:
       - local_app_net
     healthcheck:

docker/development/docker-compose.yml

@@ -37,6 +37,8 @@ services:
       interval: 15s
       timeout: 10s
       retries: 10
+    volumes:
+     - ./docker-volume/db-data:/var/lib/mysql
 
   elasticsearch:
     image: docker.elastic.co/elasticsearch/elasticsearch:7.16.0

docker/docker-compose-quickstart/Dockerfile

@@ -9,18 +9,33 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-FROM alpine:3
-
-EXPOSE 8585
+# Build stage
+FROM alpine:3.15 AS build
+ENV RELEASE_URL="https://github.com/open-metadata/OpenMetadata/releases/download/1.0.0-alpha-release/openmetadata-1.0.0-alpha.tar.gz"
+RUN mkdir -p /opt/openmetadata && \
+    wget ${RELEASE_URL} && \
+    tar zxvf openmetadata-*.tar.gz -C /opt/openmetadata --strip-components 1 && \
+    rm openmetadata-*.tar.gz
 
-RUN  apk update \
-  && apk upgrade \
-  && apk add --update wget curl bash openjdk11 \
-  && rm -rf /var/cache/apk/*
+# Final stage
+FROM alpine:3.15
+ARG BUILD_DATE
+ARG COMMIT_ID
+LABEL maintainer="OpenMetadata"
+LABEL org.open-metadata.image.authors="support@openmetadata.org"
+LABEL org.open-metadata.vendor="OpenMetadata"
+LABEL org.open-metadata.release-version="1.0.0-alpha"
+LABEL org.open-metadata.description="OpenMetadata is an open source platform for metadata management and discovery."
+LABEL org.open-metadata.url="https://open-metadata.org/"
+LABEL org.open-metadata.vcs-url="https://github.com/open-metadata/OpenMetadata"
+LABEL org.open-metadata.build-date=$BUILD_DATE
+LABEL org.open-metadata.commit-id=$COMMIT_ID
 
-COPY docker/openmetadata-start.sh docker/openmetadata.yaml ./
-RUN wget https://github.com/open-metadata/OpenMetadata/releases/download/1.0.0-alpha-release/openmetadata-1.0.0-alpha.tar.gz && \
-    tar zxvf openmetadata-*.tar.gz && \
-    rm openmetadata-*.tar.gz
-RUN chmod 777 openmetadata-start.sh
-CMD ["./openmetadata-start.sh"]
+EXPOSE 8585 8586
+COPY docker/openmetadata-start.sh ./
+COPY --from=build /opt/openmetadata /opt/openmetadata
+RUN apk add --update --no-cache bash openjdk11-jre && \
+    chmod 777 openmetadata-start.sh
+WORKDIR /opt/openmetadata
+ENTRYPOINT [ "/bin/bash" ]
+CMD ["/openmetadata-start.sh"]

docker/openmetadata-start.sh

@@ -11,7 +11,6 @@
 #  limitations under the License.
 
 echo "Initializing OpenMetadata Server...";
-cd /openmetadata-*/
 echo "Migrating the database to the latest version and the indexes in ElasticSearch...";
 ./bootstrap/bootstrap_storage.sh migrate-all
 echo "    ||||||| "

docker/run_local_docker.sh

@@ -21,11 +21,11 @@ helpFunction()
    printf "\t-s Skip maven build: [true, false]. Default [false]\n"
    printf "\t-x Open JVM debug port on 5005: [true, false]. Default [false]\n"
    printf "\t-h For usage help\n"
-   printf "\t-r For Cleaning DB Volumes"
+   printf "\t-r For Cleaning DB Volumes. [true, false]. Default [true]\n"
    exit 1 # Exit script after printing help
 }
 
-while getopts "m:d:s:x:h" opt
+while getopts "m:d:s:x:r:h" opt
 do
    case "$opt" in
       m ) mode="$OPTARG" ;;
@@ -43,8 +43,9 @@ database="${database:=mysql}"
 skipMaven="${skipMaven:=false}"
 debugOM="${debugOM:=false}"
 authorizationToken="eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg"
+cleanDbVolumes="${cleanDbVolumes:=true}"
 
-echo "Running local docker using mode [$mode] database [$database] and skipping maven build [$skipMaven]"
+echo "Running local docker using mode [$mode] database [$database] and skipping maven build [$skipMaven] with cleanDB as [$cleanDbVolumes]"
 
 cd ../
 
@@ -76,9 +77,9 @@ fi
 
 if [[ $cleanDbVolumes == "true" ]]
 then
-  if [[ -d "/docker-volume" ]]
+  if [[ -d "$PWD/docker/development/docker-volume/" ]]
   then
-      rm -rf $PWD/docker-volume
+      rm -rf $PWD/docker/development/docker-volume
     fi
 fi
 

ingestion/examples/sample_data/topics/topics.json

@@ -9,7 +9,11 @@
 			"maximumMessageSize":167,
 			"cleanupPolicies": ["delete"],
 			"schemaType": "Avro",
-			"schemaText": "{\"namespace\":\"openmetadata.kafka\",\"name\":\"Customer\",\"type\":\"record\",\"fields\":[{\"name\":\"id\",\"type\":\"string\"},{\"name\":\"first_name\",\"type\":\"string\"},{\"name\":\"last_name\",\"type\":\"string\"},{\"name\":\"email\",\"type\":\"string\"},{\"name\":\"address_line_1\",\"type\":\"string\"},{\"name\":\"address_line_2\",\"type\":\"string\"},{\"name\":\"post_code\",\"type\":\"string\"},{\"name\":\"country\",\"type\":\"string\"}]}"
+			"schemaText": "{\"namespace\":\"openmetadata.kafka\",\"name\":\"Customer\",\"type\":\"record\",\"fields\":[{\"name\":\"id\",\"type\":\"string\"},{\"name\":\"first_name\",\"type\":\"string\"},{\"name\":\"last_name\",\"type\":\"string\"},{\"name\":\"email\",\"type\":\"string\"},{\"name\":\"address_line_1\",\"type\":\"string\"},{\"name\":\"address_line_2\",\"type\":\"string\"},{\"name\":\"post_code\",\"type\":\"string\"},{\"name\":\"country\",\"type\":\"string\"}]}",
+			"sampleData": [
+				"{\"id\": 1, \"first_name\": \"Lévy\", \"country\": \"ES\"}",
+				"{\"id\": 2, \"first_name\": \"Lima\", \"country\": \"ES\"}"
+			]
 		},
 		{
 			"name": "product_events",

ingestion/src/metadata/ingestion/source/database/sample_data.py

@@ -63,6 +63,7 @@
     TableData,
     TableProfile,
 )
+from metadata.generated.schema.entity.data.topic import Topic, TopicSampleData
 from metadata.generated.schema.entity.policies.policy import Policy
 from metadata.generated.schema.entity.services.connections.database.customDatabaseConnection import (
     CustomDatabaseConnection,
@@ -86,7 +87,7 @@
 from metadata.generated.schema.tests.testSuite import TestSuite
 from metadata.generated.schema.type.entityLineage import EntitiesEdge, LineageDetails
 from metadata.generated.schema.type.entityReference import EntityReference
-from metadata.generated.schema.type.schema import Topic
+from metadata.generated.schema.type.schema import Topic as TopicSchema
 from metadata.ingestion.api.common import Entity
 from metadata.ingestion.api.source import InvalidSourceException, Source
 from metadata.ingestion.models.pipeline_status import OMetaPipelineStatus
@@ -679,7 +680,7 @@ def ingest_topics(self) -> Iterable[CreateTopicRequest]:
                     )
                 schema_fields = load_parser_fn(topic["name"], topic["schemaText"])
 
-                create_topic.messageSchema = Topic(
+                create_topic.messageSchema = TopicSchema(
                     schemaText=topic["schemaText"],
                     schemaType=topic["schemaType"],
                     schemaFields=schema_fields,
@@ -688,6 +689,22 @@ def ingest_topics(self) -> Iterable[CreateTopicRequest]:
             self.status.scanned(f"Topic Scanned: {create_topic.name.__root__}")
             yield create_topic
 
+            if topic.get("sampleData"):
+
+                topic_fqn = fqn.build(
+                    self.metadata,
+                    entity_type=Topic,
+                    service_name=self.kafka_service.name.__root__,
+                    topic_name=topic["name"],
+                )
+
+                topic_entity = self.metadata.get_by_name(entity=Topic, fqn=topic_fqn)
+
+                self.metadata.ingest_topic_sample_data(
+                    topic=topic_entity,
+                    sample_data=TopicSampleData(messages=topic["sampleData"]),
+                )
+
     def ingest_looker(self) -> Iterable[Entity]:
         """
         Looker sample data

ingestion/tests/unit/test_query_parser.py

@@ -230,14 +230,10 @@ def test_ctes_column_lineage(self):
         expected_lineage = [
             (
                 Column("testdb.public.users.id"),
-                Column("cte_table.id"),
-                Column("cte_table2.id"),
                 Column("testdb.public.target.id"),
             ),
             (
                 Column("testdb.public.users.name"),
-                Column("cte_table.name"),
-                Column("cte_table2.name"),
                 Column("testdb.public.target.name"),
             ),
         ]

openmetadata-service/src/main/java/org/openmetadata/service/clients/pipeline/airflow/AirflowRESTClient.java

@@ -22,6 +22,7 @@
 import java.net.http.HttpResponse;
 import java.security.KeyStoreException;
 import java.time.Duration;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 import javax.net.ssl.SSLContext;
@@ -219,10 +220,11 @@ public List<PipelineStatus> getQueuedPipelineStatus(IngestionPipeline ingestionP
     } catch (Exception e) {
       throw PipelineServiceClientException.byMessage(ingestionPipeline.getName(), e.getMessage());
     }
-    throw PipelineServiceClientException.byMessage(
-        ingestionPipeline.getName(),
-        "Failed to fetch ingestion pipeline runs",
-        Response.Status.fromStatusCode(response.statusCode()));
+    // Return an empty list. We'll just show the stored status from the Ingestion Pipeline
+    LOG.error(
+        String.format(
+            "Got status code [%s] trying to get queued statuses: [%s]", response.statusCode(), response.body()));
+    return new ArrayList<>();
   }
 
   /**

openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java

@@ -78,6 +78,7 @@
 import org.openmetadata.service.resources.databases.DatabaseUtil;
 import org.openmetadata.service.resources.databases.TableResource;
 import org.openmetadata.service.resources.feeds.MessageParser.EntityLink;
+import org.openmetadata.service.security.mask.PIIMasker;
 import org.openmetadata.service.util.EntityUtil;
 import org.openmetadata.service.util.EntityUtil.Fields;
 import org.openmetadata.service.util.FullyQualifiedName;
@@ -243,7 +244,11 @@ public Table getSampleData(UUID tableId, boolean authorizePII) throws IOExceptio
     setFieldsInternal(table, Fields.EMPTY_FIELDS);
 
     // Set the column tags. Will be used to mask the sample data
-    if (!authorizePII) getColumnTags(true, table.getColumns());
+    if (!authorizePII) {
+      getColumnTags(true, table.getColumns());
+      table.setTags(getTags(table.getFullyQualifiedName()));
+      return PIIMasker.getSampleData(table);
+    }
 
     return table;
   }
@@ -472,7 +477,10 @@ public Table getLatestTableProfile(String fqn, boolean authorizePII) throws IOEx
     setColumnProfile(table.getColumns());
 
     // Set the column tags. Will be used to hide the data
-    if (!authorizePII) getColumnTags(true, table.getColumns());
+    if (!authorizePII) {
+      getColumnTags(true, table.getColumns());
+      return PIIMasker.getTableProfile(table);
+    }
 
     return table;
   }