feat(ui): supported: #11857 show detailed run status for ingestion pi…

openmetadata-ui/src/main/resources/ui/src/components/Ingestion/IngestionRecentRun/IngestionRecentRun.test.tsx

@@ -11,19 +11,54 @@
  *  limitations under the License.
  */
 
-import { act, render, screen } from '@testing-library/react';
+import {
+  act,
+  findByRole,
+  fireEvent,
+  render,
+  screen,
+} from '@testing-library/react';
 import React from 'react';
 import { IngestionPipeline } from '../../../generated/entity/services/ingestionPipelines/ingestionPipeline';
 import { getRunHistoryForPipeline } from '../../../rest/ingestionPipelineAPI';
+import ConnectionStepCard from '../../common/TestConnection/ConnectionStepCard/ConnectionStepCard';
 import { IngestionRecentRuns } from './IngestionRecentRuns.component';
 
+const failure = {
+  name: 'FILES',
+  error:
+    'Unexpected exception to yield table [FILES]: (pymysql.err.OperationalError) (1227, \'Access denied; you need (at least one of) the PROCESS privilege(s) for this operation\')\n[SQL: /* {"app": "OpenMetadata", "version": "1.3.0.0.dev0"} */\nSHOW CREATE TABLE `information_schema`.`FILES`]\n(Background on this error at: https://sqlalche.me/e/14/e3q8)',
+  stackTrace:
+    'Traceback (most recent call last):\n  File "/home/airflow/.local/lib/python3.10/site-packages/sqlalchemy/engine/base.py", line 1910, in _execute_context\n    self.dialect.do_execute(\n  File "/home/airflow/.local/lib/python3.10/site-packages/sqlalchemy/engine/default.py", line 736, in do_execute\n    cursor.execute(statement, parameters)\n  File "/home/airflow/.local/lib/python3.10/site-packages/pymysql/cursors.py", line 153, in execute\n    result = self._query(query)\n  File "/home/airflow/.local/lib/python3.10/site-packages/pymysql/cursors.py", line 322, in _query\n    conn.query(q)\n  File "/home/airflow/.local/lib/python3.10/site-packages/pymysql/connections.py", line 558, in query\n    self._affected_rows = self._read_query_result(unbuffered=unbuffered)\n  File "/home/airflow/.local/lib/python3.10/site-packages/pymysql/connections.py", line 822, in _read_query_result\n    result.read()\n  File "/home/airflow/.local/lib/python3.10/site-packages/pymysql/connections.py", line 1200, in read\n    first_packet = self.connection._read_packet()\n  File "/home/airflow/.local/lib/python3.10/site-packages/pymysql/connections.py", line 772, in _read_packet\n    packet.raise_for_error()\n  File "/home/airflow/.local/lib/python3.10/site-packages/pymysql/protocol.py", line 221, in raise_for_error\n    err.raise_mysql_exception(self._data)\n  File "/home/airflow/.local/lib/python3.10/site-packages/pymysql/err.py", line 143, in raise_mysql_exception\n    raise errorclass(errno, errval)\npymysql.err.OperationalError: (1227, \'Access denied; you need (at least one of) the PROCESS privilege(s) for this operation\')\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File "/home/airflow/.local/lib/python3.10/site-packages/metadata/ingestion/source/database/common_db_source.py", line 422, in yield_table\n    ) = self.get_columns_and_constraints(\n  File "/home/airflow/.local/lib/python3.10/site-packages/metadata/ingestion/source/database/sql_column_handler.py", line 214, in get_columns_and_constraints\n    ) = self._get_columns_with_constraints(schema_name, table_name, inspector)\n  File "/home/airflow/.local/lib/python3.10/site-packages/metadata/ingestion/source/database/sql_column_handler.py", line 114, in _get_columns_with_constraints\n    pk_constraints = inspector.get_pk_constraint(table_name, schema_name)\n  File "/home/airflow/.local/lib/python3.10/site-packages/sqlalchemy/engine/reflection.py", line 528, in get_pk_constraint\n    return self.dialect.get_pk_constraint(\n  File "<string>", line 2, in get_pk_constraint\n  File "/home/airflow/.local/lib/python3.10/site-packages/sqlalchemy/engine/reflection.py", line 55, in cache\n    ret = fn(self, con, *args, **kw)\n  File "/home/airflow/.local/lib/python3.10/site-packages/sqlalchemy/dialects/mysql/base.py", line 2842, in get_pk_constraint\n    parsed_state = self._parsed_state_or_create(\n  File "/home/airflow/.local/lib/python3.10/site-packages/sqlalchemy/dialects/mysql/base.py", line 3085, in _parsed_state_or_create\n    return self._setup_parser(\n  File "<string>", line 2, in _setup_parser\n  File "/home/airflow/.local/lib/python3.10/site-packages/sqlalchemy/engine/reflection.py", line 55, in cache\n    ret = fn(self, con, *args, **kw)\n  File "/home/airflow/.local/lib/python3.10/site-packages/sqlalchemy/dialects/mysql/base.py", line 3112, in _setup_parser\n    sql = self._show_create_table(\n  File "/home/airflow/.local/lib/python3.10/site-packages/sqlalchemy/dialects/mysql/base.py", line 3220, in _show_create_table\n    ).exec_driver_sql(st)\n  File "/home/airflow/.local/lib/python3.10/site-packages/sqlalchemy/engine/base.py", line 1770, in exec_driver_sql\n    return self._exec_driver_sql(\n  File "/home/airflow/.local/lib/python3.10/site-packages/sqlalchemy/engine/base.py", line 1674, in _exec_driver_sql\n    ret = self._execute_context(\n  File "/home/airflow/.local/lib/python3.10/site-packages/sqlalchemy/engine/base.py", line 1953, in _execute_context\n    self._handle_dbapi_exception(\n  File "/home/airflow/.local/lib/python3.10/site-packages/sqlalchemy/engine/base.py", line 2134, in _handle_dbapi_exception\n    util.raise_(\n  File "/home/airflow/.local/lib/python3.10/site-packages/sqlalchemy/util/compat.py", line 211, in raise_\n    raise exception\n  File "/home/airflow/.local/lib/python3.10/site-packages/sqlalchemy/engine/base.py", line 1910, in _execute_context\n    self.dialect.do_execute(\n  File "/home/airflow/.local/lib/python3.10/site-packages/sqlalchemy/engine/default.py", line 736, in do_execute\n    cursor.execute(statement, parameters)\n  File "/home/airflow/.local/lib/python3.10/site-packages/pymysql/cursors.py", line 153, in execute\n    result = self._query(query)\n  File "/home/airflow/.local/lib/python3.10/site-packages/pymysql/cursors.py", line 322, in _query\n    conn.query(q)\n  File "/home/airflow/.local/lib/python3.10/site-packages/pymysql/connections.py", line 558, in query\n    self._affected_rows = self._read_query_result(unbuffered=unbuffered)\n  File "/home/airflow/.local/lib/python3.10/site-packages/pymysql/connections.py", line 822, in _read_query_result\n    result.read()\n  File "/home/airflow/.local/lib/python3.10/site-packages/pymysql/connections.py", line 1200, in read\n    first_packet = self.connection._read_packet()\n  File "/home/airflow/.local/lib/python3.10/site-packages/pymysql/connections.py", line 772, in _read_packet\n    packet.raise_for_error()\n  File "/home/airflow/.local/lib/python3.10/site-packages/pymysql/protocol.py", line 221, in raise_for_error\n    err.raise_mysql_exception(self._data)\n  File "/home/airflow/.local/lib/python3.10/site-packages/pymysql/err.py", line 143, in raise_mysql_exception\n    raise errorclass(errno, errval)\nsqlalchemy.exc.OperationalError: (pymysql.err.OperationalError) (1227, \'Access denied; you need (at least one of) the PROCESS privilege(s) for this operation\')\n[SQL: /* {"app": "OpenMetadata", "version": "1.3.0.0.dev0"} */\nSHOW CREATE TABLE `information_schema`.`FILES`]\n(Background on this error at: https://sqlalche.me/e/14/e3q8)\n',
+};
+
 const executionRuns = [
   {
     runId: 'c95cc97b-9ea2-465c-9b5a-255401674324',
-    pipelineState: 'success',
+    pipelineState: 'partialSuccess',
     startDate: 1667304123,
     timestamp: 1667304123,
     endDate: 1667304126,
+    status: [
+      {
+        name: 'Source',
+        records: 155,
+        updated_records: 0,
+        warnings: 0,
+        errors: 1,
+        filtered: 0,
+        failures: [failure],
+      },
+      {
+        name: 'Sink',
+        records: 155,
+        updated_records: 0,
+        warnings: 0,
+        errors: 0,
+        filtered: 0,
+        failures: [],
+      },
+    ],
   },
   {
     runId: '60b3e15c-3865-4c81-a1ee-36ff85d2be8e',
@@ -41,6 +76,13 @@ const executionRuns = [
   },
 ];
 
+jest.mock(
+  '../../common/TestConnection/ConnectionStepCard/ConnectionStepCard',
+  () => {
+    return jest.fn().mockImplementation(() => <p>testConnectionStepCard</p>);
+  }
+);
+
 jest.mock('../../../rest/ingestionPipelineAPI', () => ({
   getRunHistoryForPipeline: jest.fn().mockImplementation(() =>
     Promise.resolve({
@@ -188,4 +230,82 @@ describe('Test IngestionRecentRun component', () => {
     expect(successRun).toBeInTheDocument();
     expect(runs).toHaveLength(4);
   });
+
+  it('should show additional details for click on run', async () => {
+    (getRunHistoryForPipeline as jest.Mock).mockResolvedValueOnce({
+      data: [...executionRuns],
+      paging: { total: 4 },
+    });
+
+    await act(async () => {
+      render(<IngestionRecentRuns ingestion={mockIngestion} />);
+    });
+
+    const runs = await screen.findAllByTestId('pipeline-status');
+    const partialSuccess = await screen.findByText(/Partial Success/);
+
+    expect(partialSuccess).toBeInTheDocument();
+    expect(runs).toHaveLength(3);
+
+    act(() => {
+      fireEvent.click(partialSuccess);
+    });
+
+    expect(await findByRole(document.body, 'dialog')).toBeInTheDocument();
+
+    expect(await screen.findByText(/Source/)).toBeInTheDocument();
+
+    expect(await screen.findByText(/Sink/)).toBeInTheDocument();
+    expect(await screen.findAllByText(/label.log-plural/)).toHaveLength(1);
+  });
+
+  it('should show stacktrace when click on logs', async () => {
+    (getRunHistoryForPipeline as jest.Mock).mockResolvedValueOnce({
+      data: [...executionRuns],
+      paging: { total: 4 },
+    });
+
+    await act(async () => {
+      render(<IngestionRecentRuns ingestion={mockIngestion} />);
+    });
+
+    const runs = await screen.findAllByTestId('pipeline-status');
+    const partialSuccess = await screen.findByText(/Partial Success/);
+
+    expect(partialSuccess).toBeInTheDocument();
+    expect(runs).toHaveLength(3);
+
+    act(() => {
+      fireEvent.click(partialSuccess);
+    });
+
+    expect(await findByRole(document.body, 'dialog')).toBeInTheDocument();
+
+    await act(async () => {
+      fireEvent.click(await screen.findByText(/label.log-plural/));
+    });
+
+    expect(ConnectionStepCard).toHaveBeenNthCalledWith(
+      2,
+      {
+        isTestingConnection: false,
+        testConnectionStep: {
+          description: failure.error,
+          mandatory: false,
+          name: 'FILES',
+        },
+        testConnectionStepResult: {
+          errorLog: failure.stackTrace,
+          mandatory: false,
+          message: failure.error,
+          name: 'FILES',
+          passed: false,
+        },
+      },
+      {}
+    );
+    expect(
+      await screen.findByText(/testConnectionStepCard/)
+    ).toBeInTheDocument();
+  });
 });

openmetadata-ui/src/main/resources/ui/src/components/Ingestion/IngestionRecentRun/IngestionRecentRuns.component.tsx

@@ -11,26 +11,34 @@
  *  limitations under the License.
  */
 
-import { Popover, Skeleton, Space, Tag } from 'antd';
+import { Button, Popover, Skeleton, Space, Tag } from 'antd';
+import Modal from 'antd/lib/modal/Modal';
+import { ColumnType } from 'antd/lib/table';
+import { ExpandableConfig } from 'antd/lib/table/interface';
 import { isEmpty, startCase } from 'lodash';
 import React, {
   FunctionComponent,
   useCallback,
   useEffect,
+  useMemo,
   useState,
 } from 'react';
 import { useTranslation } from 'react-i18next';
+import { NO_DATA } from '../../../constants/constants';
 import { PIPELINE_INGESTION_RUN_STATUS } from '../../../constants/pipeline.constants';
 import {
   IngestionPipeline,
   PipelineStatus,
+  StepSummary,
 } from '../../../generated/entity/services/ingestionPipelines/ingestionPipeline';
 import { getRunHistoryForPipeline } from '../../../rest/ingestionPipelineAPI';
 import {
   formatDateTime,
   getCurrentMillis,
   getEpochMillisForPastDays,
 } from '../../../utils/date-time/DateTimeUtils';
+import Table from '../../common/Table/Table';
+import ConnectionStepCard from '../../common/TestConnection/ConnectionStepCard/ConnectionStepCard';
 import './ingestion-recent-run.style.less';
 
 interface Props {
@@ -49,6 +57,80 @@ export const IngestionRecentRuns: FunctionComponent<Props> = ({
   const { t } = useTranslation();
   const [recentRunStatus, setRecentRunStatus] = useState<PipelineStatus[]>([]);
   const [loading, setLoading] = useState(true);
+  const [selectedStatus, setSelectedStatus] = useState<PipelineStatus>();
+  const [expandedKeys, setExpandedKeys] = useState<string[]>([]);
+  const columns: ColumnType<StepSummary>[] = useMemo(
+    () => [
+      {
+        title: t('label.step'),
+        dataIndex: 'name',
+      },
+      {
+        title: t('label.record-plural'),
+        dataIndex: 'records',
+      },
+      {
+        title: t('label.filtered'),
+        dataIndex: 'filtered',
+      },
+      {
+        title: t('label.warning-plural'),
+        dataIndex: 'warnings',
+      },
+      {
+        title: t('label.error-plural'),
+        dataIndex: 'errors',
+      },
+
+      {
+        title: t('label.failure-plural'),
+        dataIndex: 'failures',
+        render: (failures: StepSummary['failures'], record: StepSummary) =>
+          (failures?.length ?? 0) > 0 ? (
+            <Button
+              size="small"
+              type="link"
+              onClick={() => setExpandedKeys([record.name])}>
+              {t('label.log-plural')}
+            </Button>
+          ) : (
+            NO_DATA
+          ),
+      },
+    ],
+    [setExpandedKeys]
+  );
+  const expandable: ExpandableConfig<StepSummary> = useMemo(
+    () => ({
+      expandedRowRender: (record) => {
+        return (
+          record.failures?.map((failure) => (
+            <ConnectionStepCard
+              isTestingConnection={false}
+              key={failure.name}
+              testConnectionStep={{
+                name: failure.name,
+                mandatory: false,
+                description: failure.error,
+              }}
+              testConnectionStepResult={{
+                name: failure.name,
+                passed: false,
+                mandatory: false,
+                message: failure.error,
+                errorLog: failure.stackTrace,
+              }}
+            />
+          )) ?? []
+        );
+      },
+      indentSize: 0,
+      expandIcon: () => null,
+      expandedRowKeys: expandedKeys,
+      rowExpandable: (record) => (record.failures?.length ?? 0) > 0,
+    }),
+    [expandedKeys]
+  );
 
   const fetchPipelineStatus = useCallback(async () => {
     setLoading(true);
@@ -87,17 +169,18 @@ export const IngestionRecentRuns: FunctionComponent<Props> = ({
           const status =
             i === recentRunStatus.length - 1 ? (
               <Tag
-                className="ingestion-run-badge latest"
+                className="ingestion-run-badge latest cursor-pointer"
                 color={
                   PIPELINE_INGESTION_RUN_STATUS[r?.pipelineState ?? 'success']
                 }
                 data-testid="pipeline-status"
-                key={i}>
+                key={i}
+                onClick={() => setSelectedStatus(r)}>
                 {startCase(r?.pipelineState)}
               </Tag>
             ) : (
               <Tag
-                className="ingestion-run-badge"
+                className="ingestion-run-badge cursor-pointer"
                 color={
                   PIPELINE_INGESTION_RUN_STATUS[r?.pipelineState ?? 'success']
                 }
@@ -139,6 +222,29 @@ export const IngestionRecentRuns: FunctionComponent<Props> = ({
           );
         }) ?? '--'
       )}
+
+      <Modal
+        centered
+        closeIcon={<></>}
+        maskClosable={false}
+        okButtonProps={{ style: { display: 'none' } }}
+        open={Boolean(selectedStatus)}
+        title={`Run status: ${startCase(
+          selectedStatus?.pipelineState
+        )} at ${formatDateTime(selectedStatus?.timestamp)}`}
+        width="80%"
+        onCancel={() => setSelectedStatus(undefined)}>
+        <Table
+          bordered
+          columns={columns}
+          dataSource={selectedStatus?.status ?? []}
+          expandable={expandable}
+          indentSize={0}
+          pagination={false}
+          rowKey="name"
+          size="small"
+        />
+      </Modal>
     </Space>
   );
 };

openmetadata-ui/src/main/resources/ui/src/components/common/TestConnection/ConnectionStepCard/ConnectionStepCard.tsx

@@ -16,6 +16,7 @@ import classNames from 'classnames';
 import { isUndefined } from 'lodash';
 import React from 'react';
 import { useTranslation } from 'react-i18next';
+import { LazyLog } from 'react-lazylog';
 import { ReactComponent as AttentionIcon } from '../../../../assets/svg/attention.svg';
 import { ReactComponent as FailIcon } from '../../../../assets/svg/fail-badge.svg';
 import { ReactComponent as SuccessIcon } from '../../../../assets/svg/success-badge.svg';
@@ -47,6 +48,10 @@ const ConnectionStepCard = ({
   const isNonMandatoryStepsFailing =
     failed && !testConnectionStepResult?.mandatory;
 
+  const logs =
+    testConnectionStepResult?.errorLog ??
+    t('label.no-entity', { entity: t('label.log-plural') });
+
   return (
     <div
       className={classNames('connection-step-card', {
@@ -127,12 +132,17 @@ const ConnectionStepCard = ({
               <Collapse ghost>
                 <Panel
                   className="connection-step-card-content-logs"
-                  header="Show logs"
+                  data-testid="lazy-log"
+                  header={t('label.show-log-plural')}
                   key="show-log">
-                  <p className="text-grey-muted">
-                    {testConnectionStepResult?.errorLog ||
-                      t('label.no-entity', { entity: t('label.log-plural') })}
-                  </p>
+                  <LazyLog
+                    caseInsensitive
+                    enableSearch
+                    selectableLines
+                    extraLines={1} // 1 is to be add so that linux users can see last line of the log
+                    height={300}
+                    text={logs}
+                  />
                 </Panel>
               </Collapse>
             </>