Merge pull request #855 from mandy-chessell/dev-dojo

Add context events and lineage relationships
odpi · Nov 12, 2023 · 910dfd9 · 910dfd9
2 parents 0c8193b + 2def7df
commit 910dfd9
Show file tree

Hide file tree

Showing 32 changed files with 5,737 additions and 358 deletions.
diff --git a/site/docs/concepts/index.md b/site/docs/concepts/index.md
@@ -292,7 +292,8 @@
 - [Peer Network](/concepts/peer-network)
 - [Person Role](/concepts/person-role)
 - [Personal Message](/concepts/personal-message)
-- [Personal Note](/concepts/personal-note)
+- [Personal Note](/concepts/personal-notes)
+- [Personal Profile](/concepts/personal-profile)
 - [Personal Roles](/concepts/personal-roles)
 - [Platform Chassis](/services/platform-chassis)
 - [Platform Metadata Security Connector](/concepts/platform-metadata-security-connector)

diff --git a/site/docs/connectors/connector-catalog.drawio b/site/docs/connectors/connector-catalog.drawio
@@ -1,6 +1,6 @@
-<mxfile host="Electron" modified="2023-04-21T12:55:48.365Z" agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) draw.io/21.1.2 Chrome/106.0.5249.199 Electron/21.4.3 Safari/537.36" etag="ucMBowalRGI1Ylc7chPC" version="21.1.2" type="device" pages="25">
+<mxfile host="Electron" modified="2023-10-29T12:08:33.938Z" agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) draw.io/21.6.8 Chrome/114.0.5735.289 Electron/25.5.0 Safari/537.36" etag="ADUVZf2RFDpK96rKyDfJ" version="21.6.8" type="device" pages="25">
   <diagram id="zwH0E6yuSuxNEa-D1nxI" name="compare-use-of-connectors">
-    <mxGraphModel dx="1142" dy="775" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="1169" pageHeight="827" math="0" shadow="0">
+    <mxGraphModel dx="1194" dy="775" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="1169" pageHeight="827" math="0" shadow="0">
       <root>
         <mxCell id="YbopbjHzWyyc6AShLxuW-0" />
         <mxCell id="YbopbjHzWyyc6AShLxuW-1" parent="YbopbjHzWyyc6AShLxuW-0" />
@@ -3381,76 +3381,82 @@
     </mxGraphModel>
   </diagram>
   <diagram id="8t2bc5HyNLUx5RqxSolL" name="catalog-integration-connectors">
-    <mxGraphModel dx="1142" dy="775" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="827" pageHeight="1169" math="0" shadow="0">
+    <mxGraphModel dx="1194" dy="775" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="827" pageHeight="1169" math="0" shadow="0">
       <root>
         <mxCell id="0" />
         <mxCell id="1" parent="0" />
-        <mxCell id="eKeLNaiYI03M0CuCHnDJ-1" value="Integration Daemon" style="rounded=0;whiteSpace=wrap;html=1;verticalAlign=top;fillColor=#d5e8d4;strokeColor=#82b366;" vertex="1" parent="1">
+        <mxCell id="eKeLNaiYI03M0CuCHnDJ-1" value="Integration Daemon" style="rounded=0;whiteSpace=wrap;html=1;verticalAlign=top;fillColor=#d5e8d4;strokeColor=#82b366;" parent="1" vertex="1">
           <mxGeometry x="300.5" y="450" width="247.5" height="287.5" as="geometry" />
         </mxCell>
-        <mxCell id="eKeLNaiYI03M0CuCHnDJ-2" value="Catalog Integrator OMIS" style="rounded=0;whiteSpace=wrap;html=1;verticalAlign=top;" vertex="1" parent="1">
+        <mxCell id="eKeLNaiYI03M0CuCHnDJ-2" value="Catalog Integrator OMIS" style="rounded=0;whiteSpace=wrap;html=1;verticalAlign=top;" parent="1" vertex="1">
           <mxGeometry x="316.5" y="510" width="213.5" height="205" as="geometry" />
         </mxCell>
-        <mxCell id="eKeLNaiYI03M0CuCHnDJ-3" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0;exitY=0.5;exitDx=0;exitDy=0;entryX=1;entryY=0.5;entryDx=0;entryDy=0;startArrow=none;startFill=0;" edge="1" parent="1" source="eKeLNaiYI03M0CuCHnDJ-4" target="eKeLNaiYI03M0CuCHnDJ-8">
+        <mxCell id="eKeLNaiYI03M0CuCHnDJ-3" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0;exitY=0.75;exitDx=0;exitDy=0;entryX=1;entryY=0.75;entryDx=0;entryDy=0;startArrow=none;startFill=0;" parent="1" source="eKeLNaiYI03M0CuCHnDJ-4" target="eKeLNaiYI03M0CuCHnDJ-8" edge="1">
           <mxGeometry relative="1" as="geometry" />
         </mxCell>
-        <mxCell id="eKeLNaiYI03M0CuCHnDJ-4" value="Apache Atlas&lt;br&gt;&amp;nbsp;Integration Connector" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#e1d5e7;strokeColor=#9673a6;" vertex="1" parent="1">
+        <mxCell id="eKeLNaiYI03M0CuCHnDJ-4" value="Apache Atlas&lt;br&gt;&amp;nbsp;Integration Connector" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#e1d5e7;strokeColor=#9673a6;" parent="1" vertex="1">
           <mxGeometry x="347" y="573" width="150" height="60" as="geometry" />
         </mxCell>
-        <mxCell id="eKeLNaiYI03M0CuCHnDJ-8" value="Apache Atlas" style="rounded=0;whiteSpace=wrap;html=1;shadow=0;verticalAlign=middle;fillColor=#60a917;strokeColor=#2D7600;fontColor=#ffffff;" vertex="1" parent="1">
+        <mxCell id="3qc-pWaL1qywKjwzyJdO-1" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=1;exitY=0.25;exitDx=0;exitDy=0;entryX=0;entryY=0.25;entryDx=0;entryDy=0;" edge="1" parent="1" source="eKeLNaiYI03M0CuCHnDJ-8" target="eKeLNaiYI03M0CuCHnDJ-4">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+        <mxCell id="eKeLNaiYI03M0CuCHnDJ-8" value="Apache Atlas" style="rounded=0;whiteSpace=wrap;html=1;shadow=0;verticalAlign=middle;fillColor=#60a917;strokeColor=#2D7600;fontColor=#ffffff;" parent="1" vertex="1">
           <mxGeometry x="40" y="573" width="120" height="60" as="geometry" />
         </mxCell>
-        <mxCell id="eKeLNaiYI03M0CuCHnDJ-9" value="Metadata Access Server" style="rounded=0;whiteSpace=wrap;html=1;verticalAlign=top;fillColor=#d5e8d4;strokeColor=#82b366;" vertex="1" parent="1">
+        <mxCell id="eKeLNaiYI03M0CuCHnDJ-9" value="Metadata Access Server" style="rounded=0;whiteSpace=wrap;html=1;verticalAlign=top;fillColor=#d5e8d4;strokeColor=#82b366;" parent="1" vertex="1">
           <mxGeometry x="700" y="510" width="250" height="170" as="geometry" />
         </mxCell>
-        <mxCell id="eKeLNaiYI03M0CuCHnDJ-10" value="Asset Manager OMAS" style="rounded=0;whiteSpace=wrap;html=1;verticalAlign=top;" vertex="1" parent="1">
+        <mxCell id="eKeLNaiYI03M0CuCHnDJ-10" value="Asset Manager OMAS" style="rounded=0;whiteSpace=wrap;html=1;verticalAlign=top;" parent="1" vertex="1">
           <mxGeometry x="722" y="570" width="206" height="70.5" as="geometry" />
         </mxCell>
-        <mxCell id="eKeLNaiYI03M0CuCHnDJ-11" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=1;exitY=0.25;exitDx=0;exitDy=0;startArrow=none;startFill=0;entryX=0;entryY=0.25;entryDx=0;entryDy=0;" edge="1" parent="1" source="eKeLNaiYI03M0CuCHnDJ-4" target="eKeLNaiYI03M0CuCHnDJ-10">
+        <mxCell id="eKeLNaiYI03M0CuCHnDJ-11" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=1;exitY=0.25;exitDx=0;exitDy=0;startArrow=classic;startFill=1;entryX=0;entryY=0.25;entryDx=0;entryDy=0;" parent="1" source="eKeLNaiYI03M0CuCHnDJ-4" target="eKeLNaiYI03M0CuCHnDJ-10" edge="1">
           <mxGeometry relative="1" as="geometry">
             <mxPoint x="497" y="625" as="sourcePoint" />
             <mxPoint x="710" y="570" as="targetPoint" />
           </mxGeometry>
         </mxCell>
-        <mxCell id="eKeLNaiYI03M0CuCHnDJ-12" value="Retrieve glossary content" style="text;html=1;align=center;verticalAlign=middle;resizable=0;points=[];autosize=1;" vertex="1" parent="1">
-          <mxGeometry x="543.5" y="560" width="160" height="30" as="geometry" />
+        <mxCell id="eKeLNaiYI03M0CuCHnDJ-12" value="Retrieve and update&lt;br&gt;open metadata" style="text;html=1;align=center;verticalAlign=middle;resizable=0;points=[];autosize=1;" parent="1" vertex="1">
+          <mxGeometry x="558" y="550" width="130" height="40" as="geometry" />
         </mxCell>
-        <mxCell id="eKeLNaiYI03M0CuCHnDJ-13" style="edgeStyle=orthogonalEdgeStyle;rounded=0;sketch=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0;exitY=0.75;exitDx=0;exitDy=0;entryX=0.75;entryY=0;entryDx=0;entryDy=0;shadow=0;startArrow=none;startFill=0;" edge="1" parent="1" source="eKeLNaiYI03M0CuCHnDJ-10" target="eKeLNaiYI03M0CuCHnDJ-17">
+        <mxCell id="eKeLNaiYI03M0CuCHnDJ-13" style="edgeStyle=orthogonalEdgeStyle;rounded=0;sketch=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0;exitY=0.75;exitDx=0;exitDy=0;entryX=0.75;entryY=0;entryDx=0;entryDy=0;shadow=0;startArrow=none;startFill=0;" parent="1" source="eKeLNaiYI03M0CuCHnDJ-10" target="eKeLNaiYI03M0CuCHnDJ-17" edge="1">
           <mxGeometry relative="1" as="geometry">
             <mxPoint x="689" y="500" as="sourcePoint" />
           </mxGeometry>
         </mxCell>
-        <mxCell id="eKeLNaiYI03M0CuCHnDJ-14" value="Asset Manager&lt;br&gt;OMAS OutTopic" style="text;html=1;align=center;verticalAlign=middle;resizable=0;points=[];autosize=1;strokeColor=none;fontSize=12;fontFamily=Helvetica;fontColor=#000000;" vertex="1" parent="1">
+        <mxCell id="eKeLNaiYI03M0CuCHnDJ-14" value="Asset Manager&lt;br&gt;OMAS OutTopic" style="text;html=1;align=center;verticalAlign=middle;resizable=0;points=[];autosize=1;strokeColor=none;fontSize=12;fontFamily=Helvetica;fontColor=#000000;" parent="1" vertex="1">
           <mxGeometry x="568" y="711" width="110" height="40" as="geometry" />
         </mxCell>
-        <mxCell id="eKeLNaiYI03M0CuCHnDJ-15" value="" style="rounded=0;whiteSpace=wrap;html=1;fontFamily=Helvetica;fontSize=12;align=center;fillColor=#d5e8d4;strokeColor=#82b366;" vertex="1" parent="1">
+        <mxCell id="eKeLNaiYI03M0CuCHnDJ-15" value="" style="rounded=0;whiteSpace=wrap;html=1;fontFamily=Helvetica;fontSize=12;align=center;fillColor=#d5e8d4;strokeColor=#82b366;" parent="1" vertex="1">
           <mxGeometry x="598" y="659" width="50" height="50" as="geometry" />
         </mxCell>
-        <mxCell id="eKeLNaiYI03M0CuCHnDJ-16" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.25;exitY=0;exitDx=0;exitDy=0;entryX=1;entryY=0.75;entryDx=0;entryDy=0;startArrow=none;startFill=0;" edge="1" parent="1" source="eKeLNaiYI03M0CuCHnDJ-17" target="eKeLNaiYI03M0CuCHnDJ-4">
+        <mxCell id="eKeLNaiYI03M0CuCHnDJ-16" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.25;exitY=0;exitDx=0;exitDy=0;entryX=1;entryY=0.75;entryDx=0;entryDy=0;startArrow=none;startFill=0;" parent="1" source="eKeLNaiYI03M0CuCHnDJ-17" target="eKeLNaiYI03M0CuCHnDJ-4" edge="1">
           <mxGeometry relative="1" as="geometry">
             <Array as="points">
               <mxPoint x="615" y="618" />
             </Array>
           </mxGeometry>
         </mxCell>
-        <mxCell id="eKeLNaiYI03M0CuCHnDJ-17" value="" style="rounded=0;whiteSpace=wrap;html=1;fontFamily=Helvetica;fontSize=12;fontColor=#000000;align=center;" vertex="1" parent="1">
+        <mxCell id="eKeLNaiYI03M0CuCHnDJ-17" value="" style="rounded=0;whiteSpace=wrap;html=1;fontFamily=Helvetica;fontSize=12;fontColor=#000000;align=center;" parent="1" vertex="1">
           <mxGeometry x="606" y="659" width="35" height="40" as="geometry" />
         </mxCell>
-        <mxCell id="eKeLNaiYI03M0CuCHnDJ-18" value="Update&lt;br&gt;glossary content" style="text;html=1;align=center;verticalAlign=middle;resizable=0;points=[];autosize=1;strokeColor=none;fillColor=none;" vertex="1" parent="1">
-          <mxGeometry x="179" y="555" width="110" height="40" as="geometry" />
+        <mxCell id="eKeLNaiYI03M0CuCHnDJ-18" value="Update&lt;br&gt;Apache Atlas metadata" style="text;html=1;align=center;verticalAlign=middle;resizable=0;points=[];autosize=1;strokeColor=none;fillColor=none;" parent="1" vertex="1">
+          <mxGeometry x="160" y="620.25" width="150" height="40" as="geometry" />
         </mxCell>
-        <mxCell id="eKeLNaiYI03M0CuCHnDJ-21" value="New glossary content" style="text;html=1;align=center;verticalAlign=middle;resizable=0;points=[];autosize=1;strokeColor=none;fillColor=none;" vertex="1" parent="1">
-          <mxGeometry x="558" y="590.25" width="140" height="30" as="geometry" />
+        <mxCell id="eKeLNaiYI03M0CuCHnDJ-21" value="Metadata changes" style="text;html=1;align=center;verticalAlign=middle;resizable=0;points=[];autosize=1;strokeColor=none;fillColor=none;" parent="1" vertex="1">
+          <mxGeometry x="568" y="590.25" width="120" height="30" as="geometry" />
         </mxCell>
-        <mxCell id="eKeLNaiYI03M0CuCHnDJ-23" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=1;exitDx=0;exitDy=0;" edge="1" parent="1" source="eKeLNaiYI03M0CuCHnDJ-22" target="eKeLNaiYI03M0CuCHnDJ-8">
+        <mxCell id="eKeLNaiYI03M0CuCHnDJ-23" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=1;exitDx=0;exitDy=0;" parent="1" source="eKeLNaiYI03M0CuCHnDJ-22" target="eKeLNaiYI03M0CuCHnDJ-8" edge="1">
           <mxGeometry relative="1" as="geometry" />
         </mxCell>
-        <mxCell id="eKeLNaiYI03M0CuCHnDJ-22" value="" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#60a917;fontColor=#ffffff;strokeColor=#2D7600;" vertex="1" parent="1">
+        <mxCell id="eKeLNaiYI03M0CuCHnDJ-22" value="" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#60a917;fontColor=#ffffff;strokeColor=#2D7600;" parent="1" vertex="1">
           <mxGeometry x="40" y="470" width="120" height="60" as="geometry" />
         </mxCell>
-        <mxCell id="eKeLNaiYI03M0CuCHnDJ-24" value="" style="rounded=1;whiteSpace=wrap;html=1;" vertex="1" parent="1">
+        <mxCell id="eKeLNaiYI03M0CuCHnDJ-24" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
           <mxGeometry x="55" y="475" width="90" height="50" as="geometry" />
         </mxCell>
+        <mxCell id="3qc-pWaL1qywKjwzyJdO-2" value="Read&lt;br&gt;Apache Atlas metadata" style="text;html=1;align=center;verticalAlign=middle;resizable=0;points=[];autosize=1;strokeColor=none;fillColor=none;" vertex="1" parent="1">
+          <mxGeometry x="160" y="550" width="150" height="40" as="geometry" />
+        </mxCell>
       </root>
     </mxGraphModel>
   </diagram>

diff --git a/site/docs/connectors/integration/apache-atlas-catalog-integration-connector.md b/site/docs/connectors/integration/apache-atlas-catalog-integration-connector.md
@@ -24,23 +24,28 @@ Reasons for integrating Apache Atlas into a wider open metadata ecosystem includ
 * To share glossaries and classification definitions that have been developed by an Apache Atlas user community in order to seed other projects and tools.
 * To share details of the data sets that Apache Atlas has catalogued with other catalogs and tools.  For example, the Apache Atlas catalog of data sets may be shared with data science or stewardship tools.  Tags, classifications and semantic assignments made to these assets by users of these tools could then be pushed back to Apache Atlas to enhance the use and governance of their associated data through Apache Atlas.
 * To extract lineage information for data flowing through the data platforms and processes that Apache Atlas is capturing.
+* To improve the usefulness of Apache Atlas metadata by augmenting it with additional insight from the open metadata ecosystem.
+* To improve the governance of Apache Atlas metadata by analysing and reporting on the content and usage of Apache Atlas metadata.
 
 ## Integration Connector function
 
-The Apache Atlas integration connector provides an ongoing exchange of metadata between Apache Atlas and the open metadata ecosystem.  In its initial state, it focuses on the two-way exchange of glossaries.  Future versions will add entities (such as data sets and process), lineage and classifications.
+The Apache Atlas integration connector provides an ongoing exchange of metadata between Apache Atlas and the open metadata ecosystem.
 
 ![Figure 1](apache-atlas-catalog-integration-connector.svg)
 > **Figure 1:** Operation of the Apache Atlas integration connector
 
-Specifically this connector:
+The type of metadata exchanged, and the direction of the exchange, is controlled by the configuration properties added to this connector's connection.  Specifically this connector can optionally:
 
-* Publishes glossaries, glossary categories and ACTIVE glossary terms that originate from the open metadata ecosystem into Apache Atlas.  The connector makes no attempt to detect changes to these glossary elements in Apache Atlas.  Additionally, such changes will be overridden the next time the integration connector refreshes the glossary metadata in Apache Atlas.
+* Publishes glossaries, glossary categories and ACTIVE glossary terms that originate from the open metadata ecosystem into Apache Atlas.  The connector makes no attempt to detect changes to these glossary elements in Apache Atlas.  Such changes will be overridden the next time the integration connector refreshes the glossary metadata in Apache Atlas.
 * Publishes glossaries, glossary categories and glossary terms that originate in Apache Atlas into the open metadata ecosystem.  These elements will be read-only in the open metadata ecosystem.
-
+* Catalog all Apache Atlas entities of type *DataSet* in the open metadata ecosystem.  
+* Catalog all Apache Atlas lineage relationships (*dataset_process_inputs* and *dataset_process_outputs*) in the open metadata ecosystem.  These relationships link the DataSet entities from Apache Atlas and contribute to the enterprise's view of lineage.
+* Maintain a reference data set of classifications used in Apache Atlas.  This reference data set can be used to create new classification definitions in Apache Atlas, catalog the Apache Atlas classificationDefs in the reference data set and record the classifications in use in Apache Atlas.
+* Add definitions of open metadata entities and classifications linked to an open metadata entity that originated in Apache Atlas to its counterpart in Apache Atlas.  This way, the Apache Atlas users can see the same information about their data sets and the native users of the open metadata ecosystem.  There is specialized behaviour in how informal tags from the open metadata ecosystem are handled in Apache Atlas.
 
 ### Metadata ownership 
 
-The concept of ownership of each metadata instance (such as glossary, glossary catalog and glossary term) is strictly maintained.  Simply:
+The concept of ownership of each metadata instance (such as glossary, glossary category, glossary term and data set) is strictly maintained.  Simply:
 
 * If the instance was originally created in the open metadata ecosystem, it is owned by the open metadata ecosystem and any changes or deletions made to the copy in Apache Atlas will be reversed the next time the connector refreshes the Apache Atlas metadata.
 * If the instance was originally created in Apache Atlas, it is owned by Apache Atlas.  When the instance is copied into the open metadata ecosystem, its [provenance](/features/metadata-provenance/overview) is set to indicate that this Apache Atlas server is its origin, which effectively makes it read-only in the open metadata ecosystem.
@@ -49,8 +54,6 @@ It is possible to create a new glossary term or glossary category in Apache Atla
 
 Similarly, it is possible to create new glossary terms and categories in the open metadata ecosystem that are linked to a glossary that originated from Apache Atlas.  These new entities will be owned by the open metadata ecosystem.
 
-
-
 ### Correlation
 
 In order to correlate the instances in Apache Atlas with the open metadata ecosystem instance, the connector: