feat: Pre-work needed for the lineage graph visualization (#1084)

* parent f6b21b3 author verdan <[email protected]> 1620122095 +0200 committer verdan <[email protected]> 1620776994 +0200 initial commit Fixes the lineage query, updates the common model more updates updates the look and feel removes commented code Signed-off-by: verdan <[email protected]> passes the paramters to backend adds the zooming feature code refactor, addresses review comments Signed-off-by: verdan <[email protected]> docs: update faq.md (#1038) Signed-off-by: mgorsk1 <[email protected]> Modified methods to throw NOTFOUNDEXCEPTION instead RuntimeEror in neo4j_proxy (metadata_service) (#1028) Signed-off-by: aparna-cimpress <[email protected]> perf: Update ModeDashboardLastModifiedTimestampExtractor to use Mode discovery api (#1040) * update ModeDashboardLastModifiedTimestampExtractor to use Mode discovery api Signed-off-by: Junda Yang <[email protected]> * add empty line at the end of the file Signed-off-by: Junda Yang <[email protected]> * update comment Signed-off-by: Junda Yang <[email protected]> * fix lint Signed-off-by: Junda Yang <[email protected]> * fix import Signed-off-by: Junda Yang <[email protected]> ci: fix metadata ci config (#1039) Signed-off-by: Dorian Johnson <[email protected]> perf: update ModeDashboardLastSuccessfulExecutionExtractor to discovery api (#1042) * update ModeDashboardLastSuccessfulExecutionExtractor to discovery api Signed-off-by: Junda Yang <[email protected]> * lint Signed-off-by: Junda Yang <[email protected]> * remove unused import Signed-off-by: Junda Yang <[email protected]> perf: Update ModeDashboardOwner extractor to use Mode discovery api (#1043) * update ModeDashboardOwnerExtractor to discovery api Signed-off-by: Junda Yang <[email protected]> * update import Signed-off-by: Junda Yang <[email protected]> update ModeDashboardQueryExtractor to discovery api (#1044) Signed-off-by: Junda Yang <[email protected]> chore: fix databuilder dep typo (#1046) Signed-off-by: feng-tao <[email protected]> build(deps): bump py from 1.5.3 to 1.10.0 in /databuilder (#1049) chore: Update ModeDashboardLastModifiedTimestampExtractor typing (#1048) update Mode dashboard extractor tests (#1047) fix: Temporarily fix potential 404 in mode dashboard executions extractor (#1051) * fix ModeDashboardExecutionsExtractor Signed-off-by: Junda Yang <[email protected]> * nit Signed-off-by: Junda Yang <[email protected]> test: refactor: update test and directory of ModeDashboardChartsBatchExtractor (#1052) Temporarily fix potential 404 in ModeDashboardUsageExtractor (#1053) feat: Add dbt extractor for table lineage + metadata (#1015) * converted dbt extractor to mono repo Signed-off-by: Grant Seward <[email protected]> * Flake8 formatting Signed-off-by: Grant Seward <[email protected]> * Updates per feedback from Convoy Signed-off-by: Grant Seward <[email protected]> * Commented sample logging Signed-off-by: Grant Seward <[email protected]> * Added dbt to main amundsen README Signed-off-by: Grant Seward <[email protected]> fix bug in rest_api_query (#1056) Signed-off-by: Junda Yang <[email protected]> feat: Add merge query results functionality to rest_api_query (#1058) * Add merge query results functionality Signed-off-by: Junda Yang <[email protected]> * fix mypy Signed-off-by: Junda Yang <[email protected]> * nit Signed-off-by: Junda Yang <[email protected]> chore: fix gh page build workflow (#1062) Signed-off-by: feng-tao <[email protected]> chore: add mailmap to associate the right user (#1061) Signed-off-by: feng-tao <[email protected]> fix: Update ModeDashboardExtractor to Mode discovery api (#1063) * Update ModeDashboardExtractor to Mode discovery api Signed-off-by: Junda Yang <[email protected]> * nit Signed-off-by: Junda Yang <[email protected]> chore: delete legacy ModeDashboardChartsExtractor (#1064) * remove-legacy-charts-extractor Signed-off-by: Junda Yang <[email protected]> * nit Signed-off-by: Junda Yang <[email protected]> bump up databuilder version (#1065) Signed-off-by: Junda Yang <[email protected]> fix in another PR Signed-off-by: verdan <[email protected]> makes Verdan and Dorian the code owners (#1081) Signed-off-by: verdan <[email protected]> Adds a new "parent" field in lineage model (#1080) * adds 'parent' field to help build the graph Signed-off-by: verdan <[email protected]> * make the default to None Signed-off-by: verdan <[email protected]> deploy_release: on release, publish to pypi/docker (#1079) Given a release with a title of `<module_name> <version_number>`, this will push that release to PyPi and docker. Note that this does not attempt to create the changelog or any of that stuff. That will need to be done by a separate action. This creates a nice modularity boundary, in that this action doesn't care where the release came from (manual or another automated step) Signed-off-by: Dorian Johnson <[email protected]> Fixes the test cases Signed-off-by: verdan <[email protected]> Removes extra functionality Signed-off-by: verdan <[email protected]> removes redundant flag Signed-off-by: verdan <[email protected]> * FIxes the betterer issues Signed-off-by: verdan <[email protected]> * code review changes Signed-off-by: verdan <[email protected]> * updates the common version Signed-off-by: verdan <[email protected]> * Fixes lint errors for metadata Signed-off-by: verdan <[email protected]>
amundsen-io · May 17, 2021 · 32499e6 · 32499e6
1 parent c441ed7
commit 32499e6
Show file tree

Hide file tree

Showing 33 changed files with 912 additions and 535 deletions.
diff --git a/frontend/amundsen_application/api/metadata/v0.py b/frontend/amundsen_application/api/metadata/v0.py
@@ -800,7 +800,9 @@ def get_table_lineage() -> Response:
     try:
         table_endpoint = _get_table_endpoint()
         table_key = get_query_param(request.args, 'key')
-        url = f'{table_endpoint}/{table_key}/lineage'
+        depth = get_query_param(request.args, 'depth')
+        direction = get_query_param(request.args, 'direction')
+        url = f'{table_endpoint}/{table_key}/lineage?depth={depth}&direction={direction}'
         response = request_metadata(url=url, method=request.method)
         json = response.json()
         downstream = [marshall_lineage_table(table) for table in json.get('downstream_entities')]

diff --git a/frontend/amundsen_application/static/.betterer.results b/frontend/amundsen_application/static/.betterer.results
@@ -228,8 +228,8 @@ exports[`eslint`] = {
       [146, 18, 5, "\'props\' is assigned a value but never used.", "187023499"],
       [146, 25, 7, "\'wrapper\' is already declared in the upper scope.", "990908086"]
     ],
-    "js/components/ResourceListItem/DashboardListItem/index.tsx:2297434190": [
-      [16, 0, 41, "\`../types\` import should occur before import of \`./constants\`", "2620565544"]
+    "js/components/ResourceListItem/DashboardListItem/index.tsx:2194588229": [
+      [17, 0, 41, "\`../types\` import should occur before import of \`./constants\`", "2620565544"]
     ],
     "js/components/ResourceListItem/SchemaInfo/tests/index.spec.tsx:3506535360": [
       [34, 6, 25, "Use object destructuring.", "354229464"],
@@ -505,12 +505,12 @@ exports[`eslint`] = {
     "js/components/Tags/index.tsx:3468508233": [
       [38, 4, 21, "Must use destructuring props assignment", "4236634811"]
     ],
-    "js/config/config-default.ts:3074333736": [
+    "js/config/config-default.ts:1497564975": [
       [2, 0, 72, "\`../interfaces\` import should occur before import of \`./config-types\`", "1449508543"],
-      [234, 6, 21, "\'partitionKey\' is defined but never used.", "399589312"],
-      [235, 6, 23, "\'partitionValue\' is defined but never used.", "793372348"]
+      [236, 6, 21, "\'partitionKey\' is defined but never used.", "399589312"],
+      [237, 6, 23, "\'partitionValue\' is defined but never used.", "793372348"]
     ],
-    "js/config/config-utils.ts:2989233210": [
+    "js/config/config-utils.ts:2658622130": [
       [11, 0, 45, "\`../interfaces\` import should occur before import of \`./config-types\`", "3885176344"]
     ],
     "js/ducks/announcements/api/v0.ts:908063915": [
@@ -563,6 +563,10 @@ exports[`eslint`] = {
     "js/ducks/lastIndexed/sagas.ts:1498244597": [
       [7, 2, 29, "\'action\' is defined but never used.", "566797395"]
     ],
+    "js/ducks/lineage/api/v0.ts:63450243": [
+      [19, 13, 26, "Expected the Promise rejection reason to be an Error.", "875716520"],
+      [44, 13, 26, "Expected the Promise rejection reason to be an Error.", "875716520"]
+    ],
     "js/ducks/middlewares/analyticsMiddleware.ts:3943673455": [
       [11, 7, 8, "\'getState\' is defined but never used.", "1919118020"]
     ],
@@ -588,19 +592,17 @@ exports[`eslint`] = {
       [19, 2, 8, "Assignment to function parameter \'resource\'.", "2131237679"],
       [20, 2, 248, "Expected a default case.", "1034339850"]
     ],
-    "js/ducks/tableMetadata/api/v0.ts:1918235014": [
+    "js/ducks/tableMetadata/api/v0.ts:1722284731": [
       [75, 8, 23, "Use object destructuring.", "1142306891"],
       [78, 13, 39, "Expected the Promise rejection reason to be an Error.", "3871091426"],
       [125, 23, -4008, "Expected to return a value at the end of arrow function.", "5381"],
-      [181, 13, 32, "Expected the Promise rejection reason to be an Error.", "2773728948"],
-      [198, 13, 26, "Expected the Promise rejection reason to be an Error.", "875716520"],
-      [218, 13, 26, "Expected the Promise rejection reason to be an Error.", "875716520"]
+      [181, 13, 32, "Expected the Promise rejection reason to be an Error.", "2773728948"]
     ],
-    "js/ducks/tableMetadata/index.spec.ts:2722355878": [
-      [567, 22, 11, "\'mockSuccess\' is already declared in the upper scope.", "1120045516"],
-      [593, 22, 11, "\'mockFailure\' is already declared in the upper scope.", "692282639"],
-      [699, 22, 11, "\'mockSuccess\' is already declared in the upper scope.", "1120045516"],
-      [731, 22, 11, "\'mockFailure\' is already declared in the upper scope.", "692282639"]
+    "js/ducks/tableMetadata/index.spec.ts:2466548220": [
+      [480, 22, 11, "\'mockSuccess\' is already declared in the upper scope.", "1120045516"],
+      [506, 22, 11, "\'mockFailure\' is already declared in the upper scope.", "692282639"],
+      [612, 22, 11, "\'mockSuccess\' is already declared in the upper scope.", "1120045516"],
+      [644, 22, 11, "\'mockFailure\' is already declared in the upper scope.", "692282639"]
     ],
     "js/ducks/tableMetadata/owners/index.spec.ts:655040122": [
       [15, 0, 91, "\`../reducer\` import should occur before import of \`./reducer\`", "2216296793"],
@@ -1034,11 +1036,11 @@ exports[`eslint`] = {
     "js/pages/TableDetailPage/index.spec.tsx:1320221888": [
       [64, 6, 25, "Use object destructuring.", "1230260048"]
     ],
-    "js/pages/TableDetailPage/index.tsx:1010332848": [
-      [136, 2, 20, "key should be placed after componentDidUpdate", "3916788587"],
-      [146, 22, 12, "\'getTableData\' is already declared in the upper scope.", "3938384029"],
-      [159, 22, 12, "\'getTableData\' is already declared in the upper scope.", "3938384029"],
-      [239, 6, 28, "\'openRequestDescriptionDialog\' is already declared in the upper scope.", "4102713454"]
+    "js/pages/TableDetailPage/index.tsx:2009672874": [
+      [135, 2, 20, "key should be placed after componentDidUpdate", "3916788587"],
+      [145, 22, 12, "\'getTableData\' is already declared in the upper scope.", "3938384029"],
+      [158, 22, 12, "\'getTableData\' is already declared in the upper scope.", "3938384029"],
+      [238, 6, 28, "\'openRequestDescriptionDialog\' is already declared in the upper scope.", "4102713454"]
     ],
     "js/utils/textUtils.ts:2545492889": [
       [19, 6, 46, "Unexpected lexical declaration in case block.", "156477898"]

diff --git a/frontend/amundsen_application/static/js/config/config-default.ts b/frontend/amundsen_application/static/js/config/config-default.ts
@@ -205,20 +205,22 @@ const configDefault: AppConfig = {
     },
   },
   tableLineage: {
+    inAppListEnabled: false,
+    inAppPageEnabled: false,
+    externalEnabled: false,
     iconPath: 'PATH_TO_ICON',
     isBeta: false,
-    isEnabled: false,
     urlGenerator: (
       database: string,
       cluster: string,
       schema: string,
       table: string
     ) =>
       `https://DEFAULT_LINEAGE_URL?schema=${schema}&cluster=${cluster}&db=${database}&table=${table}`,
-    inAppListEnabled: false,
   },
   columnLineage: {
     inAppListEnabled: false,
+    inAppPageEnabled: false,
     urlGenerator: (
       database: string,
       cluster: string,

diff --git a/frontend/amundsen_application/static/js/config/config-types.ts b/frontend/amundsen_application/static/js/config/config-types.ts
@@ -281,14 +281,15 @@ interface TableProfileConfig {
 interface TableLineageConfig {
   iconPath: string;
   isBeta: boolean;
-  isEnabled: boolean;
   urlGenerator: (
     database: string,
     cluster: string,
     schema: string,
     table: string
   ) => string;
+  externalEnabled: boolean;
   inAppListEnabled: boolean;
+  inAppPageEnabled: boolean;
 }
 
 /**
@@ -298,6 +299,7 @@ interface TableLineageConfig {
  */
 interface ColumnLineageConfig {
   inAppListEnabled: boolean;
+  inAppPageEnabled: boolean;
   urlGenerator: (
     database: string,
     cluster: string,

diff --git a/frontend/amundsen_application/static/js/config/config-utils.ts b/frontend/amundsen_application/static/js/config/config-utils.ts
@@ -331,6 +331,20 @@ export function isColumnListLineageEnabled() {
   return AppConfig.columnLineage.inAppListEnabled;
 }
 
+/**
+ * Returns whether the in-app table lineage page is enabled.
+ */
+export function isTableLineagePageEnabled() {
+  return AppConfig.tableLineage.inAppPageEnabled;
+}
+
+/**
+ * Returns whether the in-app column lineage page is enabled.
+ */
+export function isColumnLineagePageEnabled() {
+  return AppConfig.columnLineage.inAppPageEnabled;
+}
+
 /**
  * Returns the lineage link for a given column
  */

diff --git a/frontend/amundsen_application/static/js/config/index.spec.ts b/frontend/amundsen_application/static/js/config/index.spec.ts
@@ -423,6 +423,22 @@ describe('isColumnListLineageEnabled', () => {
   });
 });
 
+describe('isTableLineagePageEnabled', () => {
+  it('returns isTableLineagePageEnabled defined in config', () => {
+    const actual = ConfigUtils.isTableLineagePageEnabled();
+    const expected = AppConfig.tableLineage.inAppPageEnabled;
+    expect(actual).toBe(expected);
+  });
+});
+
+describe('isColumnLineagePageEnabled', () => {
+  it('returns isColumnLineagePageEnabled defined in config', () => {
+    const actual = ConfigUtils.isColumnLineagePageEnabled();
+    const expected = AppConfig.columnLineage.inAppPageEnabled;
+    expect(actual).toBe(expected);
+  });
+});
+
 describe('getColumnLineageLink', () => {
   it('calls the column lineage link with the right params', () => {
     const tableData = {

diff --git a/frontend/amundsen_application/static/js/ducks/lineage/api/api.spec.ts b/frontend/amundsen_application/static/js/ducks/lineage/api/api.spec.ts
@@ -0,0 +1,58 @@
+import axios from 'axios';
+
+import { tableLineage } from 'fixtures/metadata/table';
+
+import * as API from './v0';
+
+jest.mock('axios');
+
+describe('getLineage', () => {
+  let axiosMockGet;
+  it('resolves with object containing table lineage and status code', async () => {
+    const mockStatus = 200;
+    const mockResponse = {
+      data: tableLineage,
+      status: mockStatus,
+    };
+    axiosMockGet = jest
+      .spyOn(axios, 'get')
+      .mockImplementationOnce(() => Promise.resolve(mockResponse));
+    expect.assertions(2);
+    await API.getTableLineage(
+      'database://cluster.schema/table_name',
+      1,
+      'both'
+    ).then((processedResponse) => {
+      expect(processedResponse).toEqual({
+        data: tableLineage,
+        statusCode: mockStatus,
+      });
+    });
+    expect(axiosMockGet).toHaveBeenCalled();
+  });
+
+  it('catches error and resolves with object containing error information', async () => {
+    const mockStatus = 500;
+    const mockMessage = 'oops';
+    const mockResponse = {
+      response: {
+        data: {
+          msg: mockMessage,
+        },
+        status: mockStatus,
+      },
+    };
+    axiosMockGet = jest
+      .spyOn(axios, 'get')
+      .mockImplementationOnce(() => Promise.reject(mockResponse));
+    expect.assertions(2);
+    await API.getTableLineage('testUri', 1, 'both').catch(
+      (processedResponse) => {
+        expect(processedResponse).toEqual({
+          status: mockStatus,
+        });
+      }
+    );
+    expect(axiosMockGet).toHaveBeenCalled();
+  });
+});
diff --git a/frontend/amundsen_application/static/js/ducks/lineage/api/v0.ts b/frontend/amundsen_application/static/js/ducks/lineage/api/v0.ts
@@ -0,0 +1,47 @@
+import axios, { AxiosError, AxiosResponse } from 'axios';
+import { Lineage } from 'interfaces/Lineage';
+import { getQueryParams } from 'ducks/utilMethods';
+
+export const API_PATH = '/api/metadata/v0';
+
+export type LineageAPI = { lineage: Lineage };
+
+export function getTableLineage(key: string, depth: number, direction: string) {
+  const tableQueryParams = getQueryParams({ key, depth, direction });
+  return axios
+    .get(`${API_PATH}/get_table_lineage?${tableQueryParams}`)
+    .then((response: AxiosResponse<LineageAPI>) => ({
+      data: response.data,
+      statusCode: response.status,
+    }))
+    .catch((e: AxiosError<LineageAPI>) => {
+      const { response } = e;
+      const status = response ? response.status : null;
+      return Promise.reject({ status });
+    });
+}
+
+export function getColumnLineage(
+  key: string,
+  columnName: string,
+  depth: number,
+  direction: string
+) {
+  const tableQueryParams = getQueryParams({
+    key,
+    depth,
+    direction,
+    column_name: columnName,
+  });
+  return axios
+    .get(`${API_PATH}/get_column_lineage?${tableQueryParams}`)
+    .then((response: AxiosResponse<LineageAPI>) => ({
+      data: response.data,
+      statusCode: response.status,
+    }))
+    .catch((e: AxiosError<LineageAPI>) => {
+      const { response } = e;
+      const status = response ? response.status : null;
+      return Promise.reject({ status });
+    });
+}