Skip to content

Commit

Permalink
Merge pull request #106 from atlanhq/staging
Browse files Browse the repository at this point in the history 
Main << Staging
  • Loading branch information
@Jaagrav
Jaagrav authored Jun 22, 2023
2 parents 7236c40 + 7c5a7ad commit 9c29dbb
Show file tree
Hide file tree
Showing 4 changed files with 222 additions and 129 deletions.
177 changes: 112 additions & 65 deletions dist/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17830,7 +17830,14 @@ const isIgnoreModelAliasMatching = () => core.getInput("IGNORE_MODEL_ALIAS_MATCH
const create_comment_IS_DEV = isDev();
const create_comment_ATLAN_INSTANCE_URL =
getInstanceUrl();
const ASSETS_LIMIT = 100;

function truncate(value) {
if (typeof value === 'string')
return value.length > 100 ? value.substring(0, 100) + "..." : value;
if (Array.isArray(value))
return value.length > 10 ? value.slice(0, 10).join(", ") + "..." : value.join(", ");
return ""
}

async function renderDownstreamAssetsComment(
octokit,
Expand All @@ -17840,47 +17847,73 @@ async function renderDownstreamAssetsComment(
downstreamAssets,
classifications
) {
let impactedData = downstreamAssets.slice(0, ASSETS_LIMIT).map(
({displayText, guid, typeName, attributes, meanings, classificationNames}) => {
// Mapping the downstream assets data
let impactedData = downstreamAssets.entities.map(
({
displayText,
guid,
typeName,
attributes,
meanings,
classificationNames
}) => {
// Modifying the typeName and getting the readableTypeName
let readableTypeName = typeName
.toLowerCase()
.replace(attributes.connectorName, "")
.toUpperCase(),
classificationsObj = classifications.filter(({name}) => classificationNames.includes(name));
readableTypeName = readableTypeName.charAt(0).toUpperCase() + readableTypeName.slice(1).toLowerCase()
.toLowerCase()
.replace(attributes.connectorName, "")
.toUpperCase();

// Filtering classifications based on classificationNames
let classificationsObj = classifications.filter(({name}) =>
classificationNames.includes(name)
);

// Modifying the readableTypeName
readableTypeName = readableTypeName.charAt(0).toUpperCase() + readableTypeName.slice(1).toLowerCase();

return [
guid, displayText, attributes.connectorName, readableTypeName, attributes?.userDescription || attributes?.description || "", attributes?.certificateStatus || "", [...attributes?.ownerUsers, ...attributes?.ownerGroups] || [], meanings.map(
({displayText, termGuid}) =>
`[${displayText}](${create_comment_ATLAN_INSTANCE_URL}/assets/${termGuid}/overview?utm_source=dbt_github_action)`,
)
?.join(", ") || " ",
classificationsObj?.map(({
name,
displayName
}) => `\`${displayName}\``)?.join(', ') || " ", attributes?.sourceURL || ""
guid,
truncate(displayText),
truncate(attributes.connectorName),
truncate(readableTypeName),
truncate(attributes?.userDescription || attributes?.description || ""),
attributes?.certificateStatus || "",
truncate([...attributes?.ownerUsers, ...attributes?.ownerGroups] || []),
truncate(meanings.map(({displayText, termGuid}) =>
`[${displayText}](${create_comment_ATLAN_INSTANCE_URL}/assets/${termGuid}/overview?utm_source=dbt_github_action)`
)),
truncate(classificationsObj?.map(({name, displayName}) =>
`\`${displayName}\``
)),
attributes?.sourceURL || ""
];
}
);

impactedData = impactedData.sort((a, b) => a[3].localeCompare(b[3])); // Sort by typeName
impactedData = impactedData.sort((a, b) => a[2].localeCompare(b[2])); // Sort by connectorName

let rows = impactedData.map(([guid, displayText, connectorName, typeName, description, certificateStatus, owners, meanings, classifications, sourceUrl]) => {
const connectorImage = getConnectorImage(connectorName),
certificationImage = certificateStatus
? getCertificationImage(certificateStatus)
: "";

return [`${connectorImage} [${displayText}](${create_comment_ATLAN_INSTANCE_URL}/assets/${guid}/overview?utm_source=dbt_github_action) ${certificationImage}`,
`\`${typeName}\``,
description,
owners.join(", ") || " ",
meanings,
classifications,
sourceUrl ? `[Open in ${connectorName}](${sourceUrl})` : " "]
})
// Sorting the impactedData first by typeName and then by connectorName
impactedData = impactedData.sort((a, b) => a[3].localeCompare(b[3]));
impactedData = impactedData.sort((a, b) => a[2].localeCompare(b[2]));

// Creating rows for the downstream table
let rows = impactedData.map(
([guid, displayText, connectorName, typeName, description, certificateStatus, owners, meanings, classifications, sourceUrl]) => {
// Getting connector and certification images
const connectorImage = getConnectorImage(connectorName);
const certificationImage = certificateStatus ? getCertificationImage(certificateStatus) : "";

return [
`${connectorImage} [${displayText}](${create_comment_ATLAN_INSTANCE_URL}/assets/${guid}/overview?utm_source=dbt_github_action) ${certificationImage}`,
`\`${typeName}\``,
description,
owners,
meanings,
classifications,
sourceUrl ? `[Open in ${connectorName}](${sourceUrl})` : " "
];
}
);

// Generating asset information
const assetInfo = `### ${getConnectorImage(asset.attributes.connectorName)} [${
asset.displayText
}](${create_comment_ATLAN_INSTANCE_URL}/assets/${asset.guid}/overview?utm_source=dbt_github_action) ${
Expand All @@ -17894,34 +17927,39 @@ Materialised asset: ${getConnectorImage(materialisedAsset.attributes.connectorNa
materialisedAsset.attributes?.certificateStatus
? getCertificationImage(materialisedAsset.attributes.certificateStatus)
: ""
} | Environment Name: \`${materialisedAsset.attributes.assetDbtEnvironmentName}\` | Project Name: \`${materialisedAsset.attributes.assetDbtProjectName}\``
} | Environment Name: \`${materialisedAsset.attributes.assetDbtEnvironmentName}\` | Project Name: \`${materialisedAsset.attributes.assetDbtProjectName}\``;

const downstreamTable = `<details><summary><b>${downstreamAssets.length} downstream assets 👇</b></summary><br/>
// Generating the downstream table
const downstreamTable = `<details><summary><b>${downstreamAssets.entityCount} downstream assets 👇</b></summary><br/>

Name | Type | Description | Owners | Terms | Classifications | Source URL
--- | --- | --- | --- | --- | --- | ---
${rows.map((row) => row.map(i => i.replace(/\|/g, "•").replace(/\n/g, "")).join(" | ")).join("\n")}

${downstreamAssets.length > ASSETS_LIMIT ? `[+${downstreamAssets.length - ASSETS_LIMIT} assets more](${create_comment_ATLAN_INSTANCE_URL}/assets/${materialisedAsset.guid}/lineage?utm_source=dbt_github_action)` : ""}
${downstreamAssets.hasMore ? `[See more downstream assets at Atlan](${create_comment_ATLAN_INSTANCE_URL}/assets/${materialisedAsset.guid}/lineage?utm_source=dbt_github_action)` : ""}

</details>`
</details>`;

const viewAssetButton = `${getImageURL("atlan-logo", 15, 15)} [View asset in Atlan](${create_comment_ATLAN_INSTANCE_URL}/assets/${asset.guid}/overview?utm_source=dbt_github_action)`
// Generating the "View asset in Atlan" button
const viewAssetButton = `${getImageURL("atlan-logo", 15, 15)} [View asset in Atlan](${create_comment_ATLAN_INSTANCE_URL}/assets/${asset.guid}/overview?utm_source=dbt_github_action)`;

if (downstreamAssets.length > 0)
// Generating the final comment based on the presence of downstream assets
if (downstreamAssets.entities.length > 0) {
return `${assetInfo}

${downstreamTable}

${viewAssetButton}`;

return `${assetInfo}
} else {
return `${assetInfo}

No downstream assets found.

${viewAssetButton}`
${viewAssetButton}`;
}
}


async function checkCommentExists(octokit, context) {
if (create_comment_IS_DEV) return null;

Expand Down Expand Up @@ -17966,6 +18004,7 @@ async function deleteComment(octokit, context, comment_id) {
comment_id,
});
}

;// CONCATENATED MODULE: ./src/utils/file-system.js
async function getFileContents(octokit, context, filePath) {
const {repository, pull_request} = context.payload,
Expand Down Expand Up @@ -18125,24 +18164,36 @@ const get_downstream_assets_ATLAN_INSTANCE_URL =
getInstanceUrl();
const get_downstream_assets_ATLAN_API_TOKEN =
getAPIToken();
const ASSETS_LIMIT = 100;

async function getDownstreamAssets(asset, guid, octokit, context) {
async function getDownstreamAssets(asset, guid, totalModifiedFiles) {
var myHeaders = {
authorization: `Bearer ${get_downstream_assets_ATLAN_API_TOKEN}`,
"content-type": "application/json",
};

var raw = stringify({
depth: 21,
guid: guid,
hideProcess: true,
allowDeletedProcess: false,
entityFilters: {
attributeName: "__state",
operator: "eq",
attributeValue: "ACTIVE",
"guid": guid,
"size": Math.round(ASSETS_LIMIT / totalModifiedFiles),
"from": 0,
"depth": 21,
"direction": "OUTPUT",
"entityFilters": {
"condition": "AND",
"criterion": [
{
"attributeName": "__typeName",
"operator": "not_contains",
"attributeValue": "Process"
},
{
"attributeName": "__state",
"operator": "eq",
"attributeValue": "ACTIVE"
}
]
},
attributes: [
"attributes": [
"name",
"description",
"userDescription",
Expand All @@ -18155,9 +18206,10 @@ async function getDownstreamAssets(asset, guid, octokit, context) {
"ownerUsers",
"ownerGroups",
"classificationNames",
"meanings",
"meanings"
],
direction: "OUTPUT",
"excludeMeanings": false,
"excludeClassifications": false
});

var requestOptions = {
Expand Down Expand Up @@ -18191,7 +18243,7 @@ ${getImageURL("atlan-logo", 15, 15)} [View lineage in Atlan](${get_downstream_as
}

var response = await fetch(
`${get_downstream_assets_ATLAN_INSTANCE_URL}/api/meta/lineage/getlineage`,
`${get_downstream_assets_ATLAN_INSTANCE_URL}/api/meta/lineage/list`,
requestOptions
).then((e) => {
if (e.status === 200) {
Expand All @@ -18207,13 +18259,7 @@ ${getImageURL("atlan-logo", 15, 15)} [View lineage in Atlan](${get_downstream_as

if (response.error) return response;

if (!response?.relations) return [];

const relations = response.relations.map(({toEntityId}) => toEntityId);

return relations
.filter((id, index) => relations.indexOf(id) === index)
.map((id) => response.guidEntityMap[id]);
return response;
}

;// CONCATENATED MODULE: ./src/api/get-asset.js
Expand Down Expand Up @@ -18539,7 +18585,8 @@ Its a new model and not present in Atlan yet, you'll see the downstream impact f

const materialisedAsset = asset.attributes.dbtModelSqlAssets[0];
const timeStart = Date.now();
const downstreamAssets = await getDownstreamAssets(asset, materialisedAsset.guid, octokit, context);
const totalModifiedFiles = changedFiles.filter(i => i.status === "modified").length
const downstreamAssets = await getDownstreamAssets(asset, materialisedAsset.guid, totalModifiedFiles);

if (downstreamAssets.error) {
comments += downstreamAssets.error;
Expand All @@ -18550,7 +18597,7 @@ Its a new model and not present in Atlan yet, you'll see the downstream impact f
sendSegmentEvent("dbt_ci_action_downstream_unfurl", {
asset_guid: asset.guid,
asset_type: asset.typeName,
downstream_count: downstreamAssets.length,
downstream_count: downstreamAssets.entities.length,
total_fetch_time: Date.now() - timeStart,
});

Expand Down
47 changes: 27 additions & 20 deletions src/api/get-downstream-assets.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,24 +7,36 @@ const ATLAN_INSTANCE_URL =
getInstanceUrl();
const ATLAN_API_TOKEN =
getAPIToken();
const ASSETS_LIMIT = 100;

export default async function getDownstreamAssets(asset, guid, octokit, context) {
export default async function getDownstreamAssets(asset, guid, totalModifiedFiles) {
var myHeaders = {
authorization: `Bearer ${ATLAN_API_TOKEN}`,
"content-type": "application/json",
};

var raw = stringify({
depth: 21,
guid: guid,
hideProcess: true,
allowDeletedProcess: false,
entityFilters: {
attributeName: "__state",
operator: "eq",
attributeValue: "ACTIVE",
"guid": guid,
"size": Math.round(ASSETS_LIMIT / totalModifiedFiles),
"from": 0,
"depth": 21,
"direction": "OUTPUT",
"entityFilters": {
"condition": "AND",
"criterion": [
{
"attributeName": "__typeName",
"operator": "not_contains",
"attributeValue": "Process"
},
{
"attributeName": "__state",
"operator": "eq",
"attributeValue": "ACTIVE"
}
]
},
attributes: [
"attributes": [
"name",
"description",
"userDescription",
Expand All @@ -37,9 +49,10 @@ export default async function getDownstreamAssets(asset, guid, octokit, context)
"ownerUsers",
"ownerGroups",
"classificationNames",
"meanings",
"meanings"
],
direction: "OUTPUT",
"excludeMeanings": false,
"excludeClassifications": false
});

var requestOptions = {
Expand Down Expand Up @@ -73,7 +86,7 @@ ${getImageURL("atlan-logo", 15, 15)} [View lineage in Atlan](${ATLAN_INSTANCE_UR
}

var response = await fetch(
`${ATLAN_INSTANCE_URL}/api/meta/lineage/getlineage`,
`${ATLAN_INSTANCE_URL}/api/meta/lineage/list`,
requestOptions
).then((e) => {
if (e.status === 200) {
Expand All @@ -89,11 +102,5 @@ ${getImageURL("atlan-logo", 15, 15)} [View lineage in Atlan](${ATLAN_INSTANCE_UR

if (response.error) return response;

if (!response?.relations) return [];

const relations = response.relations.map(({toEntityId}) => toEntityId);

return relations
.filter((id, index) => relations.indexOf(id) === index)
.map((id) => response.guidEntityMap[id]);
return response;
}
5 changes: 3 additions & 2 deletions src/main/print-downstream-assets.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,8 @@ Its a new model and not present in Atlan yet, you'll see the downstream impact f

const materialisedAsset = asset.attributes.dbtModelSqlAssets[0];
const timeStart = Date.now();
const downstreamAssets = await getDownstreamAssets(asset, materialisedAsset.guid, octokit, context);
const totalModifiedFiles = changedFiles.filter(i => i.status === "modified").length
const downstreamAssets = await getDownstreamAssets(asset, materialisedAsset.guid, totalModifiedFiles);

if (downstreamAssets.error) {
comments += downstreamAssets.error;
Expand All @@ -49,7 +50,7 @@ Its a new model and not present in Atlan yet, you'll see the downstream impact f
sendSegmentEvent("dbt_ci_action_downstream_unfurl", {
asset_guid: asset.guid,
asset_type: asset.typeName,
downstream_count: downstreamAssets.length,
downstream_count: downstreamAssets.entities.length,
total_fetch_time: Date.now() - timeStart,
});

Expand Down
Loading

0 comments on commit 9c29dbb

Please sign in to comment.