Updating RestResource: ds_donorsearch_REST_test

Updating pipeline: pl_donorsearch_http_test Updating Json: ds_donorsearch_sink_REST_test
abhishekanand10 · May 20, 2024 · 689bc35 · 689bc35
1 parent 7233fde
commit 689bc35
Show file tree

Hide file tree

Showing 3 changed files with 118 additions and 43 deletions.
diff --git a/dataset/ds_donorsearch_REST_test.json b/dataset/ds_donorsearch_REST_test.json
@@ -5,10 +5,24 @@
 			"referenceName": "ls_donorsearch_REST_test",
 			"type": "LinkedServiceReference"
 		},
+		"parameters": {
+			"firstName": {
+				"type": "string"
+			},
+			"lastName": {
+				"type": "string"
+			},
+			"homeState": {
+				"type": "string"
+			}
+		},
 		"annotations": [],
 		"type": "RestResource",
 		"typeProperties": {
-			"relativeUrl": "?key=5d95e4d6342b5&firstName=@{pipeline().parameters.firstName}&lastName=@{pipeline().parameters.lastName}&homeState=@{pipeline().parameters.homeState}"
+			"relativeUrl": {
+				"value": "?key=5d95e4d6342b5&firstName=@{dataset().firstName}&lastName=@{dataset().lastName}&homeState=@{dataset().homeState}",
+				"type": "Expression"
+			}
 		},
 		"schema": []
 	}

diff --git a/dataset/ds_donorsearch_sink_REST_test.json b/dataset/ds_donorsearch_sink_REST_test.json
@@ -5,13 +5,21 @@
 			"referenceName": "ls_ablob_cvidreportingsa",
 			"type": "LinkedServiceReference"
 		},
+		"parameters": {
+			"id": {
+				"type": "string"
+			}
+		},
 		"annotations": [],
 		"type": "Json",
 		"typeProperties": {
 			"location": {
 				"type": "AzureBlobStorageLocation",
-				"fileName": "donor.json",
-				"folderPath": "test",
+				"fileName": {
+					"value": "@concat('donor',dataset().id,'.json')",
+					"type": "Expression"
+				},
+				"folderPath": "test/temp/",
 				"container": "donorsearchhttpblob"
 			}
 		},

diff --git a/pipeline/pl_donorsearch_http_test.json b/pipeline/pl_donorsearch_http_test.json
@@ -3,8 +3,8 @@
 	"properties": {
 		"activities": [
 			{
-				"name": "Copy donorsearch http data",
-				"type": "Copy",
+				"name": "Lookup List of constituents",
+				"type": "Lookup",
 				"dependsOn": [],
 				"policy": {
 					"timeout": "0.12:00:00",
@@ -16,51 +16,104 @@
 				"userProperties": [],
 				"typeProperties": {
 					"source": {
-						"type": "RestSource",
-						"httpRequestTimeout": "00:01:40",
-						"requestInterval": "00.00:00:00.010",
-						"requestMethod": "GET",
-						"paginationRules": {
-							"supportRFC5988": "true"
-						}
+						"type": "AzureSqlSource",
+						"queryTimeout": "02:00:00",
+						"partitionOption": "None"
 					},
-					"sink": {
-						"type": "JsonSink",
-						"storeSettings": {
-							"type": "AzureBlobStorageWriteSettings",
-							"copyBehavior": "MergeFiles"
-						},
-						"formatSettings": {
-							"type": "JsonWriteSettings"
-						}
+					"dataset": {
+						"referenceName": "ds_mock_constituents_test",
+						"type": "DatasetReference"
 					},
-					"enableStaging": false
-				},
-				"inputs": [
+					"firstRowOnly": false
+				}
+			},
+			{
+				"name": "ForEach constituent",
+				"type": "ForEach",
+				"dependsOn": [
 					{
-						"referenceName": "ds_donorsearch_REST_test",
-						"type": "DatasetReference"
+						"activity": "Lookup List of constituents",
+						"dependencyConditions": [
+							"Succeeded"
+						]
 					}
 				],
-				"outputs": [
-					{
-						"referenceName": "ds_donorsearch_sink_REST_test",
-						"type": "DatasetReference"
-					}
-				]
+				"userProperties": [],
+				"typeProperties": {
+					"items": {
+						"value": "@activity('Lookup List of constituents').output.value",
+						"type": "Expression"
+					},
+					"isSequential": false,
+					"activities": [
+						{
+							"name": "Copy donorsearch http data",
+							"type": "Copy",
+							"dependsOn": [],
+							"policy": {
+								"timeout": "0.12:00:00",
+								"retry": 0,
+								"retryIntervalInSeconds": 30,
+								"secureOutput": false,
+								"secureInput": false
+							},
+							"userProperties": [],
+							"typeProperties": {
+								"source": {
+									"type": "RestSource",
+									"httpRequestTimeout": "00:01:40",
+									"requestInterval": "00.00:00:00.010",
+									"requestMethod": "GET",
+									"paginationRules": {
+										"supportRFC5988": "true"
+									}
+								},
+								"sink": {
+									"type": "JsonSink",
+									"storeSettings": {
+										"type": "AzureBlobStorageWriteSettings",
+										"copyBehavior": "MergeFiles"
+									},
+									"formatSettings": {
+										"type": "JsonWriteSettings"
+									}
+								},
+								"enableStaging": false
+							},
+							"inputs": [
+								{
+									"referenceName": "ds_donorsearch_REST_test",
+									"type": "DatasetReference",
+									"parameters": {
+										"firstName": {
+											"value": "@item().firstName",
+											"type": "Expression"
+										},
+										"lastName": {
+											"value": "@item().lastName",
+											"type": "Expression"
+										},
+										"homeState": "@item().homeState"
+									}
+								}
+							],
+							"outputs": [
+								{
+									"referenceName": "ds_donorsearch_sink_REST_test",
+									"type": "DatasetReference",
+									"parameters": {
+										"id": {
+											"value": "@item().id",
+											"type": "Expression"
+										}
+									}
+								}
+							]
+						}
+					]
+				}
 			}
 		],
-		"parameters": {
-			"firstName": {
-				"type": "string"
-			},
-			"lastName": {
-				"type": "string"
-			},
-			"homeState": {
-				"type": "string"
-			}
-		},
 		"annotations": []
 	}
 }