Pipdreive fix polling sources (PipedreamHQ#4727)

* WIP: Changed polling sources to webhook methods

* Added fieldId lookup and filter creation for sources

components/pipedrive/actions/add-activity/add-activity.mjs

@@ -6,7 +6,7 @@ export default {
   key: "pipedrive-add-activity",
   name: "Add Activity",
   description: "Adds a new activity. Includes `more_activities_scheduled_in_context` property in response's `additional_data` which indicates whether there are more undone activities scheduled with the same deal, person or organization (depending on the supplied data). See the Pipedrive API docs for Activities [here](https://developers.pipedrive.com/docs/api/v1/#!/Activities). For info on [adding an activity in Pipedrive](https://developers.pipedrive.com/docs/api/v1/Activities#addActivity)",
-  version: "0.1.3",
+  version: "0.1.4",
   type: "action",
   props: {
     pipedriveApp,

components/pipedrive/actions/add-deal/add-deal.mjs

@@ -4,7 +4,7 @@ export default {
   key: "pipedrive-add-deal",
   name: "Add Deal",
   description: "Adds a new deal. See the Pipedrive API docs for Deals [here](https://developers.pipedrive.com/docs/api/v1/Deals#addDeal)",
-  version: "0.1.3",
+  version: "0.1.4",
   type: "action",
   props: {
     pipedriveApp,

components/pipedrive/actions/add-note/add-note.mjs

@@ -4,7 +4,7 @@ export default {
   key: "pipedrive-add-note",
   name: "Add Note",
   description: "Adds a new note. For info on [adding an note in Pipedrive](https://developers.pipedrive.com/docs/api/v1/Notes#addNote)",
-  version: "0.0.1",
+  version: "0.0.2",
   type: "action",
   props: {
     pipedriveApp,

components/pipedrive/actions/add-organization/add-organization.mjs

@@ -4,7 +4,7 @@ export default {
   key: "pipedrive-add-organization",
   name: "Add Organization",
   description: "Adds a new organization. See the Pipedrive API docs for Organizations [here](https://developers.pipedrive.com/docs/api/v1/Organizations#addOrganization)",
-  version: "0.1.3",
+  version: "0.1.4",
   type: "action",
   props: {
     pipedriveApp,

components/pipedrive/actions/add-person/add-person.mjs

@@ -4,7 +4,7 @@ export default {
   key: "pipedrive-add-person",
   name: "Add Person",
   description: "Adds a new person. See the Pipedrive API docs for People [here](https://developers.pipedrive.com/docs/api/v1/Persons#addPerson)",
-  version: "0.1.3",
+  version: "0.1.4",
   type: "action",
   props: {
     pipedriveApp,

components/pipedrive/actions/search-persons/search-persons.mjs

@@ -5,7 +5,7 @@ export default {
   key: "pipedrive-search-persons",
   name: "Search persons",
   description: "Searches all Persons by `name`, `email`, `phone`, `notes` and/or custom fields. This endpoint is a wrapper of `/v1/itemSearch` with a narrower OAuth scope. Found Persons can be filtered by Organization ID. See the Pipedrive API docs [here](https://developers.pipedrive.com/docs/api/v1/Persons#searchPersons)",
-  version: "0.1.3",
+  version: "0.1.4",
   type: "action",
   props: {
     pipedriveApp,

components/pipedrive/actions/update-deal/update-deal.mjs

@@ -4,7 +4,7 @@ export default {
   key: "pipedrive-update-deal",
   name: "Update Deal",
   description: "Updates the properties of a deal. See the Pipedrive API docs for Deals [here](https://developers.pipedrive.com/docs/api/v1/Deals#updateDeal)",
-  version: "0.1.4",
+  version: "0.1.5",
   type: "action",
   props: {
     pipedriveApp,

components/pipedrive/common/constants.mjs

@@ -1,6 +1,8 @@
 const LAST_RESOURCE_PROPERTY = "lastResourceProperty";
-const DEFAULT_PAGE_LIMIT = 20;
-const DEFAULT_MAX_ITEMS = 100;
+const FILTER_ID = "filterId";
+const FIELD_ID = "fieldId";
+const DEFAULT_PAGE_LIMIT = 20; // max is 500 per page
+const DEFAULT_MAX_ITEMS = DEFAULT_PAGE_LIMIT * 4;
 
 const STATUS_OPTIONS = [
   "open",
@@ -45,6 +47,15 @@ const EVENT_OBJECT = {
   USER: "user",
 };
 
+const FILTER_TYPE = {
+  DEALS: "deals",
+  LEADS: "leads",
+  ORG: "org",
+  PEOPLE: "people",
+  PRODUCTS: "products",
+  ACTIVITY: "activity",
+};
+
 const EVENT_ACTION = {
   ADDED: "added",
   UPDATED: "updated",
@@ -140,6 +151,8 @@ const API = {
   ],
   FILTERS: [
     "FiltersApi",
+    "AddFilterRequest",
+    "UpdateFilterRequest",
   ],
   FILES: [
     "FilesApi",
@@ -171,15 +184,24 @@ const API = {
   ],
 };
 
+const FIELD = {
+  ADD_TIME: "add_time",
+  UPDATE_TIME: "update_time",
+};
+
 export default {
   STATUS_OPTIONS,
   FIELD_OPTIONS,
   VISIBLE_TO_OPTIONS,
   INCLUDE_FIELDS_OPTIONS,
   LAST_RESOURCE_PROPERTY,
+  FILTER_ID,
+  FIELD_ID,
   EVENT_OBJECT,
   EVENT_ACTION,
   API,
   DEFAULT_PAGE_LIMIT,
   DEFAULT_MAX_ITEMS,
+  FIELD,
+  FILTER_TYPE,
 };

components/pipedrive/common/utils.mjs

@@ -7,4 +7,11 @@ export default {
       ? value
       : JSON.parse(value);
   },
+  async streamIterator(stream) {
+    const resources = [];
+    for await (const resource of stream) {
+      resources.push(resource);
+    }
+    return resources;
+  },
 };

components/pipedrive/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pipedream/pipedrive",
-  "version": "0.3.6",
+  "version": "0.3.7",
   "description": "Pipedream Pipedrive Components",
   "main": "pipedrive.app.mjs",
   "keywords": [

components/pipedrive/pipedrive.app.mjs

@@ -239,8 +239,8 @@ export default {
   },
   methods: {
     setupToken() {
-      const api = pipedrive.ApiClient.instance;
-      api.authentications.oauth2.accessToken = this.$auth.oauth_access_token;
+      const client = pipedrive.ApiClient.instance;
+      client.authentications.oauth2.accessToken = this.$auth.oauth_access_token;
     },
     api(className) {
       this.setupToken();
@@ -255,21 +255,20 @@ export default {
         addProperty,
       ] = constants.API.ACTIVITIES;
       try {
-        const activityOpts = this.buildOpts(addProperty, opts);
-        return this.api(className).addActivity(activityOpts);
+        return this.api(className).addActivity(this.buildOpts(addProperty, opts));
       } catch (error) {
         console.error(error);
         throw new Error(error);
       }
     },
-    async addDeal(opts = {}) {
+    addDeal(opts = {}) {
       const [
         className,
         addProperty,
       ] = constants.API.DEALS;
       return this.api(className).addDeal(this.buildOpts(addProperty, opts));
     },
-    async updateDeal(opts = {}) {
+    updateDeal(opts = {}) {
       const {
         dealId,
         ...otherOpts
@@ -293,13 +292,13 @@ export default {
       ] = constants.API.ORGANIZATIONS;
       return this.api(className).addOrganization(this.buildOpts(addProperty, opts));
     },
-    async addPerson(opts = {}) {
+    addPerson(opts = {}) {
       const [
         className,
       ] = constants.API.PERSONS;
       return this.api(className).addPerson(opts);
     },
-    async searchPersons(opts = {}) {
+    searchPersons(opts = {}) {
       const {
         term,
         ...otherOpts
@@ -309,13 +308,37 @@ export default {
       ] = constants.API.PERSONS;
       return this.api(className).searchPersons(term, otherOpts);
     },
-    async getDeals(opts = {}) {
+    addFilter(opts = {}) {
+      const [
+        className,
+        addProperty,
+      ] = constants.API.FILTERS;
+      return this.api(className).addFilter(this.buildOpts(addProperty, opts));
+    },
+    updateFilter(opts = {}) {
+      const {
+        filterId,
+        ...otherOpts
+      } = opts;
+      const [
+        className,,
+        updateProperty,
+      ] = constants.API.FILTERS;
+      return this.api(className).updateFilter(filterId, this.buildOpts(updateProperty, otherOpts));
+    },
+    deleteFilter(filterId) {
+      const [
+        className,
+      ] = constants.API.FILTERS;
+      return this.api(className).deleteFilter(filterId);
+    },
+    getDeals(opts = {}) {
       const [
         className,
       ] = constants.API.DEALS;
       return this.api(className).getDeals(opts);
     },
-    async getPersons(opts = {}) {
+    getPersons(opts = {}) {
       const [
         className,
       ] = constants.API.PERSONS;
@@ -333,73 +356,69 @@ export default {
       ] = constants.API.USERS;
       return this.api(className).getUsers(opts);
     },
-    async getActivityTypes(opts) {
+    getActivityTypes(opts) {
       const [
         className,
       ] = constants.API.ACTIVITY_TYPES;
       return this.api(className).getActivityTypes(opts);
     },
-    async getOrganizations(opts = {}) {
+    getOrganizations(opts = {}) {
       const [
         className,
       ] = constants.API.ORGANIZATIONS;
       return this.api(className).getOrganizations(opts);
     },
-    async getStages(opts) {
+    getStages(opts) {
       const [
         className,
       ] = constants.API.STAGES;
       return this.api(className).getStages(opts);
     },
+    getDealFields(opts = {}) {
+      const [
+        className,
+      ] = constants.API.DEAL_FIELDS;
+      return this.api(className).getDealFields(opts);
+    },
+    getPersonFields(opts = {}) {
+      const [
+        className,
+      ] = constants.API.PERSON_FIELDS;
+      return this.api(className).getPersonFields(opts);
+    },
     async *getResourcesStream({
       resourceFn,
       resourceFnArgs,
       limit = constants.DEFAULT_PAGE_LIMIT,
-      max,
-      lastResourceProperty,
-      done,
+      max = constants.DEFAULT_MAX_ITEMS,
     }) {
-      let page = 0;
+      let start = 0;
       let resourcesCount = 0;
 
       while (true) {
-        const nextResponse =
+        const response =
           await resourceFn({
             ...resourceFnArgs,
             limit,
-            start: page * limit,
+            start,
           });
 
-        if (!nextResponse) {
-          throw new Error("No response from the Pipedrive API.");
-        }
+        const nextResources = response?.data || [];
+        start = response?.additional_data?.pagination?.next_start;
 
-        const {
-          data: nextResources,
-          additional_data: additionalData,
-        } = nextResponse;
-
-        const { more_items_in_collection: moreItemsInCollection } = additionalData.pagination;
-
-        for (const resource of nextResources || []) {
-          const isDone = done && done({
-            lastResourceProperty,
-            resource,
-          });
-
-          if (isDone) {
-            return;
-          }
+        if (!nextResources.length) {
+          console.log("No more records to fetch.");
+          return;
+        }
 
+        for (const resource of nextResources) {
           resourcesCount += 1;
           yield resource;
         }
 
-        if (!moreItemsInCollection || (max && resourcesCount >= max)) {
+        if (!start || resourcesCount >= max) {
           return;
         }
-
-        page += 1;
       }
     },
   },