From 9ae9d77c01da41554950f06df52f4f4353a45ad8 Mon Sep 17 00:00:00 2001 From: michelle0927 Date: Mon, 21 Apr 2025 17:51:28 -0400 Subject: [PATCH 1/4] updates --- .../apify/actions/get-dataset-items/get-dataset-items.mjs | 2 +- components/apify/actions/run-actor/run-actor.mjs | 8 +++++++- .../run-task-synchronously/run-task-synchronously.mjs | 2 +- .../apify/actions/scrape-single-url/scrape-single-url.mjs | 2 +- .../set-key-value-store-record.mjs | 2 +- components/apify/apify.app.mjs | 2 +- components/apify/package.json | 2 +- .../new-finished-actor-run-instant.mjs | 2 +- .../new-finished-task-run-instant.mjs | 2 +- 9 files changed, 15 insertions(+), 9 deletions(-) diff --git a/components/apify/actions/get-dataset-items/get-dataset-items.mjs b/components/apify/actions/get-dataset-items/get-dataset-items.mjs index 54f483cf90d40..f596171377d99 100644 --- a/components/apify/actions/get-dataset-items/get-dataset-items.mjs +++ b/components/apify/actions/get-dataset-items/get-dataset-items.mjs @@ -5,7 +5,7 @@ export default { key: "apify-get-dataset-items", name: "Get Dataset Items", description: "Returns data stored in a dataset. [See the documentation](https://docs.apify.com/api/v2/dataset-items-get)", - version: "0.0.1", + version: "0.0.2", type: "action", props: { apify, diff --git a/components/apify/actions/run-actor/run-actor.mjs b/components/apify/actions/run-actor/run-actor.mjs index ce776c00b1be7..96fec4a087636 100644 --- a/components/apify/actions/run-actor/run-actor.mjs +++ b/components/apify/actions/run-actor/run-actor.mjs @@ -5,7 +5,7 @@ export default { key: "apify-run-actor", name: "Run Actor", description: "Performs an execution of a selected actor in Apify. [See the documentation](https://docs.apify.com/api/v2#/reference/actors/run-collection/run-actor)", - version: "0.0.2", + version: "0.0.3", type: "action", props: { apify, @@ -104,6 +104,12 @@ export default { }; const options = this.prepareOptions(value); if (options) props[key].options = options; + if (value.default) { + props[key].description += ` Default: \`${JSON.stringify(value.default)}\``; + if (props[key].type !== "object") { // default values don't work properly for object props + props[key].default = value.default; + } + } } } return props; diff --git a/components/apify/actions/run-task-synchronously/run-task-synchronously.mjs b/components/apify/actions/run-task-synchronously/run-task-synchronously.mjs index 52a3ef7a4fa4a..b2153c9f219f2 100644 --- a/components/apify/actions/run-task-synchronously/run-task-synchronously.mjs +++ b/components/apify/actions/run-task-synchronously/run-task-synchronously.mjs @@ -4,7 +4,7 @@ export default { key: "apify-run-task-synchronously", name: "Run Task Synchronously", description: "Run a specific task and return its dataset items. [See the documentation](https://docs.apify.com/api/v2/actor-task-run-sync-get-dataset-items-get)", - version: "0.0.1", + version: "0.0.2", type: "action", props: { apify, diff --git a/components/apify/actions/scrape-single-url/scrape-single-url.mjs b/components/apify/actions/scrape-single-url/scrape-single-url.mjs index 3cb7d3ab2bbbb..2de46d5b103bc 100644 --- a/components/apify/actions/scrape-single-url/scrape-single-url.mjs +++ b/components/apify/actions/scrape-single-url/scrape-single-url.mjs @@ -5,7 +5,7 @@ export default { key: "apify-scrape-single-url", name: "Scrape Single URL", description: "Executes a scraper on a specific website and returns its content as text. This action is perfect for extracting content from a single page.", - version: "0.0.2", + version: "0.0.3", type: "action", props: { apify, diff --git a/components/apify/actions/set-key-value-store-record/set-key-value-store-record.mjs b/components/apify/actions/set-key-value-store-record/set-key-value-store-record.mjs index fc3c4ddf6cbec..4586b75d940a3 100644 --- a/components/apify/actions/set-key-value-store-record/set-key-value-store-record.mjs +++ b/components/apify/actions/set-key-value-store-record/set-key-value-store-record.mjs @@ -5,7 +5,7 @@ export default { key: "apify-set-key-value-store-record", name: "Set Key-Value Store Record", description: "Create or update a record in the key-value store of Apify. [See the documentation](https://docs.apify.com/api/v2#/reference/key-value-stores/record-collection/put-record)", - version: "0.0.2", + version: "0.0.3", type: "action", props: { apify, diff --git a/components/apify/apify.app.mjs b/components/apify/apify.app.mjs index 5452ce12ca802..c2bf80036cd23 100644 --- a/components/apify/apify.app.mjs +++ b/components/apify/apify.app.mjs @@ -174,7 +174,7 @@ export default { }) { return this._makeRequest({ method: "POST", - path: `/acts/${actorId}/runs`, + path: `/acts/${actorId}/run-sync`, ...opts, }); }, diff --git a/components/apify/package.json b/components/apify/package.json index 93b7a54f2e40c..27476a27bb947 100644 --- a/components/apify/package.json +++ b/components/apify/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/apify", - "version": "0.2.0", + "version": "0.2.1", "description": "Pipedream Apify Components", "main": "apify.app.mjs", "keywords": [ diff --git a/components/apify/sources/new-finished-actor-run-instant/new-finished-actor-run-instant.mjs b/components/apify/sources/new-finished-actor-run-instant/new-finished-actor-run-instant.mjs index 0682efa6a181a..20426d00d1682 100644 --- a/components/apify/sources/new-finished-actor-run-instant/new-finished-actor-run-instant.mjs +++ b/components/apify/sources/new-finished-actor-run-instant/new-finished-actor-run-instant.mjs @@ -6,7 +6,7 @@ export default { key: "apify-new-finished-actor-run-instant", name: "New Finished Actor Run (Instant)", description: "Emit new event when a selected actor is run and finishes.", - version: "0.0.2", + version: "0.0.3", type: "source", dedupe: "unique", props: { diff --git a/components/apify/sources/new-finished-task-run-instant/new-finished-task-run-instant.mjs b/components/apify/sources/new-finished-task-run-instant/new-finished-task-run-instant.mjs index 94146ecf19f2a..3d7f8370d7532 100644 --- a/components/apify/sources/new-finished-task-run-instant/new-finished-task-run-instant.mjs +++ b/components/apify/sources/new-finished-task-run-instant/new-finished-task-run-instant.mjs @@ -6,7 +6,7 @@ export default { key: "apify-new-finished-task-run-instant", name: "New Finished Task Run (Instant)", description: "Emit new event when a selected task is run and finishes.", - version: "0.0.2", + version: "0.0.3", type: "source", dedupe: "unique", props: { From 9b61d712bf31a8613ed31b6057b7487a810b02f5 Mon Sep 17 00:00:00 2001 From: michelle0927 Date: Tue, 22 Apr 2025 14:48:25 -0400 Subject: [PATCH 2/4] updates --- .../apify/actions/run-actor/run-actor.mjs | 146 ++++++++++++++---- components/apify/apify.app.mjs | 30 +++- 2 files changed, 147 insertions(+), 29 deletions(-) diff --git a/components/apify/actions/run-actor/run-actor.mjs b/components/apify/actions/run-actor/run-actor.mjs index 96fec4a087636..e8395b04b1945 100644 --- a/components/apify/actions/run-actor/run-actor.mjs +++ b/components/apify/actions/run-actor/run-actor.mjs @@ -1,5 +1,6 @@ /* eslint-disable no-unused-vars */ import apify from "../../apify.app.mjs"; +import { parseObject } from "../../common/utils.mjs"; export default { key: "apify-run-actor", @@ -14,8 +15,53 @@ export default { apify, "actorId", ], + }, + buildId: { + propDefinition: [ + apify, + "buildId", + (c) => ({ + actorId: c.actorId, + }), + ], + reloadProps: true, + }, + runAsynchronously: { + type: "boolean", + label: "Run Asynchronously", + description: "Set to `true` to run the actor asynchronously", reloadProps: true, }, + timeout: { + type: "string", + label: "Timeout", + description: "Optional timeout for the run, in seconds. By default, the run uses a timeout specified in the default run configuration for the Actor.", + optional: true, + }, + memory: { + type: "string", + label: "Memory", + description: "Memory limit for the run, in megabytes. The amount of memory can be set to a power of 2 with a minimum of 128. By default, the run uses a memory limit specified in the default run configuration for the Actor.", + optional: true, + }, + maxItems: { + type: "string", + label: "Max Items", + description: "The maximum number of items that the Actor run should return. This is useful for pay-per-result Actors, as it allows you to limit the number of results that will be charged to your subscription. You can access the maximum number of items in your Actor by using the ACTOR_MAX_PAID_DATASET_ITEMS environment variable.", + optional: true, + }, + maxTotalChargeUsd: { + type: "string", + label: "Max Total Charge USD", + description: "Specifies the maximum cost of the Actor run. This parameter is useful for pay-per-event Actors, as it allows you to limit the amount charged to your subscription. You can access the maximum cost in your Actor by using the ACTOR_MAX_TOTAL_CHARGE_USD environment variable.", + optional: true, + }, + webhooks: { + type: "string", + label: "Webhooks", + description: "Specifies optional webhooks associated with the Actor run, which can be used to receive a notification e.g. when the Actor finished or failed. The value is a Base64-encoded JSON array of objects defining the webhooks. For more information, see [Webhooks documentation](https://docs.apify.com/platform/integrations/webhooks).", + optional: true, + }, }, methods: { getType(type) { @@ -28,16 +74,14 @@ export default { ? type : "string[]"; }, - async getSchema() { - const { data: { items: builds } } = await this.apify.listBuilds(this.actorId); - const buildId = builds.at(-1).id; + async getSchema(buildId) { const { data: { inputSchema } } = await this.apify.getBuild(buildId); return JSON.parse(inputSchema); }, async prepareData(data) { const newData = {}; - const { properties } = await this.getSchema(); + const { properties } = await this.getSchema(this.buildId); for (const [ key, value, @@ -85,31 +129,54 @@ export default { }, async additionalProps() { const props = {}; - if (this.actorId) { - const { - properties, required: requiredProps = [], - } = await this.getSchema(); + if (this.buildId) { + try { + const { + properties, required: requiredProps = [], + } = await this.getSchema(this.buildId); - for (const [ - key, - value, - ] of Object.entries(properties)) { - if (value.editor === "hidden") continue; + for (const [ + key, + value, + ] of Object.entries(properties)) { + if (value.editor === "hidden") continue; - props[key] = { - type: this.getType(value.type), - label: value.title, - description: value.description, - optional: !requiredProps.includes(key), - }; - const options = this.prepareOptions(value); - if (options) props[key].options = options; - if (value.default) { - props[key].description += ` Default: \`${JSON.stringify(value.default)}\``; - if (props[key].type !== "object") { // default values don't work properly for object props - props[key].default = value.default; + props[key] = { + type: this.getType(value.type), + label: value.title, + description: value.description, + optional: !requiredProps.includes(key), + }; + const options = this.prepareOptions(value); + if (options) props[key].options = options; + if (value.default) { + props[key].description += ` Default: \`${JSON.stringify(value.default)}\``; + if (props[key].type !== "object") { // default values don't work properly for object props + props[key].default = value.default; + } } } + } catch { + props.properties = { + type: "object", + label: "Properties", + description: "Properties to set for this actor", + }; + } + if (this.runAsynchronously) { + props.outputRecordKey = { + type: "string", + label: "Output Record Key", + description: "Key of the record from run's default key-value store to be returned in the response. By default, it is OUTPUT.", + optional: true, + }; + } else { + props.waitForFinish = { + type: "string", + label: "Wait For Finish", + description: "The maximum number of seconds the server waits for the run to finish. By default, it is 0, the maximum value is 60. If the build finishes in time then the returned run object will have a terminal status (e.g. SUCCEEDED), otherwise it will have a transitional status (e.g. RUNNING).", + optional: true, + }; } } return props; @@ -123,12 +190,37 @@ export default { prepareData, apify, actorId, + buildId, + properties, + runAsynchronously, + outputRecordKey, + timeout, + memory, + maxItems, + maxTotalChargeUsd, + waitForFinish, + webhooks, ...data } = this; - const response = await apify.runActor({ + const fn = runAsynchronously + ? apify.runActorAsynchronously + : apify.runActor; + + const response = await fn({ actorId, - data: await prepareData(data), + data: properties + ? parseObject(properties) + : await prepareData(data), + params: { + outputRecordKey, + timeout, + memory, + maxItems, + maxTotalChargeUsd, + waitForFinish, + webhooks, + }, }); $.export("$summary", `Successfully started actor run with ID: ${response.data.id}`); return response; diff --git a/components/apify/apify.app.mjs b/components/apify/apify.app.mjs index c2bf80036cd23..1a3e46a0df318 100644 --- a/components/apify/apify.app.mjs +++ b/components/apify/apify.app.mjs @@ -29,7 +29,7 @@ export default { actorId: { type: "string", label: "Actor ID", - description: "The ID of the actor to run.", + description: "Actor ID or a tilde-separated owner's username and Actor name", async options({ page }) { const { data: { items } } = await this.listActors({ params: { @@ -105,6 +105,23 @@ export default { })) || []; }, }, + buildId: { + type: "string", + label: "Build", + description: "Specifies the Actor build to run. It can be either a build tag or build number.", + async options({ + page, actorId, + }) { + const { data: { items } } = await this.listBuilds({ + actorId, + params: { + offset: LIMIT * page, + limit: LIMIT, + }, + }); + return items?.map(({ id }) => id) || []; + }, + }, clean: { type: "boolean", label: "Clean", @@ -178,6 +195,15 @@ export default { ...opts, }); }, + runActorAsynchronously({ + actorId, ...opts + }) { + return this._makeRequest({ + method: "POST", + path: `/acts/${actorId}/runs`, + ...opts, + }); + }, runTask({ taskId }) { return this._makeRequest({ method: "POST", @@ -207,7 +233,7 @@ export default { ...opts, }); }, - listBuilds(actorId) { + listBuilds({ actorId }) { return this._makeRequest({ path: `/acts/${actorId}/builds`, }); From 8b8f02445d443a79005203e3270cf871b2cff159 Mon Sep 17 00:00:00 2001 From: michelle0927 Date: Wed, 23 Apr 2025 14:03:04 -0400 Subject: [PATCH 3/4] updates --- .../apify/actions/run-actor/run-actor.mjs | 31 +++++++++++++++---- components/apify/common/constants.mjs | 9 ++++++ 2 files changed, 34 insertions(+), 6 deletions(-) diff --git a/components/apify/actions/run-actor/run-actor.mjs b/components/apify/actions/run-actor/run-actor.mjs index e8395b04b1945..447e0aa101fbe 100644 --- a/components/apify/actions/run-actor/run-actor.mjs +++ b/components/apify/actions/run-actor/run-actor.mjs @@ -1,6 +1,7 @@ /* eslint-disable no-unused-vars */ import apify from "../../apify.app.mjs"; import { parseObject } from "../../common/utils.mjs"; +import { EVENT_TYPES } from "../../common/constants.mjs"; export default { key: "apify-run-actor", @@ -56,11 +57,12 @@ export default { description: "Specifies the maximum cost of the Actor run. This parameter is useful for pay-per-event Actors, as it allows you to limit the amount charged to your subscription. You can access the maximum cost in your Actor by using the ACTOR_MAX_TOTAL_CHARGE_USD environment variable.", optional: true, }, - webhooks: { + webhook: { type: "string", - label: "Webhooks", - description: "Specifies optional webhooks associated with the Actor run, which can be used to receive a notification e.g. when the Actor finished or failed. The value is a Base64-encoded JSON array of objects defining the webhooks. For more information, see [Webhooks documentation](https://docs.apify.com/platform/integrations/webhooks).", + label: "Webhook", + description: "Specifies optional webhook associated with the Actor run, which can be used to receive a notification e.g. when the Actor finished or failed.", optional: true, + reloadProps: true, }, }, methods: { @@ -179,6 +181,14 @@ export default { }; } } + if (this.webhook) { + props.eventTypes = { + type: "string[]", + label: "Event Types", + description: "The types of events to send to the webhook", + options: EVENT_TYPES, + }; + } return props; }, async run({ $ }) { @@ -199,7 +209,8 @@ export default { maxItems, maxTotalChargeUsd, waitForFinish, - webhooks, + webhook, + eventTypes, ...data } = this; @@ -219,10 +230,18 @@ export default { maxItems, maxTotalChargeUsd, waitForFinish, - webhooks, + webhooks: webhook + ? { + eventTypes, + requestUrl: webhook, + } + : undefined, }, }); - $.export("$summary", `Successfully started actor run with ID: ${response.data.id}`); + const summary = this.runAsynchronously + ? `Successfully started actor run with ID: ${response.data.id}` + : `Successfully ran actor with ID: ${this.actorId}`; + $.export("$summary", `${summary}`); return response; }, }; diff --git a/components/apify/common/constants.mjs b/components/apify/common/constants.mjs index a16d51a32af1a..869e38c5bd157 100644 --- a/components/apify/common/constants.mjs +++ b/components/apify/common/constants.mjs @@ -1,2 +1,11 @@ export const ACTOR_ID = "aYG0l9s7dbB7j3gbS"; export const LIMIT = 100; + +export const EVENT_TYPES = [ + "ACTOR.RUN.CREATED", + "ACTOR.RUN.SUCCEEDED", + "ACTOR.RUN.FAILED", + "ACTOR.RUN.ABORTED", + "ACTOR.RUN.TIMED_OUT", + "ACTOR.RUN.RESURRECTED", +]; From a2c74af47134251f3bfad2843565f8d3b67e20d8 Mon Sep 17 00:00:00 2001 From: michelle0927 Date: Wed, 23 Apr 2025 14:11:30 -0400 Subject: [PATCH 4/4] updates --- components/apify/actions/run-actor/run-actor.mjs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/components/apify/actions/run-actor/run-actor.mjs b/components/apify/actions/run-actor/run-actor.mjs index 447e0aa101fbe..c4a7c6089c3ce 100644 --- a/components/apify/actions/run-actor/run-actor.mjs +++ b/components/apify/actions/run-actor/run-actor.mjs @@ -231,10 +231,12 @@ export default { maxTotalChargeUsd, waitForFinish, webhooks: webhook - ? { - eventTypes, - requestUrl: webhook, - } + ? btoa(JSON.stringify([ + { + eventTypes, + requestUrl: webhook, + }, + ])) : undefined, }, });