diff --git a/packages/components/credentials/ScrapeUnblockerApi.credential.ts b/packages/components/credentials/ScrapeUnblockerApi.credential.ts new file mode 100644 index 00000000000..10a4bd3c7aa --- /dev/null +++ b/packages/components/credentials/ScrapeUnblockerApi.credential.ts @@ -0,0 +1,32 @@ +import { INodeParams, INodeCredential } from '../src/Interface' + +class ScrapeUnblockerApiCredential implements INodeCredential { + label: string + name: string + version: number + description: string + inputs: INodeParams[] + + constructor() { + this.label = 'ScrapeUnblocker API' + this.name = 'scrapeUnblockerApi' + this.version = 1.0 + this.description = + 'Get your API key from the ScrapeUnblocker dashboard.' + this.inputs = [ + { + label: 'ScrapeUnblocker API Key', + name: 'scrapeUnblockerApiKey', + type: 'password' + }, + { + label: 'ScrapeUnblocker API URL', + name: 'scrapeUnblockerApiUrl', + type: 'string', + default: 'https://api.scrapeunblocker.com' + } + ] + } +} + +module.exports = { credClass: ScrapeUnblockerApiCredential } diff --git a/packages/components/nodes/documentloaders/ScrapeUnblocker/ScrapeUnblocker.ts b/packages/components/nodes/documentloaders/ScrapeUnblocker/ScrapeUnblocker.ts new file mode 100644 index 00000000000..34e757549eb --- /dev/null +++ b/packages/components/nodes/documentloaders/ScrapeUnblocker/ScrapeUnblocker.ts @@ -0,0 +1,317 @@ +import { TextSplitter } from '@langchain/classic/text_splitter' +import { Document } from '@langchain/core/documents' +import { BaseDocumentLoader } from '@langchain/classic/document_loaders/base' +import { INode, INodeData, INodeParams, ICommonObject, INodeOutputsValue } from '../../../src/Interface' +import { + getCredentialData, + getCredentialParam, + handleDocumentLoaderDocuments, + handleDocumentLoaderMetadata, + handleDocumentLoaderOutput +} from '../../../src/utils' +import { AxiosRequestConfig } from 'axios' +import { secureAxiosRequest } from '../../../src/httpSecurity' + +interface ScrapeUnblockerLoaderParameters { + apiKey?: string + apiUrl: string + mode: 'scrape' | 'search' + url?: string + keyword?: string + parsedData?: boolean + proxyCountry?: string + pagesToCheck?: number +} + +class ScrapeUnblockerLoader extends BaseDocumentLoader { + private apiKey: string + private apiUrl: string + private mode: 'scrape' | 'search' + private url?: string + private keyword?: string + private parsedData: boolean + private proxyCountry?: string + private pagesToCheck?: number + + constructor(loaderParams: ScrapeUnblockerLoaderParameters) { + super() + const { apiKey, apiUrl, mode, url, keyword, parsedData, proxyCountry, pagesToCheck } = loaderParams + if (!apiKey) { + throw new Error('ScrapeUnblocker API key not set. Please set it in the credential.') + } + + this.apiKey = apiKey + this.apiUrl = apiUrl.replace(/\/$/, '') + this.mode = mode + this.url = url + this.keyword = keyword + this.parsedData = parsedData ?? false + this.proxyCountry = proxyCountry + this.pagesToCheck = pagesToCheck + } + + private async request(path: string, params: Record): Promise { + // Empty values are dropped so the API applies its own defaults. + const query: Record = {} + for (const [key, value] of Object.entries(params)) { + if (value !== undefined && value !== null && value !== '' && value !== false) { + query[key] = value + } + } + + const config: AxiosRequestConfig = { + method: 'POST', + url: `${this.apiUrl}${path}`, + params: query, + headers: { 'X-ScrapeUnblocker-Key': this.apiKey } + } + + const response = await secureAxiosRequest(config) + return response.data + } + + private async loadPage(): Promise { + if (!this.url) { + throw new Error('ScrapeUnblocker: URL is required in Scrape mode.') + } + + const data = await this.request('/getPageSource', { + url: this.url, + parsed_data: this.parsedData, + proxy_country: this.proxyCountry + }) + + const pageContent = typeof data === 'string' ? data : JSON.stringify(data) + + return [ + new Document({ + pageContent, + metadata: { + source: this.url, + type: this.parsedData ? 'parsed_data' : 'html' + } + }) + ] + } + + private async loadSearch(): Promise { + if (!this.keyword) { + throw new Error('ScrapeUnblocker: Keyword is required in Search mode.') + } + + const data = await this.request('/serpApi', { + keyword: this.keyword, + proxy_country: this.proxyCountry, + pages_to_check: this.pagesToCheck + }) + + const organic = Array.isArray(data?.organic) ? data.organic.filter((result: any) => result) : [] + if (!organic.length) { + return [ + new Document({ + pageContent: JSON.stringify(data ?? {}), + metadata: { source: this.keyword, type: 'serp' } + }) + ] + } + + return organic.map( + (result: any) => + new Document({ + pageContent: result.description || result.title || '', + metadata: { + title: result.title, + source: result.url, + type: 'serp' + } + }) + ) + } + + public async load(): Promise { + if (this.mode === 'scrape') { + return this.loadPage() + } else if (this.mode === 'search') { + return this.loadSearch() + } + throw new Error(`Unrecognized mode '${this.mode}'. Expected one of 'scrape', 'search'.`) + } +} + +class ScrapeUnblocker_DocumentLoaders implements INode { + label: string + name: string + description: string + type: string + icon: string + version: number + category: string + baseClasses: string[] + inputs: INodeParams[] + credential: INodeParams + outputs: INodeOutputsValue[] + + constructor() { + this.label = 'ScrapeUnblocker' + this.name = 'scrapeUnblocker' + this.version = 1.0 + this.type = 'Document' + this.icon = 'scrapeunblocker.png' + this.category = 'Document Loaders' + this.description = 'Load data from pages protected by anti-bot systems using ScrapeUnblocker' + this.baseClasses = [this.type] + this.inputs = [ + { + label: 'Text Splitter', + name: 'textSplitter', + type: 'TextSplitter', + optional: true + }, + { + label: 'Mode', + name: 'mode', + type: 'options', + options: [ + { + label: 'Scrape', + name: 'scrape', + description: 'Fetch a single page, bypassing anti-bot protection' + }, + { + label: 'Search', + name: 'search', + description: 'Search Google and load the organic results' + } + ], + default: 'scrape' + }, + { + label: 'Web Page URL', + name: 'url', + type: 'string', + placeholder: 'https://www.scrapeunblocker.com', + show: { + mode: ['scrape'] + } + }, + { + label: 'Keyword', + name: 'keyword', + type: 'string', + placeholder: 'best web scraping api', + show: { + mode: ['search'] + } + }, + { + label: 'Parsed Data', + name: 'parsedData', + type: 'boolean', + description: 'Return AI-parsed structured JSON instead of raw HTML', + default: false, + optional: true, + additionalParams: true, + show: { + mode: ['scrape'] + } + }, + { + label: 'Pages To Check', + name: 'pagesToCheck', + type: 'number', + description: 'How many search result pages to load', + default: 1, + optional: true, + additionalParams: true, + show: { + mode: ['search'] + } + }, + { + label: 'Proxy Country', + name: 'proxyCountry', + type: 'string', + description: 'Two-letter country code for the exit IP, for geo-restricted content', + placeholder: 'us', + optional: true, + additionalParams: true + }, + { + label: 'Additional Metadata', + name: 'additional_metadata', + type: 'json', + description: 'Additional metadata to be added to the extracted documents', + optional: true, + additionalParams: true + }, + { + label: 'Omit Metadata Keys', + name: 'omitMetadataKeys', + type: 'string', + rows: 4, + description: + 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field', + placeholder: 'key1, key2, key3.nestedKey1', + optional: true, + additionalParams: true + } + ] + this.credential = { + label: 'Credential', + name: 'credential', + type: 'credential', + credentialNames: ['scrapeUnblockerApi'] + } + this.outputs = [ + { + label: 'Document', + name: 'document', + description: 'Array of document objects containing metadata and pageContent', + baseClasses: [...this.baseClasses, 'json'] + }, + { + label: 'Text', + name: 'text', + description: 'Concatenated string from pageContent of documents', + baseClasses: ['string', 'json'] + } + ] + } + + async init(nodeData: INodeData, _: string, options: ICommonObject): Promise { + const textSplitter = nodeData.inputs?.textSplitter as TextSplitter + const mode = nodeData.inputs?.mode as 'scrape' | 'search' + const url = nodeData.inputs?.url as string + const keyword = nodeData.inputs?.keyword as string + const parsedData = nodeData.inputs?.parsedData as boolean + const proxyCountry = nodeData.inputs?.proxyCountry as string + const pagesToCheck = nodeData.inputs?.pagesToCheck as number + const additionalMetadata = nodeData.inputs?.additional_metadata + const _omitMetadataKeys = nodeData.inputs?.omitMetadataKeys as string + const output = nodeData.outputs?.output as string + + const credentialData = await getCredentialData(nodeData.credential ?? '', options) + const apiKey = getCredentialParam('scrapeUnblockerApiKey', credentialData, nodeData) + const apiUrl = getCredentialParam('scrapeUnblockerApiUrl', credentialData, nodeData, 'https://api.scrapeunblocker.com') + + const input: ScrapeUnblockerLoaderParameters = { + apiKey, + apiUrl, + mode, + url, + keyword, + parsedData, + proxyCountry, + pagesToCheck + } + + const loader = new ScrapeUnblockerLoader(input) + + let docs = await handleDocumentLoaderDocuments(loader, textSplitter) + docs = handleDocumentLoaderMetadata(docs, _omitMetadataKeys, additionalMetadata) + + return handleDocumentLoaderOutput(docs, output) + } +} + +module.exports = { nodeClass: ScrapeUnblocker_DocumentLoaders } diff --git a/packages/components/nodes/documentloaders/ScrapeUnblocker/scrapeunblocker.png b/packages/components/nodes/documentloaders/ScrapeUnblocker/scrapeunblocker.png new file mode 100644 index 00000000000..fc893884ece Binary files /dev/null and b/packages/components/nodes/documentloaders/ScrapeUnblocker/scrapeunblocker.png differ