Connect AWS Redshift and Amazon S3 NDJSON file in our serverless environment

class AwsRedshiftSourceSelect {
    async init() {
        // TODO: Create your aws-redshift credential
        // More info at https://yepcode.io/docs/integrations/aws-redshift/#credential-configuration
        this.awsRedshift = yepcode.integration.awsRedshift(
            "your-aws-redshift-credential-name"
        );

        // TODO: Customize the command to be executed
        // More info at https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/clients/client-redshift-data/interfaces/executestatementcommandinput.html
        const {
            Id
        } = await this.awsRedshift.send(
            new ExecuteStatementCommand({
                ClusterIdentifier: "your-cluster-identifier",
                Database: "your-database-name",
                DbUser: "your-database-user",
                Sql: "SELECT * FROM your-table-name;",
            })
        );
        this.statementId = Id;
        await this._awaitStatementToFinish();
    }

    async fetch(publish, done) {
        let nextToken;
        // if result is returned paginated, iterate to publish entire statement result
        do {
            const result = await this.awsRedshift.send(
                new GetStatementResultCommand({
                    Id: this.statementId,
                    NextToken: nextToken,
                })
            );
            nextToken = result.NextToken;
            for (const record of result.Records) {
                // Each record come as a list of record cells
                // More info at https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/clients/client-redshift-data/interfaces/getstatementresultcommandoutput.html

                // TODO: Customize record columns mapping to the item that will be published
                const item = {
                    id: record.id,
                    ...record
                };

                await publish(item);
            }
        } while (nextToken);
        done();
    }

    async close() {}

    _awaitStatementToFinish() {
        return new Promise((resolve, reject) => {
            const describeStatementCommand = new DescribeStatementCommand({
                Id: this.statementId,
            });

            // Configure interval to check statement status
            const intervalId = setInterval(async () => {
                const result = await this.awsRedshift.send(describeStatementCommand);
                if (result.Status === "FINISHED") {
                    clearInterval(intervalId);
                    resolve();
                    return;
                }

                if (result.Status === "FAILED" || result.Status === "ABORTED") {
                    clearInterval(intervalId);
                    reject();
                    return;
                }
            }, 1000);
        });
    }
}

class AwsS3TargetUploadNdjson {
    async init() {
        // TODO: Create your aws-s3 credential
        // More info at https://yepcode.io/docs/integrations/aws-s3/#credential-configuration
        const awsS3 = yepcode.integration.awsS3("your-aws-s3-credential-name");
        // Transforms the items into a ndjson format
        this.stringifier = ndjson.stringify();
        const targetStream = new PassThrough();
        this.stringifier.pipe(targetStream);
        // TODO: customize the Upload content
        // More info at: https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/interfaces/_aws_sdk_lib_storage.options-1.html
        const upload = new Upload({
            client: awsS3,
            params: {
                Bucket: "your-bucket-name",
                Key: "your-file-name.ndjson",
                Body: targetStream,
            },
        });
        upload.on("httpUploadProgress", (progress) => {
            console.log(`Upload progress`, progress);
        });
        this.uploadPromise = upload.done();
    }

    async consume(item) {
        this.stringifier.write(item);
    }

    async close() {
        try {
            this.stringifier.end();
        } catch (error) {
            console.error(`Error ending stringifier`, error);
        }
        try {
            await this.uploadPromise;
        } catch (error) {
            console.error(`Error ending upload`, error);
        }
    }
}