Connect Github repository files and Dremio in our serverless environment

class GithubSourceRepositoryFiles {
    async init() {
        // TODO: Create your github credential
        // More info at https://yepcode.io/docs/integrations/github/#credential-configuration
        this.github = yepcode.integration.github("your-github-credential-name");
        // TODO: Customize the repo name and owner
        this.repositoryOwner = "your-github-username";
        this.repositoryName = "your-repository-name";
    }

    async fetch(publish, done) {
        // TODO: Customize the path you want to obtain all files in it and its subdirectories (if has)
        // It can also be a file path to only obtain one file
        // If you pass undefined you will obtain the entire repository content
        await this._publishPathContent("/path-in-repo", publish);
        done();
    }

    _isFileContent = isString;
    _isDirectoryInfo = isObject;

    _publishPathContent = async (path, publishCallback) => {
        // More info of this request at: https://docs.github.com/en/rest/repos/contents#get-repository-content
        const response = await this.github.rest.repos.getContent({
            mediaType: {
                format: "raw",
            },
            owner: this.repositoryOwner,
            repo: this.repositoryName,
            path: path,
        });
        const {
            data
        } = response;

        if (this._isFileContent(data)) {
            // TODO: Customize the item to publish for each file in the repository
            publishCallback({
                path,
                content: data
            });
            return;
        }

        if (this._isDirectoryInfo(data)) {
            for (const directoryEntry of data) {
                await this._publishPathContent(directoryEntry.path, publishCallback);
            }
            return;
        }

        console.error("Unexpected data type: ", response.headers["content-type"]);
    };

    async close() {}
}

class HttpTargetDremioInsert {
    async init() {
        // TODO: Create your http credential with dremio information:
        // More info at https://yepcode.io/docs/integrations/http/#credential-configuration
        // Official docs: https://docs.dremio.com/current/reference/api/
        this.httpClient = yepcode.integration.http("your-dremio-credential-name");
    }
    async consume(item) {
        const projectId = "<your-dremio-project-id>";
        // TODO: Map your item to row and customize INSERT statement
        // More info at https://docs.dremio.com/cloud/reference/sql/commands/insert/
        const query = `INSERT INTO your_source_path.your_table VALUES (${item.id}, ${item.name})`;

        // Use the SQL API to submit queries. The response contains the ID for the job associated with the SQL query
        // Official docs: https://docs.dremio.com/cloud/reference/api/sql
        const {
            data: job
        } = await this.httpClient.post(`${projectId}/sql`, {
            sql: query,
        });

        // Use the Job API to check the job status. Jobs final status are COMPLETED, FAILED and CANCELED
        // Official docs: https://docs.dremio.com/cloud/reference/api/job/
        const finalJobStates = ["COMPLETED", "FAILED", "CANCELED"];
        let jobInfoResponse = null;
        do {
            jobInfoResponse = await this.httpClient.get(
                `${this.projectId}/job/${job.id}`
            );
        } while (!finalJobStates.includes(jobInfoResponse.data.jobState));

        const {
            data: jobInfo
        } = jobInfoResponse;

        if (jobInfo.jobState === "COMPLETED") {
            console.log("Success!!");
        } else if (jobInfo.jobState === "FAILED") {
            console.log(`Uups! Something was wrong: ${jobInfo.errorMessage}`);
        } else {
            console.log(`The job's been cancelled: ${jobInfo.cancellationReason}`);
        }
    }

    async close() {}
}