Enable GCP Claude to be used on the Next Chat platform

Note

This post translated By AI and the original content was written in Traditional Chinese.

While GCP has a more complete UI chatbot limited to its own Gemini series, it only supports API calls for other third-party LLM models.

To improve the Claude experience, it’s necessary to integrate a third-party Web UI for LLMs to enhance convenience. In this case, we choose Next Chat as an example, which can also be self-hosted in your own environment.

Scenario

• Use Vertex AI Claude API on GCP
• Deploy a js script on CloudFlare Worker to simulate the official Claude API
• Configure Next Chat to use the CloudFlare simulated Claude API

Caution

Using this still incurs GCP Claude Token fees. For pricing details, click here

Process and Description

1. Configure Vertex AI, enable it, and download the service account key
2. Deploy the CloudFlare worker script, filling in the information from the service account created in the previous step into its environment variables
3. Configure Next Chat to use the CloudFlare worker link
4. Configure VSCode Continue extension to use the CloudFlare worker link

Execution Steps

Configure Vertex AI and enable service account key

Enable API and create service account key

GCP CloudShell

# Set variables
PROJECT_ID="YOUR_PROJECT_ID"
SERVICE_ACCOUNT_NAME="vertex-ai-user-sa"
SERVICE_ACCOUNT_DISPLAY_NAME="Vertex AI User Service Account"
KEY_FILE="vertex-ai-user-key.json"

gcloud services enable aiplatform.googleapis.com

# Create the service account
gcloud iam service-accounts create $SERVICE_ACCOUNT_NAME \
    --display-name="$SERVICE_ACCOUNT_DISPLAY_NAME" \
    --project=$PROJECT_ID

# Get the full service account email
SERVICE_ACCOUNT_EMAIL=$(gcloud iam service-accounts list \
    --filter="displayName:$SERVICE_ACCOUNT_DISPLAY_NAME" \
    --format='value(email)' \
    --project=$PROJECT_ID)

# Grant Vertex AI User role to the service account
gcloud projects add-iam-policy-binding $PROJECT_ID \
    --member="serviceAccount:$SERVICE_ACCOUNT_EMAIL" \
    --role="roles/aiplatform.user"

# Create and download a JSON key for the service account
gcloud iam service-accounts keys create $KEY_FILE \
    --iam-account=$SERVICE_ACCOUNT_EMAIL \
    --project=$PROJECT_ID

cat $KEY_FILE

Deploy CloudFlare worker script

Use the vertexai-cf-workers js script to deploy to CloudFlare Worker

Add worker

Log in to your CloudFlare account and create a worker

Click to create a worker

Add a worker name and deploy

Deploy and Edit

You can click the link after deployment to check, and then click Edit Code when confirmed

Paste the worker.js code into the left code editing area and deploy

Click Save and Deploy

Configure Environment Variables

Click to enter the worker you just deployed

Click to add environment variables

The environment variables are mainly related information from the GCP service account. You can also click to encrypt if needed

• API_KEY: Customize an API key for calls
• CLIENT_EMAIL: Email from the GCP service account
• PRIVATE_KEY: Private key from the GCP service account
• PROJECT: Project ID in GCP After setting, click Deploy

Configure Next Chat

Click here to enter the Next Chat interface, then click Settings

1. Enable custom API Endpoint
2. Choose Anthropic model provider
3. Enter the CloudFlare worker URL
4. Enter the API_KEY created earlier
5. Select claude-3.5-sonnet-20240620(Anthropic) Finally, press the “X” in the upper right corner to close

Then proceed with testing

Configure VSCode Continue extension

Install Continue extension and configure the config.json models setting as below to use.

~/.continue/config.json

  "models": [
    {
      "model": "claude-3-5-sonnet-20240620",
      "contextLength": 200000,
      "title": "Claude 3 Sonnet",
      "apiKey": "YOUR_API_KEY",
      "apiBase": "https://YOUR_CLOUDFLARE_WORKER_URL/v1/",
      "provider": "anthropic"
    }
  ],

References

worker.js Code

Source: vertexai-cf-workers js script

worker.js

const MODELS = {
    "claude-3-opus": {
        vertexName: "claude-3-opus@20240229",
        region: "us-east5",
    },
    "claude-3-sonnet": {
        vertexName: "claude-3-sonnet@20240229",
        region: "us-central1",
    },
    "claude-3-haiku": {
        vertexName: "claude-3-haiku@20240307",
        region: "us-central1",
    },
    "claude-3-5-sonnet": {
        vertexName: "claude-3-5-sonnet@20240620",
        region: "us-east5",
    },
    "claude-3-opus-20240229": {
        vertexName: "claude-3-opus@20240229",
        region: "us-east5",
    },
    "claude-3-sonnet-20240229": {
        vertexName: "claude-3-sonnet@20240229",
        region: "us-central1",
    },
    "claude-3-haiku-20240307": {
        vertexName: "claude-3-haiku@20240307",
        region: "us-central1",
    },
    "claude-3-5-sonnet-20240620": {
        vertexName: "claude-3-5-sonnet@20240620",
        region: "us-east5",
    },
};

addEventListener("fetch", (event) => {
    event.respondWith(handleRequest(event.request));
});

async function handleRequest(request) {
    let headers = new Headers({
        "Access-Control-Allow-Origin": "*",
        "Access-Control-Allow-Headers": "*",
        "Access-Control-Allow-Methods": "GET, POST, OPTIONS",
    });
    if (request.method === "OPTIONS") {
        return new Response(null, { headers });
    } else if (request.method === "GET") {
        return createErrorResponse(405, "invalid_request_error", "GET method is not allowed");
    }

    const apiKey = request.headers.get("x-api-key");
    if (!API_KEY || API_KEY !== apiKey) {
        return createErrorResponse(401, "authentication_error", "invalid x-api-key");
    }

    const signedJWT = await createSignedJWT(CLIENT_EMAIL, PRIVATE_KEY)
    const [token, err] = await exchangeJwtForAccessToken(signedJWT)
    if (token === null) {
        console.log(`Invalid jwt token: ${err}`)
        return createErrorResponse(500, "api_error", "invalid authentication credentials");
    }

    try {
        const url = new URL(request.url);
        const normalizedPathname = url.pathname.replace(/^(\/)+/, '/');
        switch(normalizedPathname) {
            case "/v1/v1/messages":
            case "/v1/messages":
            case "/messages":
                return handleMessagesEndpoint(request, token);
            default:
                return createErrorResponse(404, "not_found_error", "Not Found");
        }
    } catch (error) {
        console.error(error);
        return createErrorResponse(500, "api_error", "An unexpected error occurred");
    }
}

async function handleMessagesEndpoint(request, api_token) {
    const anthropicVersion = request.headers.get('anthropic-version');
    if (anthropicVersion && anthropicVersion !== '2023-06-01') {
        return createErrorResponse(400, "invalid_request_error", "API version not supported");
    }

    let payload;
    try {
        payload = await request.json();
    } catch (err) {
        return createErrorResponse(400, "invalid_request_error", "The request body is not valid JSON.");
    }

    payload.anthropic_version = "vertex-2023-10-16";

    if (!payload.model) {
        return createErrorResponse(400, "invalid_request_error", "Missing model in the request payload.");
    } else if (!MODELS[payload.model]) {
        return createErrorResponse(400, "invalid_request_error", `Model \`${payload.model}\` not found.`);
    }

    const stream = payload.stream || false;
    const model = MODELS[payload.model];
    const url = `https://${model.region}-aiplatform.googleapis.com/v1/projects/${PROJECT}/locations/${model.region}/publishers/anthropic/models/${model.vertexName}:streamRawPredict`;
    delete payload.model;

    let response, contentType
    try {
        response = await fetch(url, {
            method: 'POST',
            headers: {
                'Content-Type': 'application/json',
                'Authorization': `Bearer ${api_token}`
            },
            body: JSON.stringify(payload)
        });
        contentType = response.headers.get("Content-Type") || "application/json";
    } catch (error) {
        return createErrorResponse(500, "api_error", "Server Error");
    }

    if (stream && contentType.startsWith('text/event-stream')) {
        if (!(response.body instanceof ReadableStream)) {
            return createErrorResponse(500, "api_error", "Server Error");
        }

        const encoder = new TextEncoder();
        const decoder = new TextDecoder("utf-8");
        let buffer = '';
        let { readable, writable } = new TransformStream({
            transform(chunk, controller) {
                let decoded = decoder.decode(chunk, { stream: true });
                buffer += decoded
                let eventList = buffer.split(/\r\n\r\n|\r\r|\n\n/g);
                if (eventList.length === 0) return;
                buffer = eventList.pop();

                for (let event of eventList) {
                    controller.enqueue(encoder.encode(`${event}\n\n`));
                }
            },
        });
        response.body.pipeTo(writable);
        return new Response(readable, {
            status: response.status,
            headers: {
                "Content-Type": response.headers.get("Content-Type") || "text/event-stream",
                "Access-Control-Allow-Origin": "*",
            },
        });
    } else {
        try {
            let data = await response.text();
            return new Response(data, {
                status: response.status,
                headers: {
                    "Content-Type": contentType,
                    "Access-Control-Allow-Origin": "*",
                },
            });
        } catch (error) {
            return createErrorResponse(500, "api_error", "Server Error");
        }
    }
}

function createErrorResponse(status, errorType, message) {
    const errorObject = { type: "error", error: { type: errorType, message: message } };
    return new Response(JSON.stringify(errorObject), {
        status: status,
        headers: {
            "Content-Type": "application/json",
            "Access-Control-Allow-Origin": "*",
        },
    });
}

async function createSignedJWT(email, pkey) {
    pkey = pkey.replace(/-----BEGIN PRIVATE KEY-----|-----END PRIVATE KEY-----|\r|\n|\\n/g, "");
    let cryptoKey = await crypto.subtle.importKey(
        "pkcs8",
        str2ab(atob(pkey)),
        {
            name: "RSASSA-PKCS1-v1_5",
            hash: { name: "SHA-256" },
        },
        false,
        ["sign"]
    );

    const authUrl = "https://www.googleapis.com/oauth2/v4/token";
    const issued = Math.floor(Date.now() / 1000);
    const expires = issued + 600;

    const header = {
        alg: "RS256",
        typ: "JWT",
    };

    const payload = {
        iss: email,
        aud: authUrl,
        iat: issued,
        exp: expires,
        scope: "https://www.googleapis.com/auth/cloud-platform",
    };

    const encodedHeader = urlSafeBase64Encode(JSON.stringify(header));
    const encodedPayload = urlSafeBase64Encode(JSON.stringify(payload));

    const unsignedToken = `${encodedHeader}.${encodedPayload}`;

    const signature = await crypto.subtle.sign(
        "RSASSA-PKCS1-v1_5",
        cryptoKey,
        str2ab(unsignedToken)
    );

    const encodedSignature = urlSafeBase64Encode(signature);
    return `${unsignedToken}.${encodedSignature}`;
}

async function exchangeJwtForAccessToken(signed_jwt) {
    const auth_url = "https://www.googleapis.com/oauth2/v4/token";
    const params = {
        grant_type: "urn:ietf:params:oauth:grant-type:jwt-bearer",
        assertion: signed_jwt,
    };

    const r = await fetch(auth_url, {
        method: "POST",
        headers: { "Content-Type": "application/x-www-form-urlencoded" },
        body: Object.entries(params)
            .map(([k, v]) => k + "=" + v)
            .join("&"),
    }).then((res) => res.json());

    if (r.access_token) {
        return [r.access_token, ""];
    }

    return [null, JSON.stringify(r)];
}

function str2ab(str) {
    const buffer = new ArrayBuffer(str.length);
    let bufferView = new Uint8Array(buffer);
    for (let i = 0; i < str.length; i++) {
        bufferView[i] = str.charCodeAt(i);
    }
    return buffer;
}

function urlSafeBase64Encode(data) {
    let base64 = typeof data === "string" ? btoa(encodeURIComponent(data).replace(/%([0-9A-F]{2})/g, (match, p1) => String.fromCharCode(parseInt("0x" + p1)))) : btoa(String.fromCharCode(...new Uint8Array(data)));
    return base64.replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, "");
}