Langfuse CI Integration

Langfuse CI Integration

Overview

Set up CI/CD pipelines for Langfuse integrations with automated testing and validation.

Prerequisites

• GitHub repository with Actions enabled
• Langfuse test project API keys
• npm/pnpm project configured

Instructions

Step 1: Create GitHub Actions Workflow

Create .github/workflows/langfuse-integration.yml:

name: Langfuse Integration Tests

on:
  push:
    branches: [main, develop]
  pull_request:
    branches: [main]
  schedule:
    # Run daily to catch Langfuse API changes
    - cron: '0 6 * * *'

env:
  NODE_VERSION: '20'

jobs:
  test:
    runs-on: ubuntu-latest
    environment: test

    env:
      LANGFUSE_PUBLIC_KEY: ${{ secrets.LANGFUSE_PUBLIC_KEY_TEST }}
      LANGFUSE_SECRET_KEY: ${{ secrets.LANGFUSE_SECRET_KEY_TEST }}
      LANGFUSE_HOST: https://cloud.langfuse.com

    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Setup Node.js
        uses: actions/setup-node@v4
        with:
          node-version: ${{ env.NODE_VERSION }}
          cache: 'npm'

      - name: Install dependencies
        run: npm ci

      - name: Run unit tests
        run: npm test -- --coverage

      - name: Run Langfuse integration tests
        run: npm run test:integration
        timeout-minutes: 10

      - name: Upload coverage
        uses: codecov/codecov-action@v4
        with:
          token: ${{ secrets.CODECOV_TOKEN }}

  validate-traces:
    runs-on: ubuntu-latest
    needs: test
    if: github.event_name == 'push' && github.ref == 'refs/heads/main'

    env:
      LANGFUSE_PUBLIC_KEY: ${{ secrets.LANGFUSE_PUBLIC_KEY_TEST }}
      LANGFUSE_SECRET_KEY: ${{ secrets.LANGFUSE_SECRET_KEY_TEST }}

    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Setup Node.js
        uses: actions/setup-node@v4
        with:
          node-version: ${{ env.NODE_VERSION }}
          cache: 'npm'

      - name: Install dependencies
        run: npm ci

      - name: Validate Langfuse traces
        run: npm run validate:langfuse

Step 2: Configure Secrets

# Set test environment secrets
gh secret set LANGFUSE_PUBLIC_KEY_TEST --body "pk-lf-test-..."
gh secret set LANGFUSE_SECRET_KEY_TEST --body "sk-lf-test-..."

# Set production secrets (for deploy workflow)
gh secret set LANGFUSE_PUBLIC_KEY_PROD --body "pk-lf-prod-..."
gh secret set LANGFUSE_SECRET_KEY_PROD --body "sk-lf-prod-..."

Step 3: Create Integration Tests

// tests/langfuse.integration.test.ts
import { describe, it, expect, beforeAll, afterAll } from "vitest";
import { Langfuse } from "langfuse";

const SKIP_INTEGRATION = !process.env.LANGFUSE_PUBLIC_KEY;

describe.skipIf(SKIP_INTEGRATION)("Langfuse Integration", () => {
  let langfuse: Langfuse;

  beforeAll(() => {
    langfuse = new Langfuse({
      publicKey: process.env.LANGFUSE_PUBLIC_KEY!,
      secretKey: process.env.LANGFUSE_SECRET_KEY!,
      baseUrl: process.env.LANGFUSE_HOST,
    });
  });

  afterAll(async () => {
    await langfuse.shutdownAsync();
  });

  it("should connect to Langfuse", async () => {
    const trace = langfuse.trace({
      name: "ci-connection-test",
      metadata: {
        ci: true,
        commit: process.env.GITHUB_SHA,
        workflow: process.env.GITHUB_WORKFLOW,
      },
    });

    expect(trace.id).toBeDefined();
    await langfuse.flushAsync();
  });

  it("should create complete trace with generation", async () => {
    const trace = langfuse.trace({
      name: "ci-full-trace-test",
      input: { query: "test query" },
    });

    const span = trace.span({
      name: "process",
      input: { step: 1 },
    });

    const generation = span.generation({
      name: "llm-call",
      model: "gpt-4",
      input: [{ role: "user", content: "test" }],
    });

    generation.end({
      output: "test response",
      usage: { promptTokens: 5, completionTokens: 10 },
    });

    span.end({ output: { processed: true } });
    trace.update({ output: { success: true } });

    await langfuse.flushAsync();

    // Verify trace URL is valid
    const traceUrl = trace.getTraceUrl();
    expect(traceUrl).toContain("langfuse.com");
  });

  it("should handle errors gracefully", async () => {
    const trace = langfuse.trace({
      name: "ci-error-test",
    });

    trace.update({
      level: "ERROR",
      statusMessage: "Test error for CI validation",
    });

    await langfuse.flushAsync();
    expect(trace.id).toBeDefined();
  });
});

Step 4: Add Validation Script

// scripts/validate-langfuse.ts
import { Langfuse } from "langfuse";

async function validateLangfuseSetup() {
  console.log("=== Langfuse CI Validation ===\n");

  const langfuse = new Langfuse();

  const checks = [
    {
      name: "Create trace",
      test: async () => {
        const trace = langfuse.trace({
          name: "ci-validation",
          metadata: { validation: true },
        });
        return !!trace.id;
      },
    },
    {
      name: "Create span",
      test: async () => {
        const trace = langfuse.trace({ name: "span-test" });
        const span = trace.span({ name: "test-span" });
        span.end();
        return !!span.id;
      },
    },
    {
      name: "Create generation",
      test: async () => {
        const trace = langfuse.trace({ name: "gen-test" });
        const gen = trace.generation({
          name: "test-gen",
          model: "test-model",
        });
        gen.end({ output: "test" });
        return !!gen.id;
      },
    },
    {
      name: "Flush data",
      test: async () => {
        await langfuse.flushAsync();
        return true;
      },
    },
    {
      name: "Shutdown gracefully",
      test: async () => {
        await langfuse.shutdownAsync();
        return true;
      },
    },
  ];

  let passed = 0;
  let failed = 0;

  for (const { name, test } of checks) {
    try {
      const result = await test();
      if (result) {
        console.log(`✓ ${name}`);
        passed++;
      } else {
        console.log(`✗ ${name}`);
        failed++;
      }
    } catch (error) {
      console.log(`✗ ${name}: ${error}`);
      failed++;
    }
  }

  console.log(`\nResults: ${passed}/${checks.length} passed`);

  if (failed > 0) {
    process.exit(1);
  }
}

validateLangfuseSetup();

Step 5: Add package.json Scripts

{
  "scripts": {
    "test": "vitest run",
    "test:integration": "vitest run tests/*.integration.test.ts",
    "validate:langfuse": "tsx scripts/validate-langfuse.ts"
  }
}

Output

• Automated test pipeline
• PR checks configured
• Coverage reports uploaded
• Daily validation runs
• Trace validation on merge

PR Status Checks

# .github/branch-protection.yml (reference)
required_status_checks:
  strict: true
  contexts:
    - "test"
    - "validate-traces"

Error Handling

Examples

Matrix Testing (Multiple Node Versions)

jobs:
  test:
    strategy:
      matrix:
        node-version: [18, 20, 22]

    steps:
      - uses: actions/setup-node@v4
        with:
          node-version: ${{ matrix.node-version }}
      # ... rest of steps

Release Workflow with Langfuse Validation

name: Release

on:
  push:
    tags: ['v*']

jobs:
  release:
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v4

      - name: Setup Node.js
        uses: actions/setup-node@v4
        with:
          node-version: '20'
          registry-url: 'https://registry.npmjs.org'

      - name: Install dependencies
        run: npm ci

      - name: Validate Langfuse integration
        env:
          LANGFUSE_PUBLIC_KEY: ${{ secrets.LANGFUSE_PUBLIC_KEY_PROD }}
          LANGFUSE_SECRET_KEY: ${{ secrets.LANGFUSE_SECRET_KEY_PROD }}
        run: npm run validate:langfuse

      - name: Build
        run: npm run build

      - name: Publish
        run: npm publish
        env:
          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}

Resources

• GitHub Actions Documentation
• Vitest Documentation
• Langfuse SDK Testing

Next Steps

For deployment patterns, see langfuse-deploy-integration.