stef-openclaw-skills/skills/web-automation/scripts/flow.ts

#!/usr/bin/env npx tsx

import parseArgs from 'minimist';
import type { Page } from 'playwright-core';
import { launchBrowser } from './browse';

type Step =
  | { action: 'goto'; url: string }
  | { action: 'click'; selector?: string; text?: string }
  | { action: 'type'; selector?: string; text: string }
  | { action: 'press'; key: string; selector?: string }
  | { action: 'wait'; ms: number }
  | { action: 'screenshot'; path: string }
  | { action: 'extract'; selector: string; count?: number };

function normalizeKey(k: string): string {
  if (!k) return 'Enter';
  const lower = k.toLowerCase();
  if (lower === 'enter' || lower === 'return') return 'Enter';
  if (lower === 'tab') return 'Tab';
  if (lower === 'escape' || lower === 'esc') return 'Escape';
  return k;
}

function splitInstructions(instruction: string): string[] {
  return instruction
    .split(/\bthen\b|;/gi)
    .map((s) => s.trim())
    .filter(Boolean);
}

function parseInstruction(instruction: string): Step[] {
  const parts = splitInstructions(instruction);
  const steps: Step[] = [];

  for (const p of parts) {
    // go to https://...
    const goto = p.match(/^(?:go to|open|navigate to)\s+(https?:\/\/\S+)/i);
    if (goto) {
      steps.push({ action: 'goto', url: goto[1] });
      continue;
    }

    // click on "text" or click #selector
    const clickText = p.match(/^click(?: on)?\s+"([^"]+)"/i);
    if (clickText) {
      steps.push({ action: 'click', text: clickText[1] });
      continue;
    }
    const clickSelector = p.match(/^click(?: on)?\s+(#[\w-]+|\.[\w-]+|[a-z]+\[[^\]]+\])/i);
    if (clickSelector) {
      steps.push({ action: 'click', selector: clickSelector[1] });
      continue;
    }

    // type "text" [in selector]
    const typeInto = p.match(/^type\s+"([^"]+)"\s+in\s+(.+)$/i);
    if (typeInto) {
      steps.push({ action: 'type', text: typeInto[1], selector: typeInto[2].trim() });
      continue;
    }
    const typeOnly = p.match(/^type\s+"([^"]+)"$/i);
    if (typeOnly) {
      steps.push({ action: 'type', text: typeOnly[1] });
      continue;
    }

    // press enter [in selector]
    const pressIn = p.match(/^press\s+(\w+)\s+in\s+(.+)$/i);
    if (pressIn) {
      steps.push({ action: 'press', key: normalizeKey(pressIn[1]), selector: pressIn[2].trim() });
      continue;
    }
    const pressOnly = p.match(/^press\s+(\w+)$/i);
    if (pressOnly) {
      steps.push({ action: 'press', key: normalizeKey(pressOnly[1]) });
      continue;
    }

    // wait 2s / wait 500ms
    const waitS = p.match(/^wait\s+(\d+)\s*s(?:ec(?:onds?)?)?$/i);
    if (waitS) {
      steps.push({ action: 'wait', ms: parseInt(waitS[1], 10) * 1000 });
      continue;
    }
    const waitMs = p.match(/^wait\s+(\d+)\s*ms$/i);
    if (waitMs) {
      steps.push({ action: 'wait', ms: parseInt(waitMs[1], 10) });
      continue;
    }

    // screenshot path
    const shot = p.match(/^screenshot(?: to)?\s+(.+)$/i);
    if (shot) {
      steps.push({ action: 'screenshot', path: shot[1].trim() });
      continue;
    }

    throw new Error(`Could not parse step: "${p}"`);
  }

  return steps;
}

async function clickByText(page: Page, text: string) {
  const loc = page.getByRole('button', { name: text }).or(page.getByRole('link', { name: text })).or(page.getByText(text));
  await loc.first().click({ timeout: 15000 });
}

async function typeInBestTarget(page: Page, text: string, selector?: string) {
  if (selector) {
    await page.locator(selector).first().click({ timeout: 10000 });
    await page.locator(selector).first().fill(text);
    return;
  }
  const loc = page.locator('input[name="q"], input[type="search"], input[type="text"], textarea').first();
  await loc.click({ timeout: 10000 });
  await loc.fill(text);
}

async function pressOnTarget(page: Page, key: string, selector?: string) {
  if (selector) {
    await page.locator(selector).first().press(key);
    return;
  }
  await page.keyboard.press(key);
}

async function runSteps(page: Page, steps: Step[]) {
  for (const step of steps) {
    switch (step.action) {
      case 'goto':
        await page.goto(step.url, { waitUntil: 'domcontentloaded', timeout: 60000 });
        break;
      case 'click':
        if (step.selector) await page.locator(step.selector).first().click({ timeout: 15000 });
        else if (step.text) await clickByText(page, step.text);
        else throw new Error('click step missing selector/text');
        break;
      case 'type':
        await typeInBestTarget(page, step.text, step.selector);
        break;
      case 'press':
        await pressOnTarget(page, step.key, step.selector);
        break;
      case 'wait':
        await page.waitForTimeout(step.ms);
        break;
      case 'screenshot':
        await page.screenshot({ path: step.path, fullPage: true });
        break;
      case 'extract': {
        const items = await page.locator(step.selector).allTextContents();
        const out = items.slice(0, step.count ?? items.length).map((t) => t.trim()).filter(Boolean);
        console.log(JSON.stringify(out, null, 2));
        break;
      }
      default:
        throw new Error('Unknown step');
    }
  }
}

async function main() {
  const args = parseArgs(process.argv.slice(2), {
    string: ['instruction', 'steps'],
    boolean: ['headless', 'help'],
    default: { headless: true },
    alias: { i: 'instruction', s: 'steps', h: 'help' },
  });

  if (args.help || (!args.instruction && !args.steps)) {
    console.log(`
General Web Flow Runner (Camoufox)

Usage:
  npx tsx flow.ts --instruction "go to https://example.com then type \"hello\" then press enter"
  npx tsx flow.ts --steps '[{"action":"goto","url":"https://example.com"}]'

Supported natural steps:
  - go to/open/navigate to <url>
  - click on "Text"
  - click <css-selector>
  - type "text"
  - type "text" in <css-selector>
  - press <key>
  - press <key> in <css-selector>
  - wait <N>s | wait <N>ms
  - screenshot <path>
`);
    process.exit(args.help ? 0 : 1);
  }

  const browser = await launchBrowser({ headless: args.headless });
  const page = await browser.newPage();

  try {
    const steps: Step[] = args.steps ? JSON.parse(args.steps) : parseInstruction(args.instruction);
    await runSteps(page, steps);
    console.log('Flow complete. Final URL:', page.url());
  } finally {
    await browser.close();
  }
}

main().catch((e) => {
  console.error('Error:', e instanceof Error ? e.message : e);
  process.exit(1);
});