Add web-automation skill variants and inline prerequisite checks
This commit is contained in:
575
skills/web-automation/claude-code/scripts/auth.ts
Normal file
575
skills/web-automation/claude-code/scripts/auth.ts
Normal file
@@ -0,0 +1,575 @@
|
||||
#!/usr/bin/env npx tsx
|
||||
|
||||
/**
|
||||
* Authentication handler for web automation
|
||||
* Supports generic form login and Microsoft SSO (MSAL)
|
||||
*
|
||||
* Usage:
|
||||
* npx tsx auth.ts --url "https://example.com/login" --type form
|
||||
* npx tsx auth.ts --url "https://example.com" --type msal
|
||||
* npx tsx auth.ts --url "https://example.com" --type auto
|
||||
*/
|
||||
|
||||
import { getPage, launchBrowser } from './browse.js';
|
||||
import parseArgs from 'minimist';
|
||||
import type { Page, BrowserContext } from 'playwright-core';
|
||||
import { createInterface } from 'readline';
|
||||
|
||||
// Types
|
||||
type AuthType = 'auto' | 'form' | 'msal';
|
||||
|
||||
interface AuthOptions {
|
||||
url: string;
|
||||
authType: AuthType;
|
||||
credentials?: {
|
||||
username: string;
|
||||
password: string;
|
||||
};
|
||||
headless?: boolean;
|
||||
timeout?: number;
|
||||
}
|
||||
|
||||
interface AuthResult {
|
||||
success: boolean;
|
||||
finalUrl: string;
|
||||
authType: AuthType;
|
||||
message: string;
|
||||
}
|
||||
|
||||
// Get credentials from environment or options
|
||||
function getCredentials(options?: {
|
||||
username?: string;
|
||||
password?: string;
|
||||
}): { username: string; password: string } | null {
|
||||
const username = options?.username || process.env.CAMOUFOX_USERNAME;
|
||||
const password = options?.password || process.env.CAMOUFOX_PASSWORD;
|
||||
|
||||
if (!username || !password) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return { username, password };
|
||||
}
|
||||
|
||||
// Prompt user for input (for MFA or credentials)
|
||||
async function promptUser(question: string, hidden = false): Promise<string> {
|
||||
const rl = createInterface({
|
||||
input: process.stdin,
|
||||
output: process.stdout,
|
||||
});
|
||||
|
||||
return new Promise((resolve) => {
|
||||
if (hidden) {
|
||||
process.stdout.write(question);
|
||||
// Note: This is a simple implementation. For production, use a proper hidden input library
|
||||
}
|
||||
rl.question(question, (answer) => {
|
||||
rl.close();
|
||||
resolve(answer);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
// Detect authentication type from page
|
||||
async function detectAuthType(page: Page): Promise<AuthType> {
|
||||
const url = page.url();
|
||||
|
||||
// Check for Microsoft login
|
||||
if (
|
||||
url.includes('login.microsoftonline.com') ||
|
||||
url.includes('login.live.com') ||
|
||||
url.includes('login.windows.net')
|
||||
) {
|
||||
return 'msal';
|
||||
}
|
||||
|
||||
// Check for common form login patterns
|
||||
const hasLoginForm = await page.evaluate(() => {
|
||||
const passwordField = document.querySelector(
|
||||
'input[type="password"], input[name*="password"], input[id*="password"]'
|
||||
);
|
||||
const usernameField = document.querySelector(
|
||||
'input[type="email"], input[type="text"][name*="user"], input[type="text"][name*="email"], input[id*="user"], input[id*="email"]'
|
||||
);
|
||||
return !!(passwordField && usernameField);
|
||||
});
|
||||
|
||||
if (hasLoginForm) {
|
||||
return 'form';
|
||||
}
|
||||
|
||||
return 'auto';
|
||||
}
|
||||
|
||||
// Handle generic form login
|
||||
async function handleFormLogin(
|
||||
page: Page,
|
||||
credentials: { username: string; password: string },
|
||||
timeout: number
|
||||
): Promise<boolean> {
|
||||
console.log('Attempting form login...');
|
||||
|
||||
// Find and fill username/email field
|
||||
const usernameSelectors = [
|
||||
'input[type="email"]',
|
||||
'input[name*="user" i]',
|
||||
'input[name*="email" i]',
|
||||
'input[id*="user" i]',
|
||||
'input[id*="email" i]',
|
||||
'input[autocomplete="username"]',
|
||||
'input[type="text"]:first-of-type',
|
||||
];
|
||||
|
||||
let usernameField = null;
|
||||
for (const selector of usernameSelectors) {
|
||||
usernameField = await page.$(selector);
|
||||
if (usernameField) break;
|
||||
}
|
||||
|
||||
if (!usernameField) {
|
||||
console.error('Could not find username/email field');
|
||||
return false;
|
||||
}
|
||||
|
||||
await usernameField.fill(credentials.username);
|
||||
console.log('Filled username field');
|
||||
|
||||
// Find and fill password field
|
||||
const passwordSelectors = [
|
||||
'input[type="password"]',
|
||||
'input[name*="password" i]',
|
||||
'input[id*="password" i]',
|
||||
'input[autocomplete="current-password"]',
|
||||
];
|
||||
|
||||
let passwordField = null;
|
||||
for (const selector of passwordSelectors) {
|
||||
passwordField = await page.$(selector);
|
||||
if (passwordField) break;
|
||||
}
|
||||
|
||||
if (!passwordField) {
|
||||
console.error('Could not find password field');
|
||||
return false;
|
||||
}
|
||||
|
||||
await passwordField.fill(credentials.password);
|
||||
console.log('Filled password field');
|
||||
|
||||
// Check for "Remember me" checkbox and check it
|
||||
const rememberCheckbox = await page.$(
|
||||
'input[type="checkbox"][name*="remember" i], input[type="checkbox"][id*="remember" i]'
|
||||
);
|
||||
if (rememberCheckbox) {
|
||||
await rememberCheckbox.check();
|
||||
console.log('Checked "Remember me" checkbox');
|
||||
}
|
||||
|
||||
// Find and click submit button
|
||||
const submitSelectors = [
|
||||
'button[type="submit"]',
|
||||
'input[type="submit"]',
|
||||
'button:has-text("Sign in")',
|
||||
'button:has-text("Log in")',
|
||||
'button:has-text("Login")',
|
||||
'button:has-text("Submit")',
|
||||
'[role="button"]:has-text("Sign in")',
|
||||
];
|
||||
|
||||
let submitButton = null;
|
||||
for (const selector of submitSelectors) {
|
||||
submitButton = await page.$(selector);
|
||||
if (submitButton) break;
|
||||
}
|
||||
|
||||
if (!submitButton) {
|
||||
// Try pressing Enter as fallback
|
||||
await passwordField.press('Enter');
|
||||
} else {
|
||||
await submitButton.click();
|
||||
}
|
||||
|
||||
console.log('Submitted login form');
|
||||
|
||||
// Wait for navigation or error
|
||||
try {
|
||||
await page.waitForNavigation({ timeout, waitUntil: 'domcontentloaded' });
|
||||
return true;
|
||||
} catch {
|
||||
// Check if we're still on login page with error
|
||||
const errorMessages = await page.$$eval(
|
||||
'.error, .alert-danger, [role="alert"], .login-error',
|
||||
(els) => els.map((el) => el.textContent?.trim()).filter(Boolean)
|
||||
);
|
||||
|
||||
if (errorMessages.length > 0) {
|
||||
console.error('Login error:', errorMessages.join(', '));
|
||||
return false;
|
||||
}
|
||||
|
||||
return true; // Might have succeeded without navigation
|
||||
}
|
||||
}
|
||||
|
||||
// Handle Microsoft SSO login
|
||||
async function handleMsalLogin(
|
||||
page: Page,
|
||||
credentials: { username: string; password: string },
|
||||
timeout: number
|
||||
): Promise<boolean> {
|
||||
console.log('Attempting Microsoft SSO login...');
|
||||
|
||||
const currentUrl = page.url();
|
||||
|
||||
// If not already on Microsoft login, wait for redirect
|
||||
if (!currentUrl.includes('login.microsoftonline.com')) {
|
||||
try {
|
||||
await page.waitForURL('**/login.microsoftonline.com/**', { timeout: 10000 });
|
||||
} catch {
|
||||
console.log('Not redirected to Microsoft login');
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// Wait for email input
|
||||
const emailInput = await page.waitForSelector(
|
||||
'input[type="email"], input[name="loginfmt"]',
|
||||
{ timeout }
|
||||
);
|
||||
|
||||
if (!emailInput) {
|
||||
console.error('Could not find email input on Microsoft login');
|
||||
return false;
|
||||
}
|
||||
|
||||
// Fill email and submit
|
||||
await emailInput.fill(credentials.username);
|
||||
console.log('Filled email field');
|
||||
|
||||
const nextButton = await page.$('input[type="submit"], button[type="submit"]');
|
||||
if (nextButton) {
|
||||
await nextButton.click();
|
||||
} else {
|
||||
await emailInput.press('Enter');
|
||||
}
|
||||
|
||||
// Wait for password page
|
||||
try {
|
||||
await page.waitForSelector(
|
||||
'input[type="password"], input[name="passwd"]',
|
||||
{ timeout }
|
||||
);
|
||||
} catch {
|
||||
// Might be using passwordless auth or different flow
|
||||
console.log('Password field not found - might be using different auth flow');
|
||||
return false;
|
||||
}
|
||||
|
||||
// Fill password
|
||||
const passwordInput = await page.$('input[type="password"], input[name="passwd"]');
|
||||
if (!passwordInput) {
|
||||
console.error('Could not find password input');
|
||||
return false;
|
||||
}
|
||||
|
||||
await passwordInput.fill(credentials.password);
|
||||
console.log('Filled password field');
|
||||
|
||||
// Submit
|
||||
const signInButton = await page.$('input[type="submit"], button[type="submit"]');
|
||||
if (signInButton) {
|
||||
await signInButton.click();
|
||||
} else {
|
||||
await passwordInput.press('Enter');
|
||||
}
|
||||
|
||||
// Handle "Stay signed in?" prompt
|
||||
try {
|
||||
const staySignedInButton = await page.waitForSelector(
|
||||
'input[value="Yes"], button:has-text("Yes")',
|
||||
{ timeout: 5000 }
|
||||
);
|
||||
if (staySignedInButton) {
|
||||
await staySignedInButton.click();
|
||||
console.log('Clicked "Stay signed in" button');
|
||||
}
|
||||
} catch {
|
||||
// Prompt might not appear
|
||||
}
|
||||
|
||||
// Check for Conditional Access Policy error
|
||||
const caError = await page.$('text=Conditional Access policy');
|
||||
if (caError) {
|
||||
console.error('Blocked by Conditional Access Policy');
|
||||
// Take screenshot for debugging
|
||||
await page.screenshot({ path: 'ca-policy-error.png' });
|
||||
console.log('Screenshot saved: ca-policy-error.png');
|
||||
return false;
|
||||
}
|
||||
|
||||
// Wait for redirect away from Microsoft login
|
||||
try {
|
||||
await page.waitForURL(
|
||||
(url) => !url.href.includes('login.microsoftonline.com'),
|
||||
{ timeout }
|
||||
);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// Check if user is already authenticated
|
||||
async function isAuthenticated(page: Page, targetUrl: string): Promise<boolean> {
|
||||
const currentUrl = page.url();
|
||||
|
||||
// If we're on the target URL (not a login page), we're likely authenticated
|
||||
if (currentUrl.startsWith(targetUrl)) {
|
||||
// Check for common login page indicators
|
||||
const isLoginPage = await page.evaluate(() => {
|
||||
const loginIndicators = [
|
||||
'input[type="password"]',
|
||||
'form[action*="login"]',
|
||||
'form[action*="signin"]',
|
||||
'.login-form',
|
||||
'#login',
|
||||
];
|
||||
return loginIndicators.some((sel) => document.querySelector(sel) !== null);
|
||||
});
|
||||
|
||||
return !isLoginPage;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
// Main authentication function
|
||||
export async function authenticate(options: AuthOptions): Promise<AuthResult> {
|
||||
const browser = await launchBrowser({ headless: options.headless ?? true });
|
||||
const page = await browser.newPage();
|
||||
const timeout = options.timeout ?? 30000;
|
||||
|
||||
try {
|
||||
// Navigate to URL
|
||||
console.log(`Navigating to: ${options.url}`);
|
||||
await page.goto(options.url, { timeout: 60000, waitUntil: 'domcontentloaded' });
|
||||
|
||||
// Check if already authenticated
|
||||
if (await isAuthenticated(page, options.url)) {
|
||||
return {
|
||||
success: true,
|
||||
finalUrl: page.url(),
|
||||
authType: 'auto',
|
||||
message: 'Already authenticated (session persisted from profile)',
|
||||
};
|
||||
}
|
||||
|
||||
// Get credentials
|
||||
const credentials = options.credentials
|
||||
? options.credentials
|
||||
: getCredentials();
|
||||
|
||||
if (!credentials) {
|
||||
// No credentials - open interactive browser
|
||||
console.log('\nNo credentials provided. Opening browser for manual login...');
|
||||
console.log('Please complete the login process manually.');
|
||||
console.log('The session will be saved to your profile.');
|
||||
|
||||
// Switch to headed mode for manual login
|
||||
await browser.close();
|
||||
const interactiveBrowser = await launchBrowser({ headless: false });
|
||||
const interactivePage = await interactiveBrowser.newPage();
|
||||
await interactivePage.goto(options.url);
|
||||
|
||||
await promptUser('\nPress Enter when you have completed login...');
|
||||
|
||||
const finalUrl = interactivePage.url();
|
||||
await interactiveBrowser.close();
|
||||
|
||||
return {
|
||||
success: true,
|
||||
finalUrl,
|
||||
authType: 'auto',
|
||||
message: 'Manual login completed - session saved to profile',
|
||||
};
|
||||
}
|
||||
|
||||
// Detect auth type if auto
|
||||
let authType = options.authType;
|
||||
if (authType === 'auto') {
|
||||
authType = await detectAuthType(page);
|
||||
console.log(`Detected auth type: ${authType}`);
|
||||
}
|
||||
|
||||
// Handle authentication based on type
|
||||
let success = false;
|
||||
switch (authType) {
|
||||
case 'msal':
|
||||
success = await handleMsalLogin(page, credentials, timeout);
|
||||
break;
|
||||
case 'form':
|
||||
default:
|
||||
success = await handleFormLogin(page, credentials, timeout);
|
||||
break;
|
||||
}
|
||||
|
||||
const finalUrl = page.url();
|
||||
|
||||
return {
|
||||
success,
|
||||
finalUrl,
|
||||
authType,
|
||||
message: success
|
||||
? `Authentication successful - session saved to profile`
|
||||
: 'Authentication failed',
|
||||
};
|
||||
} finally {
|
||||
await browser.close();
|
||||
}
|
||||
}
|
||||
|
||||
// Navigate to authenticated page (handles auth if needed)
|
||||
export async function navigateAuthenticated(
|
||||
url: string,
|
||||
options?: {
|
||||
credentials?: { username: string; password: string };
|
||||
headless?: boolean;
|
||||
}
|
||||
): Promise<{ page: Page; browser: BrowserContext }> {
|
||||
const { page, browser } = await getPage({ headless: options?.headless ?? true });
|
||||
|
||||
await page.goto(url, { timeout: 60000, waitUntil: 'domcontentloaded' });
|
||||
|
||||
// Check if we need to authenticate
|
||||
if (!(await isAuthenticated(page, url))) {
|
||||
console.log('Session expired or not authenticated. Attempting login...');
|
||||
|
||||
// Get credentials
|
||||
const credentials = options?.credentials ?? getCredentials();
|
||||
|
||||
if (!credentials) {
|
||||
throw new Error(
|
||||
'Authentication required but no credentials provided. ' +
|
||||
'Set CAMOUFOX_USERNAME and CAMOUFOX_PASSWORD environment variables.'
|
||||
);
|
||||
}
|
||||
|
||||
// Detect and handle auth
|
||||
const authType = await detectAuthType(page);
|
||||
|
||||
let success = false;
|
||||
if (authType === 'msal') {
|
||||
success = await handleMsalLogin(page, credentials, 30000);
|
||||
} else {
|
||||
success = await handleFormLogin(page, credentials, 30000);
|
||||
}
|
||||
|
||||
if (!success) {
|
||||
await browser.close();
|
||||
throw new Error('Authentication failed');
|
||||
}
|
||||
|
||||
// Navigate back to original URL if we were redirected
|
||||
if (!page.url().startsWith(url)) {
|
||||
await page.goto(url, { timeout: 60000, waitUntil: 'domcontentloaded' });
|
||||
}
|
||||
}
|
||||
|
||||
return { page, browser };
|
||||
}
|
||||
|
||||
// CLI entry point
|
||||
async function main() {
|
||||
const args = parseArgs(process.argv.slice(2), {
|
||||
string: ['url', 'type', 'username', 'password'],
|
||||
boolean: ['headless', 'help'],
|
||||
default: {
|
||||
type: 'auto',
|
||||
headless: false, // Default to headed for auth so user can see/interact
|
||||
},
|
||||
alias: {
|
||||
u: 'url',
|
||||
t: 'type',
|
||||
h: 'help',
|
||||
},
|
||||
});
|
||||
|
||||
if (args.help || !args.url) {
|
||||
console.log(`
|
||||
Web Authentication Handler
|
||||
|
||||
Usage:
|
||||
npx tsx auth.ts --url <url> [options]
|
||||
|
||||
Options:
|
||||
-u, --url <url> URL to authenticate (required)
|
||||
-t, --type <type> Auth type: auto, form, or msal (default: auto)
|
||||
--username <user> Username/email (or set CAMOUFOX_USERNAME env var)
|
||||
--password <pass> Password (or set CAMOUFOX_PASSWORD env var)
|
||||
--headless <bool> Run in headless mode (default: false for auth)
|
||||
-h, --help Show this help message
|
||||
|
||||
Auth Types:
|
||||
auto Auto-detect authentication type
|
||||
form Generic username/password form
|
||||
msal Microsoft SSO (login.microsoftonline.com)
|
||||
|
||||
Environment Variables:
|
||||
CAMOUFOX_USERNAME Default username/email for authentication
|
||||
CAMOUFOX_PASSWORD Default password for authentication
|
||||
|
||||
Examples:
|
||||
# Interactive login (no credentials, opens browser)
|
||||
npx tsx auth.ts --url "https://example.com/login"
|
||||
|
||||
# Form login with credentials
|
||||
npx tsx auth.ts --url "https://example.com/login" --type form \\
|
||||
--username "user@example.com" --password "secret"
|
||||
|
||||
# Microsoft SSO login
|
||||
CAMOUFOX_USERNAME=user@company.com CAMOUFOX_PASSWORD=secret \\
|
||||
npx tsx auth.ts --url "https://internal.company.com" --type msal
|
||||
|
||||
Notes:
|
||||
- Session is saved to ~/.camoufox-profile/ for persistence
|
||||
- After successful auth, subsequent browses will be authenticated
|
||||
- Use --headless false if you need to handle MFA manually
|
||||
`);
|
||||
process.exit(args.help ? 0 : 1);
|
||||
}
|
||||
|
||||
const authType = args.type as AuthType;
|
||||
if (!['auto', 'form', 'msal'].includes(authType)) {
|
||||
console.error(`Invalid auth type: ${authType}. Must be auto, form, or msal.`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await authenticate({
|
||||
url: args.url,
|
||||
authType,
|
||||
credentials:
|
||||
args.username && args.password
|
||||
? { username: args.username, password: args.password }
|
||||
: undefined,
|
||||
headless: args.headless,
|
||||
});
|
||||
|
||||
console.log(`\nAuthentication result:`);
|
||||
console.log(` Success: ${result.success}`);
|
||||
console.log(` Auth type: ${result.authType}`);
|
||||
console.log(` Final URL: ${result.finalUrl}`);
|
||||
console.log(` Message: ${result.message}`);
|
||||
|
||||
process.exit(result.success ? 0 : 1);
|
||||
} catch (error) {
|
||||
console.error('Error:', error instanceof Error ? error.message : error);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
// Run if executed directly
|
||||
const isMainModule = process.argv[1]?.includes('auth.ts');
|
||||
if (isMainModule) {
|
||||
main();
|
||||
}
|
||||
195
skills/web-automation/claude-code/scripts/browse.ts
Normal file
195
skills/web-automation/claude-code/scripts/browse.ts
Normal file
@@ -0,0 +1,195 @@
|
||||
#!/usr/bin/env npx tsx
|
||||
|
||||
/**
|
||||
* Browser launcher using Camoufox with persistent profile
|
||||
*
|
||||
* Usage:
|
||||
* npx tsx browse.ts --url "https://example.com"
|
||||
* npx tsx browse.ts --url "https://example.com" --screenshot --output page.png
|
||||
* npx tsx browse.ts --url "https://example.com" --headless false --wait 5000
|
||||
*/
|
||||
|
||||
import { Camoufox } from 'camoufox-js';
|
||||
import { homedir } from 'os';
|
||||
import { join } from 'path';
|
||||
import { existsSync, mkdirSync } from 'fs';
|
||||
import parseArgs from 'minimist';
|
||||
import type { Page, BrowserContext } from 'playwright-core';
|
||||
|
||||
// Types
|
||||
interface BrowseOptions {
|
||||
url: string;
|
||||
headless?: boolean;
|
||||
screenshot?: boolean;
|
||||
output?: string;
|
||||
wait?: number;
|
||||
timeout?: number;
|
||||
interactive?: boolean;
|
||||
}
|
||||
|
||||
interface BrowseResult {
|
||||
title: string;
|
||||
url: string;
|
||||
screenshotPath?: string;
|
||||
}
|
||||
|
||||
// Get profile directory
|
||||
const getProfilePath = (): string => {
|
||||
const customPath = process.env.CAMOUFOX_PROFILE_PATH;
|
||||
if (customPath) return customPath;
|
||||
|
||||
const profileDir = join(homedir(), '.camoufox-profile');
|
||||
if (!existsSync(profileDir)) {
|
||||
mkdirSync(profileDir, { recursive: true });
|
||||
}
|
||||
return profileDir;
|
||||
};
|
||||
|
||||
// Launch browser with persistent profile
|
||||
export async function launchBrowser(options: {
|
||||
headless?: boolean;
|
||||
}): Promise<BrowserContext> {
|
||||
const profilePath = getProfilePath();
|
||||
const headless =
|
||||
options.headless ??
|
||||
(process.env.CAMOUFOX_HEADLESS ? process.env.CAMOUFOX_HEADLESS === 'true' : true);
|
||||
|
||||
console.log(`Using profile: ${profilePath}`);
|
||||
console.log(`Headless mode: ${headless}`);
|
||||
|
||||
const browser = await Camoufox({
|
||||
user_data_dir: profilePath,
|
||||
headless,
|
||||
});
|
||||
|
||||
return browser;
|
||||
}
|
||||
|
||||
// Browse to URL and optionally take screenshot
|
||||
export async function browse(options: BrowseOptions): Promise<BrowseResult> {
|
||||
const browser = await launchBrowser({ headless: options.headless });
|
||||
const page = await browser.newPage();
|
||||
|
||||
try {
|
||||
// Navigate to URL
|
||||
console.log(`Navigating to: ${options.url}`);
|
||||
await page.goto(options.url, {
|
||||
timeout: options.timeout ?? 60000,
|
||||
waitUntil: 'domcontentloaded',
|
||||
});
|
||||
|
||||
// Wait if specified
|
||||
if (options.wait) {
|
||||
console.log(`Waiting ${options.wait}ms...`);
|
||||
await page.waitForTimeout(options.wait);
|
||||
}
|
||||
|
||||
const result: BrowseResult = {
|
||||
title: await page.title(),
|
||||
url: page.url(),
|
||||
};
|
||||
|
||||
console.log(`Page title: ${result.title}`);
|
||||
console.log(`Final URL: ${result.url}`);
|
||||
|
||||
// Take screenshot if requested
|
||||
if (options.screenshot) {
|
||||
const outputPath = options.output ?? 'screenshot.png';
|
||||
await page.screenshot({ path: outputPath, fullPage: true });
|
||||
result.screenshotPath = outputPath;
|
||||
console.log(`Screenshot saved: ${outputPath}`);
|
||||
}
|
||||
|
||||
// If interactive mode, keep browser open
|
||||
if (options.interactive) {
|
||||
console.log('\nInteractive mode - browser will stay open.');
|
||||
console.log('Press Ctrl+C to close.');
|
||||
await new Promise(() => {}); // Keep running
|
||||
}
|
||||
|
||||
return result;
|
||||
} finally {
|
||||
if (!options.interactive) {
|
||||
await browser.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Export page for use in other scripts
|
||||
export async function getPage(options?: {
|
||||
headless?: boolean;
|
||||
}): Promise<{ page: Page; browser: BrowserContext }> {
|
||||
const browser = await launchBrowser({ headless: options?.headless });
|
||||
const page = await browser.newPage();
|
||||
return { page, browser };
|
||||
}
|
||||
|
||||
// CLI entry point
|
||||
async function main() {
|
||||
const args = parseArgs(process.argv.slice(2), {
|
||||
string: ['url', 'output'],
|
||||
boolean: ['screenshot', 'headless', 'interactive', 'help'],
|
||||
default: {
|
||||
headless: true,
|
||||
screenshot: false,
|
||||
interactive: false,
|
||||
},
|
||||
alias: {
|
||||
u: 'url',
|
||||
o: 'output',
|
||||
s: 'screenshot',
|
||||
h: 'help',
|
||||
i: 'interactive',
|
||||
},
|
||||
});
|
||||
|
||||
if (args.help || !args.url) {
|
||||
console.log(`
|
||||
Web Browser with Camoufox
|
||||
|
||||
Usage:
|
||||
npx tsx browse.ts --url <url> [options]
|
||||
|
||||
Options:
|
||||
-u, --url <url> URL to navigate to (required)
|
||||
-s, --screenshot Take a screenshot of the page
|
||||
-o, --output <path> Output path for screenshot (default: screenshot.png)
|
||||
--headless <bool> Run in headless mode (default: true)
|
||||
--wait <ms> Wait time after page load in milliseconds
|
||||
--timeout <ms> Navigation timeout (default: 60000)
|
||||
-i, --interactive Keep browser open for manual interaction
|
||||
-h, --help Show this help message
|
||||
|
||||
Examples:
|
||||
npx tsx browse.ts --url "https://example.com"
|
||||
npx tsx browse.ts --url "https://example.com" --screenshot --output page.png
|
||||
npx tsx browse.ts --url "https://example.com" --headless false --interactive
|
||||
|
||||
Environment Variables:
|
||||
CAMOUFOX_PROFILE_PATH Custom profile directory (default: ~/.camoufox-profile/)
|
||||
CAMOUFOX_HEADLESS Default headless mode (true/false)
|
||||
`);
|
||||
process.exit(args.help ? 0 : 1);
|
||||
}
|
||||
|
||||
try {
|
||||
await browse({
|
||||
url: args.url,
|
||||
headless: args.headless,
|
||||
screenshot: args.screenshot,
|
||||
output: args.output,
|
||||
wait: args.wait ? parseInt(args.wait, 10) : undefined,
|
||||
timeout: args.timeout ? parseInt(args.timeout, 10) : undefined,
|
||||
interactive: args.interactive,
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('Error:', error instanceof Error ? error.message : error);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
// Run if executed directly
|
||||
const isMainModule = process.argv[1]?.includes('browse.ts');
|
||||
if (isMainModule) {
|
||||
main();
|
||||
}
|
||||
27
skills/web-automation/claude-code/scripts/package.json
Normal file
27
skills/web-automation/claude-code/scripts/package.json
Normal file
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"name": "web-automation-scripts",
|
||||
"version": "1.0.0",
|
||||
"description": "Web browsing and scraping scripts using Camoufox",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"browse": "tsx browse.ts",
|
||||
"scrape": "tsx scrape.ts",
|
||||
"fetch-browser": "npx camoufox-js fetch"
|
||||
},
|
||||
"dependencies": {
|
||||
"camoufox-js": "^0.8.5",
|
||||
"playwright-core": "^1.40.0",
|
||||
"turndown": "^7.1.2",
|
||||
"turndown-plugin-gfm": "^1.0.2",
|
||||
"@mozilla/readability": "^0.5.0",
|
||||
"jsdom": "^24.0.0",
|
||||
"minimist": "^1.2.8"
|
||||
},
|
||||
"devDependencies": {
|
||||
"typescript": "^5.3.0",
|
||||
"@types/turndown": "^5.0.4",
|
||||
"@types/jsdom": "^21.1.6",
|
||||
"@types/minimist": "^1.2.5",
|
||||
"tsx": "^4.7.0"
|
||||
}
|
||||
}
|
||||
1610
skills/web-automation/claude-code/scripts/pnpm-lock.yaml
generated
Normal file
1610
skills/web-automation/claude-code/scripts/pnpm-lock.yaml
generated
Normal file
File diff suppressed because it is too large
Load Diff
351
skills/web-automation/claude-code/scripts/scrape.ts
Normal file
351
skills/web-automation/claude-code/scripts/scrape.ts
Normal file
@@ -0,0 +1,351 @@
|
||||
#!/usr/bin/env npx tsx
|
||||
|
||||
/**
|
||||
* Web scraper that extracts content to markdown
|
||||
*
|
||||
* Usage:
|
||||
* npx tsx scrape.ts --url "https://example.com" --mode main
|
||||
* npx tsx scrape.ts --url "https://example.com" --mode full --output page.md
|
||||
* npx tsx scrape.ts --url "https://example.com" --mode selector --selector ".content"
|
||||
*/
|
||||
|
||||
import TurndownService from 'turndown';
|
||||
import * as turndownPluginGfm from 'turndown-plugin-gfm';
|
||||
import { Readability } from '@mozilla/readability';
|
||||
import { JSDOM } from 'jsdom';
|
||||
import { writeFileSync } from 'fs';
|
||||
import parseArgs from 'minimist';
|
||||
import { getPage } from './browse.js';
|
||||
|
||||
// Types
|
||||
type ScrapeMode = 'main' | 'full' | 'selector';
|
||||
|
||||
interface ScrapeOptions {
|
||||
url: string;
|
||||
mode: ScrapeMode;
|
||||
selector?: string;
|
||||
output?: string;
|
||||
includeLinks?: boolean;
|
||||
includeTables?: boolean;
|
||||
includeImages?: boolean;
|
||||
headless?: boolean;
|
||||
wait?: number;
|
||||
}
|
||||
|
||||
interface ScrapeResult {
|
||||
title: string;
|
||||
url: string;
|
||||
markdown: string;
|
||||
byline?: string;
|
||||
excerpt?: string;
|
||||
}
|
||||
|
||||
// Configure Turndown for markdown conversion
|
||||
function createTurndownService(options: {
|
||||
includeLinks?: boolean;
|
||||
includeTables?: boolean;
|
||||
includeImages?: boolean;
|
||||
}): TurndownService {
|
||||
const turndown = new TurndownService({
|
||||
headingStyle: 'atx',
|
||||
hr: '---',
|
||||
bulletListMarker: '-',
|
||||
codeBlockStyle: 'fenced',
|
||||
fence: '```',
|
||||
emDelimiter: '*',
|
||||
strongDelimiter: '**',
|
||||
linkStyle: 'inlined',
|
||||
});
|
||||
|
||||
// Add GFM support (tables, strikethrough, task lists)
|
||||
turndown.use(turndownPluginGfm.gfm);
|
||||
|
||||
// Custom rule for code blocks with language detection
|
||||
turndown.addRule('codeBlockWithLanguage', {
|
||||
filter: (node) => {
|
||||
return (
|
||||
node.nodeName === 'PRE' &&
|
||||
node.firstChild?.nodeName === 'CODE'
|
||||
);
|
||||
},
|
||||
replacement: (_content, node) => {
|
||||
const codeNode = node.firstChild as HTMLElement;
|
||||
const className = codeNode.getAttribute('class') || '';
|
||||
const langMatch = className.match(/language-(\w+)/);
|
||||
const lang = langMatch ? langMatch[1] : '';
|
||||
const code = codeNode.textContent || '';
|
||||
return `\n\n\`\`\`${lang}\n${code}\n\`\`\`\n\n`;
|
||||
},
|
||||
});
|
||||
|
||||
// Remove images if not included
|
||||
if (!options.includeImages) {
|
||||
turndown.addRule('removeImages', {
|
||||
filter: 'img',
|
||||
replacement: () => '',
|
||||
});
|
||||
}
|
||||
|
||||
// Remove links but keep text if not included
|
||||
if (!options.includeLinks) {
|
||||
turndown.addRule('removeLinks', {
|
||||
filter: 'a',
|
||||
replacement: (content) => content,
|
||||
});
|
||||
}
|
||||
|
||||
// Remove script, style, nav, footer, aside elements
|
||||
turndown.remove(['script', 'style', 'nav', 'footer', 'aside', 'noscript']);
|
||||
|
||||
return turndown;
|
||||
}
|
||||
|
||||
// Extract main content using Readability
|
||||
function extractMainContent(html: string, url: string): {
|
||||
content: string;
|
||||
title: string;
|
||||
byline?: string;
|
||||
excerpt?: string;
|
||||
} {
|
||||
const dom = new JSDOM(html, { url });
|
||||
const reader = new Readability(dom.window.document);
|
||||
const article = reader.parse();
|
||||
|
||||
if (!article) {
|
||||
throw new Error('Could not extract main content from page');
|
||||
}
|
||||
|
||||
return {
|
||||
content: article.content,
|
||||
title: article.title,
|
||||
byline: article.byline || undefined,
|
||||
excerpt: article.excerpt || undefined,
|
||||
};
|
||||
}
|
||||
|
||||
// Scrape a URL and return markdown
|
||||
export async function scrape(options: ScrapeOptions): Promise<ScrapeResult> {
|
||||
const { page, browser } = await getPage({ headless: options.headless ?? true });
|
||||
|
||||
try {
|
||||
// Navigate to URL
|
||||
console.log(`Navigating to: ${options.url}`);
|
||||
await page.goto(options.url, {
|
||||
timeout: 60000,
|
||||
waitUntil: 'domcontentloaded',
|
||||
});
|
||||
|
||||
// Wait if specified
|
||||
if (options.wait) {
|
||||
console.log(`Waiting ${options.wait}ms for dynamic content...`);
|
||||
await page.waitForTimeout(options.wait);
|
||||
}
|
||||
|
||||
const pageTitle = await page.title();
|
||||
const pageUrl = page.url();
|
||||
|
||||
let html: string;
|
||||
let title = pageTitle;
|
||||
let byline: string | undefined;
|
||||
let excerpt: string | undefined;
|
||||
|
||||
// Get HTML based on mode
|
||||
switch (options.mode) {
|
||||
case 'main': {
|
||||
// Get full page HTML and extract with Readability
|
||||
const fullHtml = await page.content();
|
||||
const extracted = extractMainContent(fullHtml, pageUrl);
|
||||
html = extracted.content;
|
||||
title = extracted.title || pageTitle;
|
||||
byline = extracted.byline;
|
||||
excerpt = extracted.excerpt;
|
||||
break;
|
||||
}
|
||||
|
||||
case 'selector': {
|
||||
if (!options.selector) {
|
||||
throw new Error('Selector mode requires --selector option');
|
||||
}
|
||||
const element = await page.$(options.selector);
|
||||
if (!element) {
|
||||
throw new Error(`Selector not found: ${options.selector}`);
|
||||
}
|
||||
html = await element.innerHTML();
|
||||
break;
|
||||
}
|
||||
|
||||
case 'full':
|
||||
default: {
|
||||
// Get body content, excluding common non-content elements
|
||||
html = await page.evaluate(() => {
|
||||
// Remove common non-content elements
|
||||
const selectorsToRemove = [
|
||||
'script', 'style', 'noscript', 'iframe',
|
||||
'nav', 'header', 'footer', '.cookie-banner',
|
||||
'.advertisement', '.ads', '#ads', '.social-share',
|
||||
'.comments', '#comments', '.sidebar'
|
||||
];
|
||||
|
||||
selectorsToRemove.forEach(selector => {
|
||||
document.querySelectorAll(selector).forEach(el => el.remove());
|
||||
});
|
||||
|
||||
return document.body.innerHTML;
|
||||
});
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Convert to markdown
|
||||
const turndown = createTurndownService({
|
||||
includeLinks: options.includeLinks ?? true,
|
||||
includeTables: options.includeTables ?? true,
|
||||
includeImages: options.includeImages ?? false,
|
||||
});
|
||||
|
||||
let markdown = turndown.turndown(html);
|
||||
|
||||
// Add title as H1 if not already present
|
||||
if (!markdown.startsWith('# ')) {
|
||||
markdown = `# ${title}\n\n${markdown}`;
|
||||
}
|
||||
|
||||
// Add metadata header
|
||||
const metadataLines = [
|
||||
`<!-- Scraped from: ${pageUrl} -->`,
|
||||
byline ? `<!-- Author: ${byline} -->` : null,
|
||||
excerpt ? `<!-- Excerpt: ${excerpt} -->` : null,
|
||||
`<!-- Scraped at: ${new Date().toISOString()} -->`,
|
||||
'',
|
||||
].filter(Boolean);
|
||||
|
||||
markdown = metadataLines.join('\n') + '\n' + markdown;
|
||||
|
||||
// Clean up excessive whitespace
|
||||
markdown = markdown
|
||||
.replace(/\n{4,}/g, '\n\n\n')
|
||||
.replace(/[ \t]+$/gm, '')
|
||||
.trim();
|
||||
|
||||
const result: ScrapeResult = {
|
||||
title,
|
||||
url: pageUrl,
|
||||
markdown,
|
||||
byline,
|
||||
excerpt,
|
||||
};
|
||||
|
||||
// Save to file if output specified
|
||||
if (options.output) {
|
||||
writeFileSync(options.output, markdown, 'utf-8');
|
||||
console.log(`Markdown saved to: ${options.output}`);
|
||||
}
|
||||
|
||||
return result;
|
||||
} finally {
|
||||
await browser.close();
|
||||
}
|
||||
}
|
||||
|
||||
// CLI entry point
|
||||
async function main() {
|
||||
const args = parseArgs(process.argv.slice(2), {
|
||||
string: ['url', 'mode', 'selector', 'output'],
|
||||
boolean: ['headless', 'links', 'tables', 'images', 'help'],
|
||||
default: {
|
||||
mode: 'main',
|
||||
headless: true,
|
||||
links: true,
|
||||
tables: true,
|
||||
images: false,
|
||||
},
|
||||
alias: {
|
||||
u: 'url',
|
||||
m: 'mode',
|
||||
s: 'selector',
|
||||
o: 'output',
|
||||
h: 'help',
|
||||
},
|
||||
});
|
||||
|
||||
if (args.help || !args.url) {
|
||||
console.log(`
|
||||
Web Scraper - Extract content to Markdown
|
||||
|
||||
Usage:
|
||||
npx tsx scrape.ts --url <url> [options]
|
||||
|
||||
Options:
|
||||
-u, --url <url> URL to scrape (required)
|
||||
-m, --mode <mode> Scrape mode: main, full, or selector (default: main)
|
||||
-s, --selector <sel> CSS selector for selector mode
|
||||
-o, --output <path> Output file path for markdown
|
||||
--headless <bool> Run in headless mode (default: true)
|
||||
--wait <ms> Wait time for dynamic content
|
||||
--links Include links in output (default: true)
|
||||
--tables Include tables in output (default: true)
|
||||
--images Include images in output (default: false)
|
||||
-h, --help Show this help message
|
||||
|
||||
Scrape Modes:
|
||||
main Extract main article content using Readability (best for articles)
|
||||
full Full page content with common elements removed
|
||||
selector Extract specific element by CSS selector
|
||||
|
||||
Examples:
|
||||
npx tsx scrape.ts --url "https://docs.example.com/guide" --mode main
|
||||
npx tsx scrape.ts --url "https://example.com" --mode full --output page.md
|
||||
npx tsx scrape.ts --url "https://example.com" --mode selector --selector ".api-docs"
|
||||
npx tsx scrape.ts --url "https://example.com" --mode main --no-links --output clean.md
|
||||
|
||||
Output Format:
|
||||
- GitHub Flavored Markdown (tables, strikethrough, task lists)
|
||||
- Proper heading hierarchy
|
||||
- Code blocks with language detection
|
||||
- Metadata comments at top (source URL, date)
|
||||
`);
|
||||
process.exit(args.help ? 0 : 1);
|
||||
}
|
||||
|
||||
const mode = args.mode as ScrapeMode;
|
||||
if (!['main', 'full', 'selector'].includes(mode)) {
|
||||
console.error(`Invalid mode: ${mode}. Must be main, full, or selector.`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await scrape({
|
||||
url: args.url,
|
||||
mode,
|
||||
selector: args.selector,
|
||||
output: args.output,
|
||||
includeLinks: args.links,
|
||||
includeTables: args.tables,
|
||||
includeImages: args.images,
|
||||
headless: args.headless,
|
||||
wait: args.wait ? parseInt(args.wait, 10) : undefined,
|
||||
});
|
||||
|
||||
// Print result summary
|
||||
console.log(`\nScrape complete:`);
|
||||
console.log(` Title: ${result.title}`);
|
||||
console.log(` URL: ${result.url}`);
|
||||
if (result.byline) console.log(` Author: ${result.byline}`);
|
||||
console.log(` Markdown length: ${result.markdown.length} chars`);
|
||||
|
||||
// Print markdown if not saved to file
|
||||
if (!args.output) {
|
||||
console.log('\n--- Markdown Output ---\n');
|
||||
console.log(result.markdown);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error:', error instanceof Error ? error.message : error);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
// Run if executed directly
|
||||
const isMainModule = process.argv[1]?.includes('scrape.ts');
|
||||
if (isMainModule) {
|
||||
main();
|
||||
}
|
||||
39
skills/web-automation/claude-code/scripts/test-full.ts
Normal file
39
skills/web-automation/claude-code/scripts/test-full.ts
Normal file
@@ -0,0 +1,39 @@
|
||||
import { Camoufox } from 'camoufox-js';
|
||||
import { homedir } from 'os';
|
||||
import { join } from 'path';
|
||||
import { mkdirSync, existsSync } from 'fs';
|
||||
|
||||
async function test() {
|
||||
const profilePath = join(homedir(), '.camoufox-profile');
|
||||
if (!existsSync(profilePath)) {
|
||||
mkdirSync(profilePath, { recursive: true });
|
||||
}
|
||||
|
||||
console.log('Profile path:', profilePath);
|
||||
console.log('Launching with full options...');
|
||||
|
||||
const browser = await Camoufox({
|
||||
headless: true,
|
||||
user_data_dir: profilePath,
|
||||
// humanize: 1.5, // Test without this first
|
||||
// geoip: true, // Test without this first
|
||||
// enable_cache: true,
|
||||
// block_webrtc: false,
|
||||
});
|
||||
|
||||
console.log('Browser launched');
|
||||
const page = await browser.newPage();
|
||||
console.log('Page created');
|
||||
|
||||
await page.goto('https://github.com', { timeout: 30000 });
|
||||
console.log('Navigated to:', page.url());
|
||||
console.log('Title:', await page.title());
|
||||
|
||||
await page.screenshot({ path: '/tmp/github-test.png' });
|
||||
console.log('Screenshot saved');
|
||||
|
||||
await browser.close();
|
||||
console.log('Done');
|
||||
}
|
||||
|
||||
test().catch(console.error);
|
||||
22
skills/web-automation/claude-code/scripts/test-minimal.ts
Normal file
22
skills/web-automation/claude-code/scripts/test-minimal.ts
Normal file
@@ -0,0 +1,22 @@
|
||||
import { Camoufox } from 'camoufox-js';
|
||||
|
||||
async function test() {
|
||||
console.log('Launching Camoufox with minimal config...');
|
||||
|
||||
const browser = await Camoufox({
|
||||
headless: true,
|
||||
});
|
||||
|
||||
console.log('Browser launched');
|
||||
const page = await browser.newPage();
|
||||
console.log('Page created');
|
||||
|
||||
await page.goto('https://example.com', { timeout: 30000 });
|
||||
console.log('Navigated to:', page.url());
|
||||
console.log('Title:', await page.title());
|
||||
|
||||
await browser.close();
|
||||
console.log('Done');
|
||||
}
|
||||
|
||||
test().catch(console.error);
|
||||
32
skills/web-automation/claude-code/scripts/test-profile.ts
Normal file
32
skills/web-automation/claude-code/scripts/test-profile.ts
Normal file
@@ -0,0 +1,32 @@
|
||||
import { Camoufox } from 'camoufox-js';
|
||||
import { homedir } from 'os';
|
||||
import { join } from 'path';
|
||||
import { mkdirSync, existsSync } from 'fs';
|
||||
|
||||
async function test() {
|
||||
const profilePath = join(homedir(), '.camoufox-profile');
|
||||
if (!existsSync(profilePath)) {
|
||||
mkdirSync(profilePath, { recursive: true });
|
||||
}
|
||||
|
||||
console.log('Profile path:', profilePath);
|
||||
console.log('Launching with user_data_dir...');
|
||||
|
||||
const browser = await Camoufox({
|
||||
headless: true,
|
||||
user_data_dir: profilePath,
|
||||
});
|
||||
|
||||
console.log('Browser launched');
|
||||
const page = await browser.newPage();
|
||||
console.log('Page created');
|
||||
|
||||
await page.goto('https://example.com', { timeout: 30000 });
|
||||
console.log('Navigated to:', page.url());
|
||||
console.log('Title:', await page.title());
|
||||
|
||||
await browser.close();
|
||||
console.log('Done');
|
||||
}
|
||||
|
||||
test().catch(console.error);
|
||||
16
skills/web-automation/claude-code/scripts/tsconfig.json
Normal file
16
skills/web-automation/claude-code/scripts/tsconfig.json
Normal file
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2022",
|
||||
"module": "ESNext",
|
||||
"moduleResolution": "bundler",
|
||||
"esModuleInterop": true,
|
||||
"allowSyntheticDefaultImports": true,
|
||||
"strict": true,
|
||||
"skipLibCheck": true,
|
||||
"resolveJsonModule": true,
|
||||
"outDir": "./dist",
|
||||
"rootDir": "."
|
||||
},
|
||||
"include": ["*.ts"],
|
||||
"exclude": ["node_modules", "dist"]
|
||||
}
|
||||
8
skills/web-automation/claude-code/scripts/turndown-plugin-gfm.d.ts
vendored
Normal file
8
skills/web-automation/claude-code/scripts/turndown-plugin-gfm.d.ts
vendored
Normal file
@@ -0,0 +1,8 @@
|
||||
declare module 'turndown-plugin-gfm' {
|
||||
import TurndownService from 'turndown';
|
||||
|
||||
export function gfm(turndownService: TurndownService): void;
|
||||
export function strikethrough(turndownService: TurndownService): void;
|
||||
export function tables(turndownService: TurndownService): void;
|
||||
export function taskListItems(turndownService: TurndownService): void;
|
||||
}
|
||||
Reference in New Issue
Block a user