Perform code optimization and document cleanup (#1)
## Summary - add repository-wide quality tooling and verification scaffolding, including CI workflows, pnpm workspace setup, ESLint/Prettier/markdown checks, and generated-output verification helpers - reorganize skill sources and generation flow by introducing canonical `_source` variants, generator/manifests, reusable helper abstractions, and shared web-automation/browser utilities - clean up and expand documentation so the root README flows into docs and skill docs, with clearer development, reviewer, installer, and workflow guidance ## Notable changes - docs flow and consistency cleanup across `README.md`, `docs/README.md`, and related docs - new scripts for `check`, docs verification, generated-file verification, shell portability, and safe directory replacement - refactors in Atlassian and web-automation skill runtimes to reduce duplication and centralize reusable code - changelog, development documentation, and CI surface updates ## Test Plan - [ ] `pnpm run check` - [ ] review generated/manifests and skill sync outputs - [ ] smoke-check docs flow from `README.md` to `docs/README.md` to skill docs ## Notes - this branch currently includes tracked `skills/web-automation/shared/node_modules` content that should be reviewed carefully as potentially noisy/accidental committed artifacts Co-authored-by: Stefano Fiorini <stefano.fiorini@firsthorizon.com> Reviewed-on: #1
This commit was merged in pull request #1.
This commit is contained in:
@@ -0,0 +1,103 @@
|
||||
{
|
||||
"$schema": "https://ai-coding-skills.dev/schemas/generated-manifest/v1.json",
|
||||
"generator": "scripts/generate-skills.mjs",
|
||||
"generatedRoot": "skills/web-automation/claude-code",
|
||||
"files": [
|
||||
{
|
||||
"path": "scripts/auth.ts",
|
||||
"kind": "file",
|
||||
"mode": "644",
|
||||
"sha256": "c0940f452437b05b95e58a9a7ab265fb50aa412bd672e82fedd6a37cbfb3d505"
|
||||
},
|
||||
{
|
||||
"path": "scripts/browse.ts",
|
||||
"kind": "file",
|
||||
"mode": "644",
|
||||
"sha256": "d7e4b4c50116032e5a00f90bca27e069dfc5bbf6eeb06ec8f8edc9e5a9792ab8"
|
||||
},
|
||||
{
|
||||
"path": "scripts/check-install.js",
|
||||
"kind": "file",
|
||||
"mode": "644",
|
||||
"sha256": "e46ee8cbe103794bf1e9c3466bb0fbd21079ceddc60ad9521299e8bc0150e48f"
|
||||
},
|
||||
{
|
||||
"path": "scripts/extract.js",
|
||||
"kind": "file",
|
||||
"mode": "644",
|
||||
"sha256": "6fa2a0589de8afd6501e332e5fa263e1344187ea43a33590b431cdee59d04217"
|
||||
},
|
||||
{
|
||||
"path": "scripts/flow.ts",
|
||||
"kind": "file",
|
||||
"mode": "644",
|
||||
"sha256": "94f3e7987cab253dc3c9e80656a11759fada13b3915608bff7ae08418602f366"
|
||||
},
|
||||
{
|
||||
"path": "scripts/lib/browser.ts",
|
||||
"kind": "file",
|
||||
"mode": "644",
|
||||
"sha256": "879b5f883ff1f888d45ed20be05c2d9bc3d6fe5305a1972b7d49a7e6c0e24934"
|
||||
},
|
||||
{
|
||||
"path": "scripts/package.json",
|
||||
"kind": "file",
|
||||
"mode": "644",
|
||||
"sha256": "8fd46cc1ab9f2b45b8f2b658479276a11a3b79710a530aac7148bc396e0edf35"
|
||||
},
|
||||
{
|
||||
"path": "scripts/pnpm-lock.yaml",
|
||||
"kind": "file",
|
||||
"mode": "644",
|
||||
"sha256": "17017e15e8b04311f5d53bdd37065b2f5a514a3119f40a0403148440ed181437"
|
||||
},
|
||||
{
|
||||
"path": "scripts/scan-local-app.ts",
|
||||
"kind": "file",
|
||||
"mode": "644",
|
||||
"sha256": "9e1818c254a633e087715609152936dcb3613a0aa724d40a8a13460510691dc7"
|
||||
},
|
||||
{
|
||||
"path": "scripts/scrape.ts",
|
||||
"kind": "file",
|
||||
"mode": "644",
|
||||
"sha256": "a1a3d81d57d9e8ab1854ce3cb230bdd39ae1087ec50c9fe82cc58f5f2663ebeb"
|
||||
},
|
||||
{
|
||||
"path": "scripts/test-full.ts",
|
||||
"kind": "file",
|
||||
"mode": "644",
|
||||
"sha256": "76a647e840753621445c36894bff62e163f6a2e4d0860fa8e64d8df45fe21e08"
|
||||
},
|
||||
{
|
||||
"path": "scripts/test-minimal.ts",
|
||||
"kind": "file",
|
||||
"mode": "644",
|
||||
"sha256": "59e0b2319d3f7521b2a8a4fca2d779afaa157bf2d160160fdec8cb56bea30b4f"
|
||||
},
|
||||
{
|
||||
"path": "scripts/test-profile.ts",
|
||||
"kind": "file",
|
||||
"mode": "644",
|
||||
"sha256": "6cf0141581a9275bfa8a070a36212cef5f6417d64df3df3e614ec682008376b9"
|
||||
},
|
||||
{
|
||||
"path": "scripts/tsconfig.json",
|
||||
"kind": "file",
|
||||
"mode": "644",
|
||||
"sha256": "e5f22d72266068cf410976c880511f2ec1875445256e11739a5e1de6ffedf38d"
|
||||
},
|
||||
{
|
||||
"path": "scripts/turndown-plugin-gfm.d.ts",
|
||||
"kind": "file",
|
||||
"mode": "644",
|
||||
"sha256": "c5001c059b160eff18a4097a8a0a7b96689b4ebc374543c7d5bf6e40b0d8a5ac"
|
||||
},
|
||||
{
|
||||
"path": "SKILL.md",
|
||||
"kind": "file",
|
||||
"mode": "644",
|
||||
"sha256": "18fccff0bceba538cac6491a5ff3e7090549aa13885d0ab5c6a73d4e8ca72f01"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -3,6 +3,8 @@ name: web-automation
|
||||
description: Browse and scrape web pages using Playwright-compatible CloakBrowser. Use when automating web workflows, extracting rendered page content, handling authenticated sessions, or running multi-step browser flows.
|
||||
---
|
||||
|
||||
<!-- ⚠️ GENERATED FILE – do not edit directly. Edit the canonical source in skills/web-automation/_source/claude-code/SKILL.md and run `pnpm run sync:pi`. -->
|
||||
|
||||
# Web Automation with CloakBrowser (Claude Code)
|
||||
|
||||
Automated web browsing and scraping using Playwright-compatible CloakBrowser with two execution paths:
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
#!/usr/bin/env npx tsx
|
||||
// ⚠️ GENERATED FILE – do not edit directly. Edit the canonical source in skills/web-automation/shared/ and run `pnpm run sync:pi`.
|
||||
|
||||
/**
|
||||
* Authentication handler for web automation
|
||||
@@ -10,7 +11,7 @@
|
||||
* npx tsx auth.ts --url "https://example.com" --type auto
|
||||
*/
|
||||
|
||||
import { getPage, launchBrowser } from './browse.js';
|
||||
import { getPage, launchBrowser } from './lib/browser.js';
|
||||
import parseArgs from 'minimist';
|
||||
import type { Page, BrowserContext } from 'playwright-core';
|
||||
import { createInterface } from 'readline';
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
#!/usr/bin/env npx tsx
|
||||
// ⚠️ GENERATED FILE – do not edit directly. Edit the canonical source in skills/web-automation/shared/ and run `pnpm run sync:pi`.
|
||||
|
||||
/**
|
||||
* Browser launcher using CloakBrowser with persistent profile
|
||||
@@ -9,12 +10,13 @@
|
||||
* npx tsx browse.ts --url "https://example.com" --headless false --wait 5000
|
||||
*/
|
||||
|
||||
import { launchPersistentContext } from 'cloakbrowser';
|
||||
import { homedir } from 'os';
|
||||
import { join } from 'path';
|
||||
import { existsSync, mkdirSync } from 'fs';
|
||||
import parseArgs from 'minimist';
|
||||
import type { Page, BrowserContext } from 'playwright-core';
|
||||
import type { BrowserContext } from 'playwright-core';
|
||||
|
||||
import { getProfilePath, launchBrowser, getPage } from './lib/browser.js';
|
||||
|
||||
// Re-export shared helpers so existing imports of browse.ts continue to work.
|
||||
export { getProfilePath, launchBrowser, getPage };
|
||||
|
||||
interface BrowseOptions {
|
||||
url: string;
|
||||
@@ -36,36 +38,6 @@ function sleep(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
const getProfilePath = (): string => {
|
||||
const customPath = process.env.CLOAKBROWSER_PROFILE_PATH;
|
||||
if (customPath) return customPath;
|
||||
|
||||
const profileDir = join(homedir(), '.cloakbrowser-profile');
|
||||
if (!existsSync(profileDir)) {
|
||||
mkdirSync(profileDir, { recursive: true });
|
||||
}
|
||||
return profileDir;
|
||||
};
|
||||
|
||||
export async function launchBrowser(options: {
|
||||
headless?: boolean;
|
||||
}): Promise<BrowserContext> {
|
||||
const profilePath = getProfilePath();
|
||||
const envHeadless = process.env.CLOAKBROWSER_HEADLESS;
|
||||
const headless = options.headless ?? (envHeadless ? envHeadless === 'true' : true);
|
||||
|
||||
console.log(`Using profile: ${profilePath}`);
|
||||
console.log(`Headless mode: ${headless}`);
|
||||
|
||||
const context = await launchPersistentContext({
|
||||
userDataDir: profilePath,
|
||||
headless,
|
||||
humanize: true,
|
||||
});
|
||||
|
||||
return context;
|
||||
}
|
||||
|
||||
export async function browse(options: BrowseOptions): Promise<BrowseResult> {
|
||||
const browser = await launchBrowser({ headless: options.headless });
|
||||
const page = browser.pages()[0] || await browser.newPage();
|
||||
@@ -111,14 +83,6 @@ export async function browse(options: BrowseOptions): Promise<BrowseResult> {
|
||||
}
|
||||
}
|
||||
|
||||
export async function getPage(options?: {
|
||||
headless?: boolean;
|
||||
}): Promise<{ page: Page; browser: BrowserContext }> {
|
||||
const browser = await launchBrowser({ headless: options?.headless });
|
||||
const page = browser.pages()[0] || await browser.newPage();
|
||||
return { page, browser };
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const args = parseArgs(process.argv.slice(2), {
|
||||
string: ['url', 'output'],
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
#!/usr/bin/env node
|
||||
// ⚠️ GENERATED FILE – do not edit directly. Edit the canonical source in skills/web-automation/shared/ and run `pnpm run sync:pi`.
|
||||
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
|
||||
Executable → Regular
+1
@@ -1,4 +1,5 @@
|
||||
#!/usr/bin/env node
|
||||
// ⚠️ GENERATED FILE – do not edit directly. Edit the canonical source in skills/web-automation/shared/ and run `pnpm run sync:pi`.
|
||||
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
#!/usr/bin/env npx tsx
|
||||
// ⚠️ GENERATED FILE – do not edit directly. Edit the canonical source in skills/web-automation/shared/ and run `pnpm run sync:pi`.
|
||||
|
||||
import parseArgs from 'minimist';
|
||||
import type { Page } from 'playwright-core';
|
||||
import { launchBrowser } from './browse';
|
||||
import { launchBrowser } from './lib/browser.js';
|
||||
|
||||
type Step =
|
||||
| { action: 'goto'; url: string }
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
// ⚠️ GENERATED FILE – do not edit directly. Edit the canonical source in skills/web-automation/shared/ and run `pnpm run sync:pi`.
|
||||
/**
|
||||
* Shared browser-launch and profile helpers for web-automation scripts.
|
||||
*
|
||||
* Centralises the three reusable primitives that every command entry point
|
||||
* needs:
|
||||
* - getProfilePath() — resolve the persistent CloakBrowser profile dir
|
||||
* - launchBrowser() — launch a CloakBrowser persistent context
|
||||
* - getPage() — get a ready Page + BrowserContext pair
|
||||
*
|
||||
* All command entry points (auth.ts, browse.ts, flow.ts, scan-local-app.ts)
|
||||
* import from here instead of duplicating these bodies.
|
||||
*/
|
||||
|
||||
import { launchPersistentContext } from 'cloakbrowser';
|
||||
import { existsSync, mkdirSync } from 'fs';
|
||||
import { homedir } from 'os';
|
||||
import { join } from 'path';
|
||||
import type { BrowserContext, Page } from 'playwright-core';
|
||||
|
||||
/**
|
||||
* Return the path to the persistent CloakBrowser profile directory.
|
||||
*
|
||||
* Uses `CLOAKBROWSER_PROFILE_PATH` env var when set; otherwise defaults to
|
||||
* `~/.cloakbrowser-profile/` and creates it if it does not exist.
|
||||
*/
|
||||
export function getProfilePath(): string {
|
||||
const customPath = process.env.CLOAKBROWSER_PROFILE_PATH;
|
||||
if (customPath) return customPath;
|
||||
|
||||
const profileDir = join(homedir(), '.cloakbrowser-profile');
|
||||
if (!existsSync(profileDir)) {
|
||||
mkdirSync(profileDir, { recursive: true });
|
||||
}
|
||||
return profileDir;
|
||||
}
|
||||
|
||||
/**
|
||||
* Launch a CloakBrowser persistent context with the shared profile.
|
||||
*
|
||||
* Headless mode is resolved in order:
|
||||
* 1. `options.headless` (explicit caller preference)
|
||||
* 2. `CLOAKBROWSER_HEADLESS` env var
|
||||
* 3. `true` (safe default)
|
||||
*/
|
||||
export async function launchBrowser(options: {
|
||||
headless?: boolean;
|
||||
}): Promise<BrowserContext> {
|
||||
const profilePath = getProfilePath();
|
||||
const envHeadless = process.env.CLOAKBROWSER_HEADLESS;
|
||||
const headless = options.headless ?? (envHeadless ? envHeadless === 'true' : true);
|
||||
|
||||
console.log(`Using profile: ${profilePath}`);
|
||||
console.log(`Headless mode: ${headless}`);
|
||||
|
||||
const context = await launchPersistentContext({
|
||||
userDataDir: profilePath,
|
||||
headless,
|
||||
humanize: true,
|
||||
});
|
||||
|
||||
return context;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return a ready `{ page, browser }` pair using the shared persistent profile.
|
||||
*
|
||||
* Re-uses the first existing page or opens a new one if the context is empty.
|
||||
*/
|
||||
export async function getPage(options?: {
|
||||
headless?: boolean;
|
||||
}): Promise<{ page: Page; browser: BrowserContext }> {
|
||||
const browser = await launchBrowser({ headless: options?.headless });
|
||||
const page = browser.pages()[0] || (await browser.newPage());
|
||||
return { page, browser };
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"name": "web-automation-scripts",
|
||||
"name": "@ai-coding-skills/web-automation-claude-code",
|
||||
"version": "1.0.0",
|
||||
"description": "Web browsing and scraping scripts using CloakBrowser",
|
||||
"type": "module",
|
||||
@@ -32,5 +32,6 @@
|
||||
"tsx": "^4.7.0",
|
||||
"typescript": "^5.3.0"
|
||||
},
|
||||
"packageManager": "pnpm@10.18.1+sha512.77a884a165cbba2d8d1c19e3b4880eee6d2fcabd0d879121e282196b80042351d5eb3ca0935fa599da1dc51265cc68816ad2bddd2a2de5ea9fdf92adbec7cd34"
|
||||
"packageManager": "pnpm@10.18.1+sha512.77a884a165cbba2d8d1c19e3b4880eee6d2fcabd0d879121e282196b80042351d5eb3ca0935fa599da1dc51265cc68816ad2bddd2a2de5ea9fdf92adbec7cd34",
|
||||
"private": true
|
||||
}
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
#!/usr/bin/env npx tsx
|
||||
// ⚠️ GENERATED FILE – do not edit directly. Edit the canonical source in skills/web-automation/shared/ and run `pnpm run sync:pi`.
|
||||
|
||||
import { mkdirSync, writeFileSync } from 'fs';
|
||||
import { dirname, resolve } from 'path';
|
||||
import { getPage } from './browse.js';
|
||||
import type { Page } from 'playwright-core';
|
||||
import { getPage } from './lib/browser.js';
|
||||
|
||||
type NavResult = {
|
||||
requestedUrl: string;
|
||||
@@ -39,30 +41,34 @@ function getRoutes(baseUrl: string): string[] {
|
||||
return [baseUrl];
|
||||
}
|
||||
|
||||
async function gotoWithStatus(page: any, url: string): Promise<NavResult> {
|
||||
type GotoError = { error: unknown };
|
||||
|
||||
async function gotoWithStatus(page: Page, url: string): Promise<NavResult> {
|
||||
const response = await page
|
||||
.goto(url, { waitUntil: 'domcontentloaded', timeout: 60000 })
|
||||
.catch((error: unknown) => ({ error }));
|
||||
.catch((error: unknown): GotoError => ({ error }));
|
||||
|
||||
if (response?.error) {
|
||||
if (response !== null && response !== undefined && 'error' in response) {
|
||||
const gotoError = response as GotoError;
|
||||
return {
|
||||
requestedUrl: url,
|
||||
url: page.url(),
|
||||
status: null,
|
||||
title: await page.title().catch(() => ''),
|
||||
error: String(response.error),
|
||||
error: String(gotoError.error),
|
||||
};
|
||||
}
|
||||
|
||||
const httpResponse = response as Awaited<ReturnType<Page['goto']>>;
|
||||
return {
|
||||
requestedUrl: url,
|
||||
url: page.url(),
|
||||
status: response ? response.status() : null,
|
||||
status: httpResponse ? httpResponse.status() : null,
|
||||
title: await page.title().catch(() => ''),
|
||||
};
|
||||
}
|
||||
|
||||
async function textOrNull(page: any, selector: string): Promise<string | null> {
|
||||
async function textOrNull(page: Page, selector: string): Promise<string | null> {
|
||||
const locator = page.locator(selector).first();
|
||||
try {
|
||||
if ((await locator.count()) === 0) return null;
|
||||
@@ -73,7 +79,7 @@ async function textOrNull(page: any, selector: string): Promise<string | null> {
|
||||
}
|
||||
}
|
||||
|
||||
async function loginIfConfigured(page: any, baseUrl: string, lines: string[]) {
|
||||
async function loginIfConfigured(page: Page, baseUrl: string, lines: string[]) {
|
||||
const loginPath = env('SCAN_LOGIN_PATH');
|
||||
const username = env('SCAN_USERNAME') ?? env('CLOAKBROWSER_USERNAME');
|
||||
const password = env('SCAN_PASSWORD') ?? env('CLOAKBROWSER_PASSWORD');
|
||||
@@ -109,7 +115,7 @@ async function loginIfConfigured(page: any, baseUrl: string, lines: string[]) {
|
||||
lines.push('');
|
||||
}
|
||||
|
||||
async function checkRoutes(page: any, baseUrl: string, lines: string[]) {
|
||||
async function checkRoutes(page: Page, baseUrl: string, lines: string[]) {
|
||||
const routes = getRoutes(baseUrl);
|
||||
const routeChecks: RouteCheck[] = [];
|
||||
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
#!/usr/bin/env npx tsx
|
||||
// ⚠️ GENERATED FILE – do not edit directly. Edit the canonical source in skills/web-automation/shared/ and run `pnpm run sync:pi`.
|
||||
|
||||
/**
|
||||
* Web scraper that extracts content to markdown
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
// ⚠️ GENERATED FILE – do not edit directly. Edit the canonical source in skills/web-automation/shared/ and run `pnpm run sync:pi`.
|
||||
import { launchPersistentContext } from 'cloakbrowser';
|
||||
import { homedir } from 'os';
|
||||
import { join } from 'path';
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
// ⚠️ GENERATED FILE – do not edit directly. Edit the canonical source in skills/web-automation/shared/ and run `pnpm run sync:pi`.
|
||||
import { launch } from 'cloakbrowser';
|
||||
|
||||
async function test() {
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
// ⚠️ GENERATED FILE – do not edit directly. Edit the canonical source in skills/web-automation/shared/ and run `pnpm run sync:pi`.
|
||||
import { launchPersistentContext } from 'cloakbrowser';
|
||||
import { homedir } from 'os';
|
||||
import { join } from 'path';
|
||||
|
||||
@@ -11,6 +11,6 @@
|
||||
"outDir": "./dist",
|
||||
"rootDir": "."
|
||||
},
|
||||
"include": ["*.ts"],
|
||||
"include": ["*.ts", "lib/**/*.ts"],
|
||||
"exclude": ["node_modules", "dist"]
|
||||
}
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
// ⚠️ GENERATED FILE – do not edit directly. Edit the canonical source in skills/web-automation/shared/ and run `pnpm run sync:pi`.
|
||||
declare module 'turndown-plugin-gfm' {
|
||||
import TurndownService from 'turndown';
|
||||
|
||||
|
||||
Reference in New Issue
Block a user