Perform code optimization and document cleanup (#1)

## Summary - add repository-wide quality tooling and verification scaffolding, including CI workflows, pnpm workspace setup, ESLint/Prettier/markdown checks, and generated-output verification helpers - reorganize skill sources and generation flow by introducing canonical `_source` variants, generator/manifests, reusable helper abstractions, and shared web-automation/browser utilities - clean up and expand documentation so the root README flows into docs and skill docs, with clearer development, reviewer, installer, and workflow guidance ## Notable changes - docs flow and consistency cleanup across `README.md`, `docs/README.md`, and related docs - new scripts for `check`, docs verification, generated-file verification, shell portability, and safe directory replacement - refactors in Atlassian and web-automation skill runtimes to reduce duplication and centralize reusable code - changelog, development documentation, and CI surface updates ## Test Plan - [ ] `pnpm run check` - [ ] review generated/manifests and skill sync outputs - [ ] smoke-check docs flow from `README.md` to `docs/README.md` to skill docs ## Notes - this branch currently includes tracked `skills/web-automation/shared/node_modules` content that should be reviewed carefully as potentially noisy/accidental committed artifacts Co-authored-by: Stefano Fiorini <stefano.fiorini@firsthorizon.com> Reviewed-on: #1
2026-05-04 04:41:34 +00:00
parent 2deab1c1b4
commit 251148c3ff
373 changed files with 28504 additions and 1281 deletions
@@ -0,0 +1,103 @@
+{
+  "$schema": "https://ai-coding-skills.dev/schemas/generated-manifest/v1.json",
+  "generator": "scripts/generate-skills.mjs",
+  "generatedRoot": "skills/web-automation/pi",
+  "files": [
+    {
+      "path": "scripts/auth.ts",
+      "kind": "file",
+      "mode": "644",
+      "sha256": "c0940f452437b05b95e58a9a7ab265fb50aa412bd672e82fedd6a37cbfb3d505"
+    },
+    {
+      "path": "scripts/browse.ts",
+      "kind": "file",
+      "mode": "644",
+      "sha256": "d7e4b4c50116032e5a00f90bca27e069dfc5bbf6eeb06ec8f8edc9e5a9792ab8"
+    },
+    {
+      "path": "scripts/check-install.js",
+      "kind": "file",
+      "mode": "644",
+      "sha256": "e46ee8cbe103794bf1e9c3466bb0fbd21079ceddc60ad9521299e8bc0150e48f"
+    },
+    {
+      "path": "scripts/extract.js",
+      "kind": "file",
+      "mode": "644",
+      "sha256": "6fa2a0589de8afd6501e332e5fa263e1344187ea43a33590b431cdee59d04217"
+    },
+    {
+      "path": "scripts/flow.ts",
+      "kind": "file",
+      "mode": "644",
+      "sha256": "94f3e7987cab253dc3c9e80656a11759fada13b3915608bff7ae08418602f366"
+    },
+    {
+      "path": "scripts/lib/browser.ts",
+      "kind": "file",
+      "mode": "644",
+      "sha256": "879b5f883ff1f888d45ed20be05c2d9bc3d6fe5305a1972b7d49a7e6c0e24934"
+    },
+    {
+      "path": "scripts/package.json",
+      "kind": "file",
+      "mode": "644",
+      "sha256": "0441607db0272fc991c893133cb69560274cbbde4e2104bf50b4ae1ec58fc39a"
+    },
+    {
+      "path": "scripts/pnpm-lock.yaml",
+      "kind": "file",
+      "mode": "644",
+      "sha256": "17017e15e8b04311f5d53bdd37065b2f5a514a3119f40a0403148440ed181437"
+    },
+    {
+      "path": "scripts/scan-local-app.ts",
+      "kind": "file",
+      "mode": "644",
+      "sha256": "9e1818c254a633e087715609152936dcb3613a0aa724d40a8a13460510691dc7"
+    },
+    {
+      "path": "scripts/scrape.ts",
+      "kind": "file",
+      "mode": "644",
+      "sha256": "a1a3d81d57d9e8ab1854ce3cb230bdd39ae1087ec50c9fe82cc58f5f2663ebeb"
+    },
+    {
+      "path": "scripts/test-full.ts",
+      "kind": "file",
+      "mode": "644",
+      "sha256": "76a647e840753621445c36894bff62e163f6a2e4d0860fa8e64d8df45fe21e08"
+    },
+    {
+      "path": "scripts/test-minimal.ts",
+      "kind": "file",
+      "mode": "644",
+      "sha256": "59e0b2319d3f7521b2a8a4fca2d779afaa157bf2d160160fdec8cb56bea30b4f"
+    },
+    {
+      "path": "scripts/test-profile.ts",
+      "kind": "file",
+      "mode": "644",
+      "sha256": "6cf0141581a9275bfa8a070a36212cef5f6417d64df3df3e614ec682008376b9"
+    },
+    {
+      "path": "scripts/tsconfig.json",
+      "kind": "file",
+      "mode": "644",
+      "sha256": "e5f22d72266068cf410976c880511f2ec1875445256e11739a5e1de6ffedf38d"
+    },
+    {
+      "path": "scripts/turndown-plugin-gfm.d.ts",
+      "kind": "file",
+      "mode": "644",
+      "sha256": "c5001c059b160eff18a4097a8a0a7b96689b4ebc374543c7d5bf6e40b0d8a5ac"
+    },
+    {
+      "path": "SKILL.md",
+      "kind": "file",
+      "mode": "644",
+      "sha256": "7ff56c1c50697439875f4dd0a7f7697962c8ba2105a4f66ab7b170f5dcc762bd"
+    }
+  ]
+}
@@ -3,6 +3,8 @@ name: web-automation
 description: Browse and scrape web pages using Playwright-compatible CloakBrowser. Use when automating web workflows, extracting rendered page content, handling authenticated sessions, or running multi-step browser flows.
 ---

+<!-- ⚠️ GENERATED FILE – do not edit directly. Edit the canonical source in skills/web-automation/_source/pi/SKILL.md and run `pnpm run sync:pi`. -->
+
 # Web Automation with CloakBrowser (Pi)

 Automated web browsing and scraping for pi using the shared runtime bundle in `scripts/`.
@@ -1,4 +1,5 @@
 #!/usr/bin/env npx tsx
+// ⚠️ GENERATED FILE – do not edit directly. Edit the canonical source in skills/web-automation/shared/ and run `pnpm run sync:pi`.

 /**
 * Authentication handler for web automation
@@ -10,7 +11,7 @@
 *   npx tsx auth.ts --url "https://example.com" --type auto
 */

-import { getPage, launchBrowser } from './browse.js';
+import { getPage, launchBrowser } from './lib/browser.js';
 import parseArgs from 'minimist';
 import type { Page, BrowserContext } from 'playwright-core';
 import { createInterface } from 'readline';
@@ -1,4 +1,5 @@
 #!/usr/bin/env npx tsx
+// ⚠️ GENERATED FILE – do not edit directly. Edit the canonical source in skills/web-automation/shared/ and run `pnpm run sync:pi`.

 /**
 * Browser launcher using CloakBrowser with persistent profile
@@ -9,12 +10,13 @@
 *   npx tsx browse.ts --url "https://example.com" --headless false --wait 5000
 */

-import { launchPersistentContext } from 'cloakbrowser';
-import { homedir } from 'os';
-import { join } from 'path';
-import { existsSync, mkdirSync } from 'fs';
 import parseArgs from 'minimist';
-import type { Page, BrowserContext } from 'playwright-core';
+import type { BrowserContext } from 'playwright-core';
+
+import { getProfilePath, launchBrowser, getPage } from './lib/browser.js';
+
+// Re-export shared helpers so existing imports of browse.ts continue to work.
+export { getProfilePath, launchBrowser, getPage };

 interface BrowseOptions {
  url: string;
@@ -36,36 +38,6 @@ function sleep(ms: number): Promise<void> {
  return new Promise((resolve) => setTimeout(resolve, ms));
 }

-const getProfilePath = (): string => {
-  const customPath = process.env.CLOAKBROWSER_PROFILE_PATH;
-  if (customPath) return customPath;
-
-  const profileDir = join(homedir(), '.cloakbrowser-profile');
-  if (!existsSync(profileDir)) {
-    mkdirSync(profileDir, { recursive: true });
-  }
-  return profileDir;
-};
-
-export async function launchBrowser(options: {
-  headless?: boolean;
-}): Promise<BrowserContext> {
-  const profilePath = getProfilePath();
-  const envHeadless = process.env.CLOAKBROWSER_HEADLESS;
-  const headless = options.headless ?? (envHeadless ? envHeadless === 'true' : true);
-
-  console.log(`Using profile: ${profilePath}`);
-  console.log(`Headless mode: ${headless}`);
-
-  const context = await launchPersistentContext({
-    userDataDir: profilePath,
-    headless,
-    humanize: true,
-  });
-
-  return context;
-}
-
 export async function browse(options: BrowseOptions): Promise<BrowseResult> {
  const browser = await launchBrowser({ headless: options.headless });
  const page = browser.pages()[0] || await browser.newPage();
@@ -111,14 +83,6 @@ export async function browse(options: BrowseOptions): Promise<BrowseResult> {
  }
 }

-export async function getPage(options?: {
-  headless?: boolean;
-}): Promise<{ page: Page; browser: BrowserContext }> {
-  const browser = await launchBrowser({ headless: options?.headless });
-  const page = browser.pages()[0] || await browser.newPage();
-  return { page, browser };
-}
-
 async function main() {
  const args = parseArgs(process.argv.slice(2), {
    string: ['url', 'output'],
@@ -1,4 +1,5 @@
 #!/usr/bin/env node
+// ⚠️ GENERATED FILE – do not edit directly. Edit the canonical source in skills/web-automation/shared/ and run `pnpm run sync:pi`.

 import fs from "node:fs";
 import path from "node:path";
@@ -18,11 +19,9 @@ async function main() {
  try {
    await import("cloakbrowser");
    await import("playwright-core");
-    await import("better-sqlite3");
-    await import("esbuild");
  } catch (error) {
    fail(
-      "Missing dependency/config: web-automation requires cloakbrowser, playwright-core, better-sqlite3, and esbuild.",
+      "Missing dependency/config: web-automation requires cloakbrowser and playwright-core.",
      error instanceof Error ? error.message : String(error)
    );
  }
@@ -34,7 +33,6 @@ async function main() {
  }

  process.stdout.write("OK: cloakbrowser + playwright-core installed\n");
-  process.stdout.write("OK: better-sqlite3 + esbuild installed\n");
  process.stdout.write("OK: CloakBrowser integration detected in browse.ts\n");
 }

@@ -1,4 +1,5 @@
 #!/usr/bin/env node
+// ⚠️ GENERATED FILE – do not edit directly. Edit the canonical source in skills/web-automation/shared/ and run `pnpm run sync:pi`.

 import fs from "node:fs";
 import path from "node:path";
@@ -1,8 +1,9 @@
 #!/usr/bin/env npx tsx
+// ⚠️ GENERATED FILE – do not edit directly. Edit the canonical source in skills/web-automation/shared/ and run `pnpm run sync:pi`.

 import parseArgs from 'minimist';
 import type { Page } from 'playwright-core';
-import { launchBrowser } from './browse';
+import { launchBrowser } from './lib/browser.js';

 type Step =
  | { action: 'goto'; url: string }
@@ -0,0 +1,76 @@
+// ⚠️ GENERATED FILE – do not edit directly. Edit the canonical source in skills/web-automation/shared/ and run `pnpm run sync:pi`.
+/**
+ * Shared browser-launch and profile helpers for web-automation scripts.
+ *
+ * Centralises the three reusable primitives that every command entry point
+ * needs:
+ *   - getProfilePath()     — resolve the persistent CloakBrowser profile dir
+ *   - launchBrowser()      — launch a CloakBrowser persistent context
+ *   - getPage()            — get a ready Page + BrowserContext pair
+ *
+ * All command entry points (auth.ts, browse.ts, flow.ts, scan-local-app.ts)
+ * import from here instead of duplicating these bodies.
+ */
+
+import { launchPersistentContext } from 'cloakbrowser';
+import { existsSync, mkdirSync } from 'fs';
+import { homedir } from 'os';
+import { join } from 'path';
+import type { BrowserContext, Page } from 'playwright-core';
+
+/**
+ * Return the path to the persistent CloakBrowser profile directory.
+ *
+ * Uses `CLOAKBROWSER_PROFILE_PATH` env var when set; otherwise defaults to
+ * `~/.cloakbrowser-profile/` and creates it if it does not exist.
+ */
+export function getProfilePath(): string {
+  const customPath = process.env.CLOAKBROWSER_PROFILE_PATH;
+  if (customPath) return customPath;
+
+  const profileDir = join(homedir(), '.cloakbrowser-profile');
+  if (!existsSync(profileDir)) {
+    mkdirSync(profileDir, { recursive: true });
+  }
+  return profileDir;
+}
+
+/**
+ * Launch a CloakBrowser persistent context with the shared profile.
+ *
+ * Headless mode is resolved in order:
+ *   1. `options.headless` (explicit caller preference)
+ *   2. `CLOAKBROWSER_HEADLESS` env var
+ *   3. `true` (safe default)
+ */
+export async function launchBrowser(options: {
+  headless?: boolean;
+}): Promise<BrowserContext> {
+  const profilePath = getProfilePath();
+  const envHeadless = process.env.CLOAKBROWSER_HEADLESS;
+  const headless = options.headless ?? (envHeadless ? envHeadless === 'true' : true);
+
+  console.log(`Using profile: ${profilePath}`);
+  console.log(`Headless mode: ${headless}`);
+
+  const context = await launchPersistentContext({
+    userDataDir: profilePath,
+    headless,
+    humanize: true,
+  });
+
+  return context;
+}
+
+/**
+ * Return a ready `{ page, browser }` pair using the shared persistent profile.
+ *
+ * Re-uses the first existing page or opens a new one if the context is empty.
+ */
+export async function getPage(options?: {
+  headless?: boolean;
+}): Promise<{ page: Page; browser: BrowserContext }> {
+  const browser = await launchBrowser({ headless: options?.headless });
+  const page = browser.pages()[0] || (await browser.newPage());
+  return { page, browser };
+}
@@ -1,5 +1,5 @@
 {
-  "name": "web-automation-scripts",
+  "name": "@ai-coding-skills/web-automation-pi",
  "version": "1.0.0",
  "description": "Web browsing and scraping scripts using CloakBrowser",
  "type": "module",
@@ -32,5 +32,6 @@
    "tsx": "^4.7.0",
    "typescript": "^5.3.0"
  },
-  "packageManager": "pnpm@10.18.1+sha512.77a884a165cbba2d8d1c19e3b4880eee6d2fcabd0d879121e282196b80042351d5eb3ca0935fa599da1dc51265cc68816ad2bddd2a2de5ea9fdf92adbec7cd34"
+  "packageManager": "pnpm@10.18.1+sha512.77a884a165cbba2d8d1c19e3b4880eee6d2fcabd0d879121e282196b80042351d5eb3ca0935fa599da1dc51265cc68816ad2bddd2a2de5ea9fdf92adbec7cd34",
+  "private": true
 }
@@ -1,8 +1,10 @@
 #!/usr/bin/env npx tsx
+// ⚠️ GENERATED FILE – do not edit directly. Edit the canonical source in skills/web-automation/shared/ and run `pnpm run sync:pi`.

 import { mkdirSync, writeFileSync } from 'fs';
 import { dirname, resolve } from 'path';
-import { getPage } from './browse.js';
+import type { Page } from 'playwright-core';
+import { getPage } from './lib/browser.js';

 type NavResult = {
  requestedUrl: string;
@@ -39,30 +41,34 @@ function getRoutes(baseUrl: string): string[] {
  return [baseUrl];
 }

-async function gotoWithStatus(page: any, url: string): Promise<NavResult> {
+type GotoError = { error: unknown };
+
+async function gotoWithStatus(page: Page, url: string): Promise<NavResult> {
  const response = await page
    .goto(url, { waitUntil: 'domcontentloaded', timeout: 60000 })
-    .catch((error: unknown) => ({ error }));
+    .catch((error: unknown): GotoError => ({ error }));

-  if (response?.error) {
+  if (response !== null && response !== undefined && 'error' in response) {
+    const gotoError = response as GotoError;
    return {
      requestedUrl: url,
      url: page.url(),
      status: null,
      title: await page.title().catch(() => ''),
-      error: String(response.error),
+      error: String(gotoError.error),
    };
  }

+  const httpResponse = response as Awaited<ReturnType<Page['goto']>>;
  return {
    requestedUrl: url,
    url: page.url(),
-    status: response ? response.status() : null,
+    status: httpResponse ? httpResponse.status() : null,
    title: await page.title().catch(() => ''),
  };
 }

-async function textOrNull(page: any, selector: string): Promise<string | null> {
+async function textOrNull(page: Page, selector: string): Promise<string | null> {
  const locator = page.locator(selector).first();
  try {
    if ((await locator.count()) === 0) return null;
@@ -73,7 +79,7 @@ async function textOrNull(page: any, selector: string): Promise<string | null> {
  }
 }

-async function loginIfConfigured(page: any, baseUrl: string, lines: string[]) {
+async function loginIfConfigured(page: Page, baseUrl: string, lines: string[]) {
  const loginPath = env('SCAN_LOGIN_PATH');
  const username = env('SCAN_USERNAME') ?? env('CLOAKBROWSER_USERNAME');
  const password = env('SCAN_PASSWORD') ?? env('CLOAKBROWSER_PASSWORD');
@@ -109,7 +115,7 @@ async function loginIfConfigured(page: any, baseUrl: string, lines: string[]) {
  lines.push('');
 }

-async function checkRoutes(page: any, baseUrl: string, lines: string[]) {
+async function checkRoutes(page: Page, baseUrl: string, lines: string[]) {
  const routes = getRoutes(baseUrl);
  const routeChecks: RouteCheck[] = [];

@@ -1,4 +1,5 @@
 #!/usr/bin/env npx tsx
+// ⚠️ GENERATED FILE – do not edit directly. Edit the canonical source in skills/web-automation/shared/ and run `pnpm run sync:pi`.

 /**
 * Web scraper that extracts content to markdown
@@ -1,3 +1,4 @@
+// ⚠️ GENERATED FILE – do not edit directly. Edit the canonical source in skills/web-automation/shared/ and run `pnpm run sync:pi`.
 import { launchPersistentContext } from 'cloakbrowser';
 import { homedir } from 'os';
 import { join } from 'path';
@@ -1,3 +1,4 @@
+// ⚠️ GENERATED FILE – do not edit directly. Edit the canonical source in skills/web-automation/shared/ and run `pnpm run sync:pi`.
 import { launch } from 'cloakbrowser';

 async function test() {
@@ -1,3 +1,4 @@
+// ⚠️ GENERATED FILE – do not edit directly. Edit the canonical source in skills/web-automation/shared/ and run `pnpm run sync:pi`.
 import { launchPersistentContext } from 'cloakbrowser';
 import { homedir } from 'os';
 import { join } from 'path';
@@ -11,6 +11,6 @@
    "outDir": "./dist",
    "rootDir": "."
  },
-  "include": ["*.ts"],
+  "include": ["*.ts", "lib/**/*.ts"],
  "exclude": ["node_modules", "dist"]
 }
@@ -1,3 +1,4 @@
+// ⚠️ GENERATED FILE – do not edit directly. Edit the canonical source in skills/web-automation/shared/ and run `pnpm run sync:pi`.
 declare module 'turndown-plugin-gfm' {
  import TurndownService from 'turndown';