diff --git a/CHANGELOG.md b/CHANGELOG.md index ae02ac2c1..0396a5203 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,7 @@ This project uses selective package publishing. Each release entry lists the pub ### Changed +- The `download.antseed.com` proxy now reports download telemetry per download instead of per HTTP request. Download managers that fetch an installer as several concurrent byte ranges, and browsers resuming a paused download, previously produced one `download_started`/`download_completed` per range (about 3× inflation for those clients); now only the request covering the first byte starts a download and only the one delivering the last byte finishes it, with other segments logged locally. Every proxy event also carries an `attributed` (1/0) GA4 param saying whether the visitor's browser passed its GA ids — `attributed=0` downloads come from browsers that blocked GA and so never fired the website's `download_vpr` click, which explains why server-side download counts can exceed click counts. - Ox Alpha is no longer part of the desktop's curated free model lineup (first-run default and free-first model lists). - Desktop Connected Apps now includes Droid. Connecting adds and selects an `AntSeed Auto` custom model in the live-reloaded Factory settings shared by Droid CLI and Factory Desktop, routes it through the local VPR, refuses to overwrite an existing `antseed` custom model, and restores the user's previous default model on disconnect. diff --git a/apps/download-proxy/README.md b/apps/download-proxy/README.md index 5d524ca9d..766dfd7d4 100644 --- a/apps/download-proxy/README.md +++ b/apps/download-proxy/README.md @@ -28,8 +28,8 @@ GA4 event) and "user actually received the installer". "Latest" is resolved from the GitHub API and cached at the edge for 5 minutes, so a fresh release is picked up within minutes with one API call per -burst. Range requests are forwarded (resumed downloads work) and flagged -`partial=1` in telemetry. +burst. Range requests are forwarded (resumed and segmented downloads work); +events from a 206 response are flagged `partial=1` in telemetry. ## Telemetry @@ -40,8 +40,17 @@ GA4 credentials are configured, sent to GA4 via the Measurement Protocol — the same property that records the website's `download_vpr` clicks, so the full funnel reads in one place. -Funnel note: count completions with `partial=0`; `partial=1` completions are -resumed byte ranges. The transfer runs through workerd's native pipe (a JS +One download, one set of events. Download managers fetch an installer as +several concurrent byte ranges and browsers resume a paused download with a +new `Range` request; the worker reports per *download*, not per request: +the request covering byte 0 emits `download_started`, and only the request +delivering the file's last byte emits `download_completed` / `download_aborted` +(a plain 200 is both). Middle segments appear in the console log only, as +`download_segment_completed` / `download_segment_aborted`. `partial=1` on a +reported event therefore means "this client used ranges" — useful for +spotting download managers — without inflating the count. + +The transfer runs through workerd's native pipe (a JS per-chunk pump would exceed the Workers CPU limit on installer-sized files), so per-chunk byte counting isn't possible: `download_completed` implies the full `total_bytes` were delivered (enforced by `FixedLengthStream`), while @@ -52,10 +61,12 @@ ids to proxy links (`?cid=<_ga client id>&sid=`), and the worker sends events under that `client_id`/`session_id` — so downloads land inside the visitor's GA4 session and inherit source/campaign/landing-page. Ids are strictly validated (digits-and-dot shapes only) and dropped otherwise. -Direct or shared links without ids still count, under a random client_id -with `attributed: 0` in the console line. Each proxy event uses a random Measurement Protocol -`client_id`, so GA4 sees them as standalone hits (fine for counting; joining -to the website session would require passing the GA client id on the URL). +Direct or shared links — and visitors whose browser blocked GA, so no +`_ga` cookie existed to copy — still count, under a random client_id. Every +event carries `attributed` (1/0) in both the console line and the GA4 params; +`attributed=0` downloads can never have a matching `download_vpr` click, which +is why the proxy's `download_started` is the reliable top of the download +funnel and the click event is best read for page/section breakdowns only. ## Deploy diff --git a/apps/download-proxy/src/events.test.ts b/apps/download-proxy/src/events.test.ts index f7b49637f..c286cf42f 100644 --- a/apps/download-proxy/src/events.test.ts +++ b/apps/download-proxy/src/events.test.ts @@ -1,5 +1,14 @@ -import {describe, expect, it} from 'vitest'; -import {endEvent, parseGaIds, startEvent, unresolvedEvent, type DownloadContext} from './events'; +import {afterEach, describe, expect, it, vi} from 'vitest'; +import { + deliverEvent, + endEvent, + parseGaIds, + segmentEvent, + segmentRole, + startEvent, + unresolvedEvent, + type DownloadContext, +} from './events'; const ctx: DownloadContext = { target: {platform: 'mac', arch: 'arm64'}, @@ -79,4 +88,93 @@ describe('download events', () => { expect(event.name).toBe('download_unresolved'); expect(event.params).toMatchObject({platform: 'win', reason: 'no_matching_asset'}); }); + + describe('segmentRole', () => { + it('treats a full 200 response as the whole download', () => { + expect(segmentRole(200, null)).toEqual({first: true, final: true}); + }); + + it('splits a segmented download into one first and one final segment', () => { + // IDM-style: 4 concurrent ranges of a 400-byte file. + expect(segmentRole(206, 'bytes 0-99/400')).toEqual({first: true, final: false}); + expect(segmentRole(206, 'bytes 100-199/400')).toEqual({first: false, final: false}); + expect(segmentRole(206, 'bytes 200-299/400')).toEqual({first: false, final: false}); + expect(segmentRole(206, 'bytes 300-399/400')).toEqual({first: false, final: true}); + }); + + it('lets a browser resume end the download it started', () => { + // First attempt covered the whole file (aborted); the resume takes it to the end. + expect(segmentRole(206, 'bytes 0-399/400')).toEqual({first: true, final: true}); + expect(segmentRole(206, 'bytes 250-399/400')).toEqual({first: false, final: true}); + }); + + it('falls back to first+final when the range cannot be interpreted', () => { + expect(segmentRole(206, null)).toEqual({first: true, final: true}); + expect(segmentRole(206, 'bytes 0-99/*')).toEqual({first: true, final: true}); + expect(segmentRole(206, 'garbage')).toEqual({first: true, final: true}); + }); + }); + + it('names middle-segment events so they are distinguishable in logs', () => { + expect(segmentEvent({...ctx, partial: true}, {completed: true, durationMs: 10}).name).toBe( + 'download_segment_completed', + ); + expect(segmentEvent({...ctx, partial: true}, {completed: false, durationMs: 10}).name).toBe( + 'download_segment_aborted', + ); + }); + + describe('deliverEvent', () => { + const fetchMock = vi.fn(async () => new Response(null, {status: 204})); + const logMock = vi.fn(); + afterEach(() => { + fetchMock.mockClear(); + logMock.mockClear(); + }); + + async function sentBody(ids?: {clientId: string | null; sessionId: string | null}) { + vi.stubGlobal('fetch', fetchMock); + vi.stubGlobal('console', {...console, log: logMock}); + try { + await deliverEvent(startEvent(ctx), {measurementId: 'G-TEST', apiSecret: 's3cret', ids}); + } finally { + vi.unstubAllGlobals(); + } + const call = fetchMock.mock.calls[0] as unknown as [string, {body: string}]; + return { + url: call[0], + body: JSON.parse(call[1].body) as {client_id: string; events: {params: Record}[]}, + }; + } + + it('sends attributed=1 with the visitor ids when the website passed them', async () => { + const {body} = await sentBody({clientId: '1234567890.1699999999', sessionId: '1756223000'}); + expect(body.client_id).toBe('1234567890.1699999999'); + expect(body.events[0]!.params).toMatchObject({attributed: 1, session_id: 1756223000}); + expect(JSON.parse(logMock.mock.calls[0]![0] as string)).toMatchObject({ + event: 'download_started', + attributed: 1, + }); + }); + + it('sends attributed=0 under a random client id otherwise', async () => { + const {body} = await sentBody({clientId: null, sessionId: null}); + expect(body.client_id).toMatch(/^[0-9a-f-]{36}$/); + expect(body.events[0]!.params['attributed']).toBe(0); + expect(body.events[0]!.params['session_id']).toBeUndefined(); + }); + + it('does nothing but log when GA is not configured', async () => { + vi.stubGlobal('fetch', fetchMock); + vi.stubGlobal('console', {...console, log: logMock}); + try { + await deliverEvent(startEvent(ctx), {}); + } finally { + vi.unstubAllGlobals(); + } + expect(fetchMock).not.toHaveBeenCalled(); + expect(logMock).toHaveBeenCalledTimes(1); + }); + }); }); + diff --git a/apps/download-proxy/src/events.ts b/apps/download-proxy/src/events.ts index 2b4137443..2a586125e 100644 --- a/apps/download-proxy/src/events.ts +++ b/apps/download-proxy/src/events.ts @@ -13,8 +13,12 @@ * download_aborted client disconnected (or origin failed) mid-transfer * download_unresolved no matching installer (partial release, API failure) * - * A 206 Range response carries partial=1 — resumed downloads complete "their" - * range, so funnel analysis should count completions with partial=0. + * Segmented and resumed downloads arrive as several Range requests for one + * file. Events are emitted per *download*, not per request (see + * segmentRole): the request that covers byte 0 emits download_started, and + * only the request that delivers the file's last byte emits the end event. + * Those events still carry partial=1 when they came from a 206, so download + * managers stay identifiable — but one download counts once. */ import type {Target} from './assets'; @@ -93,6 +97,46 @@ export function unresolvedEvent( }; } +/** + * Which telemetry a response should emit for a multi-request download. + * + * Download managers (IDM & co.) fetch one installer as N concurrent byte + * ranges, and browsers resume a paused download with a `Range: bytes=X-` + * request. Reporting started/completed per request inflated both counts + * ~3x for those clients. Instead, the request covering byte 0 is "first" + * (it starts the download) and the request delivering the last byte is + * "final" (its outcome ends the download). A 200 is both. Anything else is + * a middle segment and only reaches the console log. + * + * Unparseable or unbounded (`bytes 0-99/*`) Content-Range headers fall back + * to first+final, so a misbehaving origin still produces events rather than + * silently dropping a download. + */ +export interface SegmentRole { + first: boolean; + final: boolean; +} + +const CONTENT_RANGE_RE = /^bytes (\d+)-(\d+)\/(\d+|\*)$/i; + +export function segmentRole(status: number, contentRange: string | null): SegmentRole { + if (status !== 206) return {first: true, final: true}; + const match = contentRange ? CONTENT_RANGE_RE.exec(contentRange.trim()) : null; + if (!match || match[3] === '*') return {first: true, final: true}; + const start = Number(match[1]); + const end = Number(match[2]); + const total = Number(match[3]); + return {first: start === 0, final: end === total - 1}; +} + +/** Console-only record of a middle segment (neither first nor final). */ +export function segmentEvent(ctx: DownloadContext, pump: PumpResult): DownloadEvent { + return { + name: pump.completed ? 'download_segment_completed' : 'download_segment_aborted', + params: {...baseParams(ctx), duration_ms: pump.durationMs}, + }; +} + const GA4_ENDPOINT = 'https://www.google-analytics.com/mp/collect'; const GA_CLIENT_ID_RE = /^\d{5,15}\.\d{5,15}$/; @@ -130,14 +174,18 @@ export interface Ga4Delivery { * is sent under that client_id — and session_id — so it lands inside the * same GA4 user and session as the download_vpr click, inheriting source, * campaign, and landing page. Without it (direct links, shared URLs), a - * random UUID keeps the event countable but unattributed. + * random UUID keeps the event countable but unattributed. The same flag is + * sent to GA as the `attributed` param. */ export async function deliverEvent(event: DownloadEvent, ga: Ga4Delivery): Promise { - console.log(JSON.stringify({event: event.name, ...event.params, attributed: ga.ids?.clientId ? 1 : 0})); + // `attributed` separates visitors whose browser ran GA (and so could also + // have emitted the website's download_vpr click) from those where it was + // blocked — the latter only ever exist as proxy events. + const attributed = ga.ids?.clientId ? 1 : 0; + console.log(JSON.stringify({event: event.name, ...event.params, attributed})); if (!ga.measurementId || !ga.apiSecret) return; - const params = ga.ids?.sessionId - ? {...event.params, session_id: Number(ga.ids.sessionId)} - : event.params; + const params: Record = {...event.params, attributed}; + if (ga.ids?.sessionId) params['session_id'] = Number(ga.ids.sessionId); const url = `${GA4_ENDPOINT}?measurement_id=${encodeURIComponent(ga.measurementId)}` + `&api_secret=${encodeURIComponent(ga.apiSecret)}`; diff --git a/apps/download-proxy/src/index.ts b/apps/download-proxy/src/index.ts index 059695d17..5126f9d8c 100644 --- a/apps/download-proxy/src/index.ts +++ b/apps/download-proxy/src/index.ts @@ -27,6 +27,8 @@ import { deliverEvent, endEvent, parseGaIds, + segmentEvent, + segmentRole, startEvent, unresolvedEvent, type DownloadContext, @@ -136,16 +138,28 @@ export default { botCategory: (request.cf?.verifiedBotCategory as string | undefined) || null, }; - emit(env, ctx, startEvent(downloadCtx), gaIds); + // One download can arrive as many Range requests (download managers, + // resumes). Only the segment covering byte 0 starts it and only the one + // delivering the last byte ends it; middle segments are logged, not + // reported, so GA counts downloads rather than requests. + const role = segmentRole(origin.status, origin.headers.get('content-range')); + if (role.first) { + emit(env, ctx, startEvent(downloadCtx), gaIds); + } const {readable, done} = trackedStream(origin.body, contentLength); ctx.waitUntil( - done.then(result => - deliverEvent(endEvent(downloadCtx, result), { + done.then(result => { + if (!role.final) { + const segment = segmentEvent(downloadCtx, result); + console.log(JSON.stringify({event: segment.name, ...segment.params, attributed: gaIds.clientId ? 1 : 0})); + return; + } + return deliverEvent(endEvent(downloadCtx, result), { measurementId: env.GA4_MEASUREMENT_ID, apiSecret: env.GA4_API_SECRET, ids: gaIds, - }), - ), + }); + }), ); return new Response(readable, {status: origin.status, headers}); },