Add notes

.changeset/brave-mangos-bake.md

@@ -0,0 +1,6 @@
+---
+'@boostv/process-optimizer-frontend-core': minor
+'@boostv/process-optimizer-frontend-ui': minor
+---
+
+Add per-point notes and robust CSV round-trips (quoting, configurable separator, import errors).

packages/core/src/common/types/common.test.ts

@@ -40,9 +40,9 @@ describe('Type guards', () => {
   })
 })
 
-describe('score function schema (v21)', () => {
-  it('currentVersion is 21', () => {
-    expect(currentVersion).toBe('21')
+describe('score function schema (v22)', () => {
+  it('currentVersion is 22', () => {
+    expect(currentVersion).toBe('22')
   })
 
   it('accepts a scoreVariable with a scoreFunction and a dataEntry with responses', () => {
@@ -53,7 +53,7 @@ describe('score function schema (v21)', () => {
         name: 'n',
         description: '',
         swVersion: 'v',
-        dataFormatVersion: '21',
+        dataFormatVersion: '22',
         version: 0,
         lastModified: '',
         createdAt: '',
@@ -123,7 +123,7 @@ describe('score function schema (v21)', () => {
         name: 'n',
         description: '',
         swVersion: 'v',
-        dataFormatVersion: '21',
+        dataFormatVersion: '22',
         version: 0,
         lastModified: '',
         createdAt: '',
@@ -166,3 +166,30 @@ describe('score function schema (v21)', () => {
     expect(parsed.success).toBe(true)
   })
 })
+
+describe('data point note (v22)', () => {
+  it('accepts a dataEntry with a note in meta', () => {
+    const withNote = {
+      ...emptyExperiment,
+      dataPoints: [
+        {
+          meta: { id: 1, enabled: true, valid: true, note: 'measured twice' },
+          data: [],
+        },
+      ],
+    }
+    const parsed = experimentSchema.safeParse(withNote)
+    expect(parsed.success).toBe(true)
+    if (parsed.success) {
+      expect(parsed.data.dataPoints[0]?.meta.note).toBe('measured twice')
+    }
+  })
+
+  it('still accepts a dataEntry without a note', () => {
+    const withoutNote = {
+      ...emptyExperiment,
+      dataPoints: [{ meta: { id: 1, enabled: true, valid: true }, data: [] }],
+    }
+    expect(experimentSchema.safeParse(withoutNote).success).toBe(true)
+  })
+})

packages/core/src/common/types/common.ts

@@ -3,7 +3,7 @@ import { z } from 'zod'
 // Change the current version when doing structural
 // changes to any types belonging to ExperimentType
 
-export const currentVersion = '21'
+export const currentVersion = '22'
 
 export const scoreNames = ['quality', 'cost'] as const
 // Label is shown in UI, name is used in data
@@ -98,6 +98,7 @@ const dataEntryMetaDataSchema = z.object({
   enabled: z.coerce.boolean().prefault(true),
   valid: z.coerce.boolean().prefault(true),
   description: z.optional(z.string()),
+  note: z.optional(z.string()),
 })
 
 const numericDataPoint = z.object({

packages/core/src/common/util/converters/converters.test.ts

@@ -380,6 +380,17 @@ describe('converters', () => {
       expect(actual).toEqual(expected)
     })
 
+    it('quotes meta values that contain the separator or a quote', () => {
+      const csv = dataPointsToCSV([
+        {
+          meta: { id: 1, enabled: true, valid: true, note: 'a;b "c"' },
+          data: [{ type: 'numeric', name: 'A', value: 1 }],
+        },
+      ])
+      // the note field is quoted, inner quotes doubled; other fields untouched
+      expect(csv).toBe('id;A;enabled;valid;note\n1;1;true;true;"a;b ""c"""')
+    })
+
     it('should convert known value', () => {
       const input: DataEntry[] = [
         {
@@ -872,4 +883,116 @@ describe('converters', () => {
       expect(actual[1]?.meta.description).toEqual('I am also a description')
     })
   })
+
+  describe('note round-trip', () => {
+    const valueVars = [
+      {
+        type: 'continuous' as const,
+        name: 'A',
+        description: '',
+        min: 0,
+        max: 10,
+        enabled: true,
+      },
+    ]
+
+    it('writes the note as a meta column in CSV', () => {
+      const csv = dataPointsToCSV([
+        {
+          meta: { id: 1, enabled: true, valid: true, note: 'seemed fine' },
+          data: [{ type: 'numeric', name: 'A', value: 1 }],
+        },
+      ])
+      expect(csv).toContain('note')
+      expect(csv).toContain('seemed fine')
+    })
+
+    it('parses the note back from CSV', () => {
+      const csv = 'id;A;enabled;valid;note\n1;1;true;true;seemed fine'
+      const actual = csvToDataPoints(csv, valueVars, [], [])
+      expect(actual[0]?.meta.note).toBe('seemed fine')
+    })
+
+    it('round-trips a note through export and import', () => {
+      const input = [
+        {
+          meta: { id: 1, enabled: true, valid: true, note: 'seemed fine' },
+          data: [{ type: 'numeric' as const, name: 'A', value: 1 }],
+        },
+      ]
+      const back = csvToDataPoints(dataPointsToCSV(input), valueVars, [], [])
+      expect(back[0]?.meta.note).toBe('seemed fine')
+    })
+
+    it('does not set a note when the CSV note column is empty', () => {
+      const csv = 'id;A;enabled;valid;note\n1;1;true;true;'
+      const actual = csvToDataPoints(csv, valueVars, [], [])
+      expect(actual[0]?.meta.note).toBeUndefined()
+    })
+
+    it('keeps the note aligned when rows have different optional meta keys', () => {
+      // Row 1 has a note but no description; row 2 has a description but no
+      // note. The CSV header is the union of meta keys, so meta values must be
+      // emitted by header order — otherwise row 2's description would land in
+      // the note column and be read back as a note.
+      const input = [
+        {
+          meta: { id: 1, enabled: true, valid: true, note: 'has note' },
+          data: [{ type: 'numeric' as const, name: 'A', value: 1 }],
+        },
+        {
+          meta: { id: 2, enabled: true, valid: true, description: 'has desc' },
+          data: [{ type: 'numeric' as const, name: 'A', value: 2 }],
+        },
+      ]
+      const back = csvToDataPoints(dataPointsToCSV(input), valueVars, [], [])
+      expect(back[0]?.meta.note).toBe('has note')
+      expect(back[1]?.meta.note).toBeUndefined()
+    })
+
+    it('round-trips a note containing the separator and quotes', () => {
+      const note = 'I pressed; it broke "hard"'
+      const back = csvToDataPoints(
+        dataPointsToCSV([
+          {
+            meta: { id: 1, enabled: true, valid: true, note },
+            data: [{ type: 'numeric' as const, name: 'A', value: 1 }],
+          },
+        ]),
+        valueVars,
+        [],
+        []
+      )
+      expect(back[0]?.meta.note).toBe(note)
+    })
+  })
+
+  describe.each([';', ',', '\t', '|'])('separator %j round-trip', sep => {
+    it('round-trips values and notes', () => {
+      const valueVars = [
+        {
+          type: 'continuous' as const,
+          name: 'A',
+          description: '',
+          min: 0,
+          max: 10,
+          enabled: true,
+        },
+      ]
+      const input = [
+        {
+          meta: { id: 1, enabled: true, valid: true, note: 'x;y,z\tw|q' },
+          data: [{ type: 'numeric' as const, name: 'A', value: 1 }],
+        },
+      ]
+      const back = csvToDataPoints(
+        dataPointsToCSV(input, sep),
+        valueVars,
+        [],
+        [],
+        sep
+      )
+      expect(back[0]?.meta.note).toBe('x;y,z\tw|q')
+    })
+  })
 })

packages/core/src/common/util/converters/converters.ts

@@ -109,6 +109,48 @@ export const calculateConstraints = (experiment: ExperimentType) =>
         .sort(),
     }))
     .filter(c => (c.type === 'sum' ? c.dimensions.length > 1 : true))
+// RFC-4180: a field containing the separator, a double-quote, or a newline is
+// wrapped in double-quotes with internal quotes doubled. Clean fields are left
+// untouched, so exports of ordinary data are byte-identical to before.
+const escapeCsvField = (value: string, separator: string): string =>
+  value.includes(separator) ||
+  value.includes('"') ||
+  value.includes('\n') ||
+  value.includes('\r')
+    ? `"${value.replaceAll('"', '""')}"`
+    : value
+
+// Split one CSV line into fields, honoring RFC-4180 quotes: a double-quoted
+// field may contain the separator, and "" is an escaped quote. (Embedded
+// newlines are out of scope — the app's inputs are single-line.)
+const parseCsvLine = (line: string, separator: string): string[] => {
+  const fields: string[] = []
+  let field = ''
+  let inQuotes = false
+  for (let i = 0; i < line.length; i++) {
+    const ch = line[i]
+    if (inQuotes) {
+      if (ch === '"' && line[i + 1] === '"') {
+        field += '"'
+        i++
+      } else if (ch === '"') {
+        inQuotes = false
+      } else {
+        field += ch
+      }
+    } else if (ch === '"') {
+      inQuotes = true
+    } else if (ch === separator) {
+      fields.push(field)
+      field = ''
+    } else {
+      field += ch
+    }
+  }
+  fields.push(field)
+  return fields
+}
+
 /**
  * Converts a list of DataEntry objects into a CSV string.
  * The output format:
@@ -142,7 +184,11 @@ export const dataPointsToCSV = (
   ]
   return dataPoints.length === 0
     ? ''
-    : [['id'].concat(header, meta).join(separator)]
+    : [
+        ['id', ...header, ...meta]
+          .map(field => escapeCsvField(field, separator))
+          .join(separator),
+      ]
         // Generate data lines
         .concat(
           [...dataPoints]
@@ -152,15 +198,21 @@ export const dataPointsToCSV = (
               )
               return { ...line, data: header.map(h => values.get(h) ?? '') }
             })
-            .map(
-              line =>
-                `${line.meta.id}${separator}${line.data
-                  .concat(
-                    Object.entries(line.meta as object)
-                      .filter(e => e[0] !== 'id')
-                      .map(e => e[1])
-                  )
-                  .join(separator)}`
+            .map(line =>
+              // Meta values are emitted in the `meta` header (union) order —
+              // looked up by key, not the row's own key order — so rows with
+              // different optional meta keys (e.g. note vs description) stay
+              // column-aligned on re-import.
+              [
+                String(line.meta.id),
+                ...line.data,
+                ...meta.map(key => {
+                  const value = (line.meta as Record<string, unknown>)[key]
+                  return value === undefined ? '' : String(value)
+                }),
+              ]
+                .map(field => escapeCsvField(field, separator))
+                .join(separator)
             )
         )
         .filter(s => '' !== s)
@@ -220,16 +272,17 @@ export const csvToDataPoints = (
   const lines = csv.split(newlinePattern)
   if ('' === csv || lines.length < 2) return []
   else {
-    const header = lines[0]?.split(separator).map(h => h.trim()) ?? []
+    const header = parseCsvLine(lines[0] ?? '', separator).map(h => h.trim())
     if (
       header.length >= expectedHeader.length &&
       expectedHeader.every(value => header.includes(value))
     ) {
       const data = lines.slice(1)
       const dataAsKeyValue = data.map(line =>
-        line
-          .split(separator)
-          .map((value, idx) => ({ key: header[idx] ?? '', value }))
+        parseCsvLine(line, separator).map((value, idx) => ({
+          key: header[idx] ?? '',
+          value,
+        }))
       )
       const dataList = dataAsKeyValue.map((line, idx) => ({
         data: line
@@ -286,5 +339,8 @@ const convertToMetaData = (
         : true,
     id: 'id' in parsedMeta ? Number(parsedMeta['id'] ?? idx + 1) : idx + 1,
   }
+  if (result.note === '' || result.note === undefined) {
+    delete result.note
+  }
   return result
 }