Upstream eemeli/yaml state of the touched files before ed1bc2f909

src/parse/cst-scalar.ts

@@ -0,0 +1,295 @@
+import type { ComposeContext } from '../compose/compose-node.ts'
+import type { ComposeErrorHandler } from '../compose/composer.ts'
+import { resolveBlockScalar } from '../compose/resolve-block-scalar.ts'
+import { resolveFlowScalar } from '../compose/resolve-flow-scalar.ts'
+import type { ErrorCode } from '../errors.ts'
+import { YAMLParseError } from '../errors.ts'
+import type { Range } from '../nodes/types.ts'
+import type { Scalar } from '../nodes/Scalar.ts'
+import type { StringifyContext } from '../stringify/stringify.ts'
+import { stringifyString } from '../stringify/stringifyString.ts'
+import type { BlockScalar, FlowScalar, SourceToken, Token } from './cst.ts'
+
+/**
+ * If `token` is a CST flow or block scalar, determine its string value and a few other attributes.
+ * Otherwise, return `null`.
+ */
+export function resolveAsScalar(
+  token: FlowScalar | BlockScalar,
+  strict?: boolean,
+  onError?: (offset: number, code: ErrorCode, message: string) => void
+): {
+  value: string
+  type: Scalar.Type | null
+  comment: string
+  range: Range
+}
+export function resolveAsScalar(
+  token: Token | null | undefined,
+  strict?: boolean,
+  onError?: (offset: number, code: ErrorCode, message: string) => void
+): {
+  value: string
+  type: Scalar.Type | null
+  comment: string
+  range: Range
+} | null
+export function resolveAsScalar(
+  token: Token | null | undefined,
+  strict = true,
+  onError?: (offset: number, code: ErrorCode, message: string) => void
+): {
+  value: string
+  type: Scalar.Type | null
+  comment: string
+  range: Range
+} | null {
+  if (token) {
+    const _onError: ComposeErrorHandler = (pos, code, message) => {
+      const offset =
+        typeof pos === 'number' ? pos : Array.isArray(pos) ? pos[0] : pos.offset
+      if (onError) onError(offset, code, message)
+      else throw new YAMLParseError([offset, offset + 1], code, message)
+    }
+    switch (token.type) {
+      case 'scalar':
+      case 'single-quoted-scalar':
+      case 'double-quoted-scalar':
+        return resolveFlowScalar(token, strict, _onError)
+      case 'block-scalar':
+        return resolveBlockScalar(
+          { options: { strict } } as ComposeContext,
+          token,
+          _onError
+        )
+    }
+  }
+  return null
+}
+
+/**
+ * Create a new scalar token with `value`
+ *
+ * Values that represent an actual string but may be parsed as a different type should use a `type` other than `'PLAIN'`,
+ * as this function does not support any schema operations and won't check for such conflicts.
+ *
+ * @param value The string representation of the value, which will have its content properly indented.
+ * @param context.end Comments and whitespace after the end of the value, or after the block scalar header. If undefined, a newline will be added.
+ * @param context.implicitKey Being within an implicit key may affect the resolved type of the token's value.
+ * @param context.indent The indent level of the token.
+ * @param context.inFlow Is this scalar within a flow collection? This may affect the resolved type of the token's value.
+ * @param context.offset The offset position of the token.
+ * @param context.type The preferred type of the scalar token. If undefined, the previous type of the `token` will be used, defaulting to `'PLAIN'`.
+ */
+export function createScalarToken(
+  value: string,
+  context: {
+    end?: SourceToken[]
+    implicitKey?: boolean
+    indent: number
+    inFlow?: boolean
+    offset?: number
+    type?: Scalar.Type
+  }
+): BlockScalar | FlowScalar {
+  const {
+    implicitKey = false,
+    indent,
+    inFlow = false,
+    offset = -1,
+    type = 'PLAIN'
+  } = context
+  const source = stringifyString(
+    { type, value } as Scalar,
+    {
+      implicitKey,
+      indent: indent > 0 ? ' '.repeat(indent) : '',
+      inFlow,
+      options: { blockQuote: true, lineWidth: -1 }
+    } as StringifyContext
+  )
+  const end = context.end ?? [
+    { type: 'newline', offset: -1, indent, source: '\n' }
+  ]
+  switch (source[0]) {
+    case '|':
+    case '>': {
+      const he = source.indexOf('\n')
+      const head = source.substring(0, he)
+      const body = source.substring(he + 1) + '\n'
+      const props: Token[] = [
+        { type: 'block-scalar-header', offset, indent, source: head }
+      ]
+      if (!addEndtoBlockProps(props, end))
+        props.push({ type: 'newline', offset: -1, indent, source: '\n' })
+      return { type: 'block-scalar', offset, indent, props, source: body }
+    }
+    case '"':
+      return { type: 'double-quoted-scalar', offset, indent, source, end }
+    case "'":
+      return { type: 'single-quoted-scalar', offset, indent, source, end }
+    default:
+      return { type: 'scalar', offset, indent, source, end }
+  }
+}
+
+/**
+ * Set the value of `token` to the given string `value`, overwriting any previous contents and type that it may have.
+ *
+ * Best efforts are made to retain any comments previously associated with the `token`,
+ * though all contents within a collection's `items` will be overwritten.
+ *
+ * Values that represent an actual string but may be parsed as a different type should use a `type` other than `'PLAIN'`,
+ * as this function does not support any schema operations and won't check for such conflicts.
+ *
+ * @param token Any token. If it does not include an `indent` value, the value will be stringified as if it were an implicit key.
+ * @param value The string representation of the value, which will have its content properly indented.
+ * @param context.afterKey In most cases, values after a key should have an additional level of indentation.
+ * @param context.implicitKey Being within an implicit key may affect the resolved type of the token's value.
+ * @param context.inFlow Being within a flow collection may affect the resolved type of the token's value.
+ * @param context.type The preferred type of the scalar token. If undefined, the previous type of the `token` will be used, defaulting to `'PLAIN'`.
+ */
+export function setScalarValue(
+  token: Token,
+  value: string,
+  context: {
+    afterKey?: boolean
+    implicitKey?: boolean
+    inFlow?: boolean
+    type?: Scalar.Type
+  } = {}
+): void {
+  let { afterKey = false, implicitKey = false, inFlow = false, type } = context
+  let indent = 'indent' in token ? token.indent : null
+  if (afterKey && typeof indent === 'number') indent += 2
+  if (!type)
+    switch (token.type) {
+      case 'single-quoted-scalar':
+        type = 'QUOTE_SINGLE'
+        break
+      case 'double-quoted-scalar':
+        type = 'QUOTE_DOUBLE'
+        break
+      case 'block-scalar': {
+        const header = token.props[0]
+        if (header.type !== 'block-scalar-header')
+          throw new Error('Invalid block scalar header')
+        type = header.source[0] === '>' ? 'BLOCK_FOLDED' : 'BLOCK_LITERAL'
+        break
+      }
+      default:
+        type = 'PLAIN'
+    }
+  const source = stringifyString(
+    { type, value } as Scalar,
+    {
+      implicitKey: implicitKey || indent === null,
+      indent: indent !== null && indent > 0 ? ' '.repeat(indent) : '',
+      inFlow,
+      options: { blockQuote: true, lineWidth: -1 }
+    } as StringifyContext
+  )
+  switch (source[0]) {
+    case '|':
+    case '>':
+      setBlockScalarValue(token, source)
+      break
+    case '"':
+      setFlowScalarValue(token, source, 'double-quoted-scalar')
+      break
+    case "'":
+      setFlowScalarValue(token, source, 'single-quoted-scalar')
+      break
+    default:
+      setFlowScalarValue(token, source, 'scalar')
+  }
+}
+
+function setBlockScalarValue(token: Token, source: string) {
+  const he = source.indexOf('\n')
+  const head = source.substring(0, he)
+  const body = source.substring(he + 1) + '\n'
+  if (token.type === 'block-scalar') {
+    const header = token.props[0]
+    if (header.type !== 'block-scalar-header')
+      throw new Error('Invalid block scalar header')
+    header.source = head
+    token.source = body
+  } else {
+    const { offset } = token
+    const indent = 'indent' in token ? token.indent : -1
+    const props: Token[] = [
+      { type: 'block-scalar-header', offset, indent, source: head }
+    ]
+    if (!addEndtoBlockProps(props, 'end' in token ? token.end : undefined))
+      props.push({ type: 'newline', offset: -1, indent, source: '\n' })
+
+    for (const key of Object.keys(token))
+      if (key !== 'type' && key !== 'offset') delete (token as any)[key]
+    Object.assign(token, { type: 'block-scalar', indent, props, source: body })
+  }
+}
+
+/** @returns `true` if last token is a newline */
+function addEndtoBlockProps(props: Token[], end?: SourceToken[]) {
+  if (end)
+    for (const st of end)
+      switch (st.type) {
+        case 'space':
+        case 'comment':
+          props.push(st)
+          break
+        case 'newline':
+          props.push(st)
+          return true
+      }
+  return false
+}
+
+function setFlowScalarValue(
+  token: Token,
+  source: string,
+  type: 'scalar' | 'double-quoted-scalar' | 'single-quoted-scalar'
+) {
+  switch (token.type) {
+    case 'scalar':
+    case 'double-quoted-scalar':
+    case 'single-quoted-scalar':
+      token.type = type
+      token.source = source
+      break
+    case 'block-scalar': {
+      const end = token.props.slice(1)
+      let oa = source.length
+      if (token.props[0].type === 'block-scalar-header')
+        oa -= token.props[0].source.length
+      for (const tok of end) tok.offset += oa
+      delete (token as any).props
+      Object.assign(token, { type, source, end })
+      break
+    }
+    case 'block-map':
+    case 'block-seq': {
+      const offset = token.offset + source.length
+      const nl = { type: 'newline', offset, indent: token.indent, source: '\n' }
+      delete (token as any).items
+      Object.assign(token, { type, source, end: [nl] })
+      break
+    }
+    default: {
+      const indent = 'indent' in token ? token.indent : -1
+      const end =
+        'end' in token && Array.isArray(token.end)
+          ? token.end.filter(
+              st =>
+                st.type === 'space' ||
+                st.type === 'comment' ||
+                st.type === 'newline'
+            )
+          : []
+      for (const key of Object.keys(token))
+        if (key !== 'type' && key !== 'offset') delete (token as any)[key]
+      Object.assign(token, { type, indent, source, end })
+    }
+  }
+}

tests/cst.ts

@@ -0,0 +1,203 @@
+import { CST, Parser } from 'yaml'
+import { source } from './_utils.ts'
+
+function cstDoc(src: string) {
+  const tokens = Array.from(new Parser().parse(src))
+  expect(tokens).toHaveLength(1)
+  expect(tokens[0].type).toBe('document')
+  return tokens[0] as CST.Document
+}
+
+describe('CST.visit', () => {
+  test('Visit paths in order', () => {
+    const doc = cstDoc(source`
+      foo:
+        - baz
+        - [ bar: 42 ]
+      ? { fuzz, ball }
+      : fii
+    `)
+    const paths: CST.VisitPath[] = []
+    CST.visit(doc, (_item, path) => {
+      paths.push(path)
+    })
+    expect(paths).toMatchObject([
+      [],
+      [['value', 0]],
+      [
+        ['value', 0],
+        ['value', 0]
+      ],
+      [
+        ['value', 0],
+        ['value', 1]
+      ],
+      [
+        ['value', 0],
+        ['value', 1],
+        ['value', 0]
+      ],
+      [['value', 1]],
+      [
+        ['value', 1],
+        ['key', 0]
+      ],
+      [
+        ['value', 1],
+        ['key', 1]
+      ]
+    ])
+  })
+
+  test('break visit', () => {
+    const doc = cstDoc(source`
+      - " foo"
+      - "foo"
+      - "bar"
+    `)
+    let visits = 0
+    CST.visit(doc, item => {
+      visits += 1
+      const scalar = CST.resolveAsScalar(item.value)
+      if (scalar?.value === 'foo') return CST.visit.BREAK
+    })
+    expect(visits).toBe(3)
+  })
+
+  test('remove item', () => {
+    const doc = cstDoc(source`
+      -  " foo"
+      - "foo"
+      - "bar"
+    `)
+    let visits = 0
+    CST.visit(doc, item => {
+      visits += 1
+      const scalar = CST.resolveAsScalar(item.value)
+      if (scalar?.value === 'foo') return CST.visit.REMOVE
+    })
+    expect(visits).toBe(4)
+    expect(CST.stringify(doc)).toBe(source`
+      -  " foo"
+      - "bar"
+    `)
+  })
+
+  test('replace value with block scalar in seq', () => {
+    const doc = cstDoc(source`
+      -  " foo"
+      - "foo"
+      - "bar"
+    `)
+    let visits = 0
+    CST.visit(doc, item => {
+      visits += 1
+      if (item.value) {
+        const scalar = CST.resolveAsScalar(item.value)
+        if (scalar?.value === 'foo')
+          CST.setScalarValue(item.value, 'foo', { type: 'BLOCK_LITERAL' })
+      }
+    })
+    expect(visits).toBe(4)
+    expect(CST.stringify(doc)).toBe(source`
+      -  " foo"
+      - |-
+        foo
+      - "bar"
+    `)
+  })
+
+  test('add item', () => {
+    const doc = cstDoc(source`
+      -  " foo"
+      - "foo"
+      - "bar"
+    `)
+    let visits = 0
+    CST.visit(doc, (item, path) => {
+      visits += 1
+      if (CST.isScalar(item.value)) {
+        const scalar = CST.resolveAsScalar(item.value)
+        if (scalar?.value === 'foo') {
+          const parent = CST.visit.parentCollection(doc, path)
+          const idx = path[path.length - 1][1]
+          const { indent } = item.value
+          parent.items.splice(idx, 0, {
+            start: item.start.slice(),
+            value: CST.createScalarToken('hip', { indent })
+          })
+          return idx + 2
+        }
+      }
+    })
+    expect(visits).toBe(4)
+    expect(CST.stringify(doc)).toBe(source`
+      -  " foo"
+      - hip
+      - "foo"
+      - "bar"
+    `)
+  })
+
+  test('replace value with flow scalar in map', () => {
+    const doc = cstDoc(source`
+      - a:  A
+        b: B\t#comment
+        c: C
+    `)
+    let visits = 0
+    CST.visit(doc, item => {
+      visits += 1
+      if (item.value) {
+        const scalar = CST.resolveAsScalar(item.value)
+        if (scalar?.value === 'B') {
+          CST.setScalarValue(item.value, 'foo\n\nbar', { afterKey: !!item.key })
+        }
+      }
+    })
+    expect(visits).toBe(5)
+    expect(CST.stringify(doc)).toBe(source`
+      - a:  A
+        b: foo
+
+
+          bar\t#comment
+        c: C
+    `)
+  })
+
+  test('skip children', () => {
+    const doc = cstDoc(source`
+      - " foo"
+      - [ foo, 13, 42 ]
+      - "bar"
+    `)
+    let visits = 0
+    CST.visit(doc, item => {
+      visits += 1
+      if (item.value?.type === 'flow-collection') return CST.visit.SKIP
+    })
+    expect(visits).toBe(4)
+  })
+
+  test('set next index', () => {
+    const doc = cstDoc(source`
+      - "foo"
+      - [ foo, 13, 42 ]
+      - "bar"
+    `)
+    let visits = 0
+    CST.visit(doc, item => {
+      visits += 1
+      const scalar = CST.resolveAsScalar(item.value)
+      if (scalar?.value === 'foo') return 2
+    })
+    expect(visits).toBe(3)
+  })
+})
+
+test('Line comment before unindented block-seq in block-map (#525)', () => {
+  const src = 'a:\n#\n- b'
+  const [doc] = Array.from(new Parser().parse(src))
+  expect(CST.stringify(doc)).toBe(src)
+})