Further fixes on comment handling.

emitterc.go

@@ -341,6 +341,7 @@ func yaml_emitter_emit_stream_start(emitter *yaml_emitter_t, event *yaml_event_t
 	emitter.whitespace = true
 	emitter.indention = true
 	emitter.space_above = true
+	emitter.foot_indent = -1
 
 	if emitter.encoding != yaml_UTF8_ENCODING {
 		if !yaml_emitter_write_bom(emitter) {
@@ -498,16 +499,14 @@ func yaml_emitter_emit_document_end(emitter *yaml_emitter_t, event *yaml_event_t
 	if event.typ != yaml_DOCUMENT_END_EVENT {
 		return yaml_emitter_set_emitter_error(emitter, "expected DOCUMENT-END")
 	}
-	if !yaml_emitter_write_indent(emitter) {
+	// [Go] Force document foot separation.
+	emitter.foot_indent = 0
+	if !yaml_emitter_process_foot_comment(emitter) {
 		return false
 	}
-	if len(emitter.foot_comment) > 0 {
-		if !put_break(emitter) {
-			return false
-		}
-		if !yaml_emitter_process_foot_comment(emitter) {
-			return false
-		}
+	emitter.foot_indent = -1
+	if !yaml_emitter_write_indent(emitter) {
+		return false
 	}
 	if !event.implicit {
 		// [Go] Allocate the slice elsewhere.
@@ -1097,27 +1096,15 @@ func yaml_emitter_process_head_comment(emitter *yaml_emitter_t) bool {
 			return false
 		}
 		emitter.tail_comment = emitter.tail_comment[:0]
+		emitter.foot_indent = emitter.indent
+		if emitter.foot_indent < 0 {
+			emitter.foot_indent = 0
+		}
 	}
 
 	if len(emitter.head_comment) == 0 {
 		return true
 	}
-
-	space_above := emitter.space_above
-	if !emitter.indention {
-		if !put_break(emitter) {
-			return false
-		}
-	}
-	if !space_above &&
-		emitter.state != yaml_EMIT_FLOW_SEQUENCE_FIRST_ITEM_STATE &&
-		emitter.state != yaml_EMIT_FLOW_MAPPING_FIRST_KEY_STATE &&
-		emitter.state != yaml_EMIT_BLOCK_MAPPING_FIRST_KEY_STATE &&
-		emitter.state != yaml_EMIT_BLOCK_SEQUENCE_FIRST_ITEM_STATE {
-		if !put_break(emitter) {
-			return false
-		}
-	}
 	if !yaml_emitter_write_indent(emitter) {
 		return false
 	}
@@ -1157,6 +1144,10 @@ func yaml_emitter_process_foot_comment(emitter *yaml_emitter_t) bool {
 		return false
 	}
 	emitter.foot_comment = emitter.foot_comment[:0]
+	emitter.foot_indent = emitter.indent
+	if emitter.foot_indent < 0 {
+		emitter.foot_indent = 0
+	}
 	return true
 }
 
@@ -1478,6 +1469,11 @@ func yaml_emitter_write_indent(emitter *yaml_emitter_t) bool {
 			return false
 		}
 	}
+	if emitter.foot_indent == indent {
+		if !put_break(emitter) {
+			return false
+		}
+	}
 	for emitter.column < indent {
 		if !put(emitter, ' ') {
 			return false
@@ -1486,6 +1482,7 @@ func yaml_emitter_write_indent(emitter *yaml_emitter_t) bool {
 	emitter.whitespace = true
 	//emitter.indention = true
 	emitter.space_above = false
+	emitter.foot_indent = -1
 	return true
 }
 
@@ -1582,7 +1579,7 @@ func yaml_emitter_write_tag_content(emitter *yaml_emitter_t, value []byte, need_
 }
 
 func yaml_emitter_write_plain_scalar(emitter *yaml_emitter_t, value []byte, allow_breaks bool) bool {
-	if !emitter.whitespace {
+	if len(value) > 0 && !emitter.whitespace {
 		if !put(emitter, ' ') {
 			return false
 		}
@@ -1629,7 +1626,9 @@ func yaml_emitter_write_plain_scalar(emitter *yaml_emitter_t, value []byte, allo
 		}
 	}
 
-	emitter.whitespace = false
+	if len(value) > 0 {
+		emitter.whitespace = false
+	}
 	emitter.indention = false
 	if emitter.root_context {
 		emitter.open_ended = true

node_test.go

@@ -714,6 +714,26 @@ var nodeTests = []struct {
 			}},
 		},
 	}, {
+
+		"[decode]\n# One\n\n# Two\n\n# Three\ntrue # Four\n# Five\n\n# Six\n\n# Seven\n",
+		yaml.Node{
+			Kind:   yaml.DocumentNode,
+			Line:   7,
+			Column: 1,
+			HeadComment: "# One\n\n# Two",
+			FootComment: "# Six\n\n# Seven",
+			Content: []*yaml.Node{{
+				Kind:        yaml.ScalarNode,
+				Value:       "true",
+				Tag:         "!!bool",
+				Line:        7,
+				Column:      1,
+				HeadComment: "# Three",
+				LineComment: "# Four",
+				FootComment: "# Five",
+			}},
+		},
+	}, {
 		// Write out the pound character if missing from comments.
 		"[encode]# One\n# Two\ntrue # Three\n# Four\n# Five\n",
 		yaml.Node{
@@ -848,7 +868,7 @@ var nodeTests = []struct {
 			}},
 		},
 	}, {
-		"# DH1\n\n- la # IA\n\n# HB1\n- lb\n",
+		"# DH1\n\n- la # IA\n# HB1\n- lb\n",
 		yaml.Node{
 			Kind:        yaml.DocumentNode,
 			Line:        3,
@@ -869,7 +889,7 @@ var nodeTests = []struct {
 				}, {
 					Kind:        yaml.ScalarNode,
 					Tag:         "!!str",
-					Line:        6,
+					Line:        5,
 					Column:      3,
 					Value:       "lb",
 					HeadComment: "# HB1",
@@ -1275,8 +1295,8 @@ var nodeTests = []struct {
 			},
 		},
 	}, {
-		// Same as above, but with two newlines at the end. Decode-only for that.
-		"[decode]# HA1\nka:\n  # HB1\n  kb: vb\n  # FB1\n# FA1\n\n",
+		// Same as above, but without FB1.
+		"# HA1\nka:\n  # HB1\n  kb: vb\n# FA1\n",
 		yaml.Node{
 			Kind:   yaml.DocumentNode,
 			Line:   2,
@@ -1304,7 +1324,6 @@ var nodeTests = []struct {
 						Tag:         "!!str",
 						Value:       "kb",
 						HeadComment: "# HB1",
-						FootComment: "# FB1",
 						Line:        4,
 						Column:      3,
 					}, {
@@ -1319,7 +1338,8 @@ var nodeTests = []struct {
 			},
 		},
 	}, {
-		"# HA1\nka:\n  # HB1\n  kb: vb\n  # FB1\nkc: vc\n# FC1\n",
+		// Same as above, but with two newlines at the end. Decode-only for that.
+		"[decode]# HA1\nka:\n  # HB1\n  kb: vb\n  # FB1\n# FA1\n\n",
 		yaml.Node{
 			Kind:   yaml.DocumentNode,
 			Line:   2,
@@ -1334,6 +1354,7 @@ var nodeTests = []struct {
 					Tag:         "!!str",
 					Value:       "ka",
 					HeadComment: "# HA1",
+					FootComment: "# FA1",
 					Line:        2,
 					Column:      1,
 				}, {
@@ -1356,11 +1377,205 @@ var nodeTests = []struct {
 						Line:   4,
 						Column: 7,
 					}},
+				}},
+			},
+			},
+		},
+	}, {
+		"ka:\n  kb: vb\n# FA1\n\nkc: vc\n",
+		yaml.Node{
+			Kind:   yaml.DocumentNode,
+			Line:   1,
+			Column: 1,
+			Content: []*yaml.Node{{
+				Kind:   yaml.MappingNode,
+				Tag:    "!!map",
+				Line:   1,
+				Column: 1,
+				Content: []*yaml.Node{{
+					Kind:        yaml.ScalarNode,
+					Tag:         "!!str",
+					Value:       "ka",
+					Line:        1,
+					Column:      1,
+					FootComment: "# FA1",
+				}, {
+					Kind:   yaml.MappingNode,
+					Tag:    "!!map",
+					Line:   2,
+					Column: 3,
+					Content: []*yaml.Node{{
+						Kind:        yaml.ScalarNode,
+						Tag:         "!!str",
+						Value:       "kb",
+						Line:        2,
+						Column:      3,
+					}, {
+						Kind:   yaml.ScalarNode,
+						Tag:    "!!str",
+						Value:  "vb",
+						Line:   2,
+						Column: 7,
+					}},
 				}, {
 					Kind:        yaml.ScalarNode,
 					Tag:         "!!str",
 					Value:       "kc",
-					FootComment: "# FC1",
+					Line:        5,
+					Column:      1,
+				}, {
+					Kind:   yaml.ScalarNode,
+					Tag:    "!!str",
+					Value:  "vc",
+					Line:   5,
+					Column: 5,
+				}},
+			}},
+		},
+	}, {
+		"ka:\n  kb: vb\n# HC1\nkc: vc\n",
+		yaml.Node{
+			Kind:   yaml.DocumentNode,
+			Line:   1,
+			Column: 1,
+			Content: []*yaml.Node{{
+				Kind:   yaml.MappingNode,
+				Tag:    "!!map",
+				Line:   1,
+				Column: 1,
+				Content: []*yaml.Node{{
+					Kind:        yaml.ScalarNode,
+					Tag:         "!!str",
+					Value:       "ka",
+					Line:        1,
+					Column:      1,
+				}, {
+					Kind:   yaml.MappingNode,
+					Tag:    "!!map",
+					Line:   2,
+					Column: 3,
+					Content: []*yaml.Node{{
+						Kind:        yaml.ScalarNode,
+						Tag:         "!!str",
+						Value:       "kb",
+						Line:        2,
+						Column:      3,
+					}, {
+						Kind:   yaml.ScalarNode,
+						Tag:    "!!str",
+						Value:  "vb",
+						Line:   2,
+						Column: 7,
+					}},
+				}, {
+					Kind:        yaml.ScalarNode,
+					Tag:         "!!str",
+					Value:       "kc",
+					HeadComment: "# HC1",
+					Line:        4,
+					Column:      1,
+				}, {
+					Kind:   yaml.ScalarNode,
+					Tag:    "!!str",
+					Value:  "vc",
+					Line:   4,
+					Column: 5,
+				}},
+			}},
+		},
+	}, {
+		// Decode only due to empty line before HC1.
+		"[decode]ka:\n  kb: vb\n\n# HC1\nkc: vc\n",
+		yaml.Node{
+			Kind:   yaml.DocumentNode,
+			Line:   1,
+			Column: 1,
+			Content: []*yaml.Node{{
+				Kind:   yaml.MappingNode,
+				Tag:    "!!map",
+				Line:   1,
+				Column: 1,
+				Content: []*yaml.Node{{
+					Kind:        yaml.ScalarNode,
+					Tag:         "!!str",
+					Value:       "ka",
+					Line:        1,
+					Column:      1,
+				}, {
+					Kind:   yaml.MappingNode,
+					Tag:    "!!map",
+					Line:   2,
+					Column: 3,
+					Content: []*yaml.Node{{
+						Kind:        yaml.ScalarNode,
+						Tag:         "!!str",
+						Value:       "kb",
+						Line:        2,
+						Column:      3,
+					}, {
+						Kind:   yaml.ScalarNode,
+						Tag:    "!!str",
+						Value:  "vb",
+						Line:   2,
+						Column: 7,
+					}},
+				}, {
+					Kind:        yaml.ScalarNode,
+					Tag:         "!!str",
+					Value:       "kc",
+					HeadComment: "# HC1",
+					Line:        5,
+					Column:      1,
+				}, {
+					Kind:   yaml.ScalarNode,
+					Tag:    "!!str",
+					Value:  "vc",
+					Line:   5,
+					Column: 5,
+				}},
+			}},
+		},
+	}, {
+		// Decode-only due to empty lines around HC1.
+		"[decode]ka:\n  kb: vb\n\n# HC1\n\nkc: vc\n",
+		yaml.Node{
+			Kind:   yaml.DocumentNode,
+			Line:   1,
+			Column: 1,
+			Content: []*yaml.Node{{
+				Kind:   yaml.MappingNode,
+				Tag:    "!!map",
+				Line:   1,
+				Column: 1,
+				Content: []*yaml.Node{{
+					Kind:        yaml.ScalarNode,
+					Tag:         "!!str",
+					Value:       "ka",
+					Line:        1,
+					Column:      1,
+				}, {
+					Kind:   yaml.MappingNode,
+					Tag:    "!!map",
+					Line:   2,
+					Column: 3,
+					Content: []*yaml.Node{{
+						Kind:        yaml.ScalarNode,
+						Tag:         "!!str",
+						Value:       "kb",
+						Line:        2,
+						Column:      3,
+					}, {
+						Kind:   yaml.ScalarNode,
+						Tag:    "!!str",
+						Value:  "vb",
+						Line:   2,
+						Column: 7,
+					}},
+				}, {
+					Kind:        yaml.ScalarNode,
+					Tag:         "!!str",
+					Value:       "kc",
+					HeadComment: "# HC1\n",
 					Line:        6,
 					Column:      1,
 				}, {
@@ -1373,8 +1588,46 @@ var nodeTests = []struct {
 			}},
 		},
 	}, {
-		// Decode only as encoding adds an empty line between ka's value and kc's headers.
-		"[decode]# HA1\nka:\n  # HB1\n  kb: vb\n  # FB1\n# HC1\n# HC2\nkc: vc\n# FC1\n# FC2\n",
+		"ka: # IA\n  kb: # IB\n",
+		yaml.Node{
+			Kind:   yaml.DocumentNode,
+			Line:   1,
+			Column: 1,
+			Content: []*yaml.Node{{
+				Kind:   yaml.MappingNode,
+				Tag:    "!!map",
+				Line:   1,
+				Column: 1,
+				Content: []*yaml.Node{{
+					Kind:        yaml.ScalarNode,
+					Tag:         "!!str",
+					Value:       "ka",
+					Line:        1,
+					Column:      1,
+					LineComment: "# IA",
+				}, {
+					Kind:   yaml.MappingNode,
+					Tag:    "!!map",
+					Line:   2,
+					Column: 3,
+					Content: []*yaml.Node{{
+						Kind:        yaml.ScalarNode,
+						Tag:         "!!str",
+						Value:       "kb",
+						Line:        2,
+						Column:      3,
+						LineComment: "# IB",
+					}, {
+						Kind:   yaml.ScalarNode,
+						Tag:    "!!null",
+						Line:   2,
+						Column: 6,
+					}},
+				}},
+			}},
+		},
+	}, {
+		"# HA1\nka:\n  # HB1\n  kb: vb\n  # FB1\n# HC1\n# HC2\nkc: vc\n# FC1\n# FC2\n",
 		yaml.Node{
 			Kind:   yaml.DocumentNode,
 			Line:   2,
@@ -1429,8 +1682,8 @@ var nodeTests = []struct {
 			}},
 		},
 	}, {
-		// Same as above, but with the empty line between ka's value and kc's headers.
-		"# HA1\nka:\n  # HB1\n  kb: vb\n  # FB1\n\n# HC1\n# HC2\nkc: vc\n# FC1\n# FC2\n",
+		// Same as above, but decode only due to empty line between ka's value and kc's headers.
+		"[decode]# HA1\nka:\n  # HB1\n  kb: vb\n  # FB1\n\n# HC1\n# HC2\nkc: vc\n# FC1\n# FC2\n",
 		yaml.Node{
 			Kind:   yaml.DocumentNode,
 			Line:   2,

scannerc.go

@@ -758,7 +758,7 @@ func yaml_parser_fetch_next_token(parser *yaml_parser_t) (ok bool) {
 	}
 
 	comment_mark := parser.mark
-	if parser.flow_level > 0 && buf[pos] == ',' && len(parser.tokens) > 0 {
+	if len(parser.tokens) > 0 && (parser.flow_level == 0 && buf[pos] == ':' || parser.flow_level > 0 && buf[pos] == ',') {
 		// Associate any following comments with the prior token.
 		comment_mark = parser.tokens[len(parser.tokens)-1].start_mark
 	}
@@ -2839,7 +2839,7 @@ func yaml_parser_scan_comments(parser *yaml_parser_t, scan_mark yaml_mark_t) boo
 	var start_mark yaml_mark_t
 
 	var recent_empty = false
-	var first_empty = true
+	var first_empty = parser.newlines <= 1
 
 	var line = parser.mark.line
 	var column = parser.mark.column
@@ -2848,11 +2848,14 @@ func yaml_parser_scan_comments(parser *yaml_parser_t, scan_mark yaml_mark_t) boo
 
 	// The foot line is the place where a comment must start to
 	// still be considered as a foot of the prior content.
-	// If there's some content in the currently parsed line, then the foot
-	// is the line below it.
-	var foot_line = parser.mark.line-parser.newlines+1
-	if parser.newlines == 0 && parser.mark.column > 1 {
-		foot_line++
+	// If there's some content in the currently parsed line, then
+	// the foot is the line below it.
+	var foot_line = -1
+	if scan_mark.line > 0 {
+		foot_line = parser.mark.line-parser.newlines+1
+		if parser.newlines == 0 && parser.mark.column > 1 {
+			foot_line++
+		}
 	}
 
 	var peek = 0
@@ -2868,11 +2871,15 @@ func yaml_parser_scan_comments(parser *yaml_parser_t, scan_mark yaml_mark_t) boo
 		if is_breakz(parser.buffer, parser.buffer_pos+peek) || parser.flow_level > 0 && (c == ']' || c == '}') {
 			// Got line break or terminator.
 			if !recent_empty {
-				if first_empty && (start_mark.line > 0 && start_mark.line == foot_line || start_mark.column-1 < parser.indent) {
+				if first_empty && (start_mark.line == foot_line || start_mark.column-1 < parser.indent) {
 					// This is the first empty line and there were no empty lines before,
 					// so this initial part of the comment is a foot of the prior token
 					// instead of being a head for the following one. Split it up.
 					if len(text) > 0 {
+						if start_mark.column-1 < parser.indent {
+							// If dedented it's unrelated to the prior token.
+							token_mark = start_mark
+						}
 						parser.comments = append(parser.comments, yaml_comment_t{
 							scan_mark:  scan_mark,
 							token_mark: token_mark,

yamlh.go

@@ -753,7 +753,8 @@ type yaml_emitter_t struct {
 	indention  bool // If the last character was an indentation character (' ', '-', '?', ':')?
 	open_ended bool // If an explicit document end is required?
 
-	space_above bool // If there's an empty line right above?
+	space_above bool // Is there's an empty line above?
+	foot_indent int  // The indent used to write the foot comment above, or -1 if none.
 
 	// Anchor analysis.
 	anchor_data struct {