More efficient front matter processor

cmd/evaluate_all_command.go

@@ -98,10 +98,7 @@ func evaluateAll(cmd *cobra.Command, args []string) error {
 		args[firstFileIndex] = frontMatterHandler.GetYamlFrontMatterFilename()
 
 		if frontMatter == "process" {
-			reader, err := os.Open(frontMatterHandler.GetContentFilename()) // #nosec
+			reader := frontMatterHandler.GetContentReader()
-			if err != nil {
-				return err
-			}
 			printer.SetAppendix(reader)
 			defer yqlib.SafelyCloseReader(reader)
 		}

cmd/evalute_sequence_command.go

@@ -112,10 +112,7 @@ func evaluateSequence(cmd *cobra.Command, args []string) error {
 		args[firstFileIndex] = frontMatterHandler.GetYamlFrontMatterFilename()
 
 		if frontMatter == "process" {
-			reader, err := os.Open(frontMatterHandler.GetContentFilename()) // #nosec
+			reader := frontMatterHandler.GetContentReader()
-			if err != nil {
-				return err
-			}
 			printer.SetAppendix(reader)
 			defer yqlib.SafelyCloseReader(reader)
 		}

examples/front-matter.yaml

@@ -2,4 +2,5 @@
 a: apple
 b: bannana
 ---
-<h1>I like {{a}} and {{b}} </h1>
+hello there
+apples: great

pkg/yqlib/front_matter.go

@@ -9,47 +9,48 @@ import (
 type frontMatterHandler interface {
 	Split() error
 	GetYamlFrontMatterFilename() string
-	GetContentFilename() string
+	GetContentReader() io.Reader
 	CleanUp()
 }
 
 type frontMatterHandlerImpl struct {
 	originalFilename        string
 	yamlFrontMatterFilename string
-	contentFilename         string
+	contentReader           io.Reader
 }
 
 func NewFrontMatterHandler(originalFilename string) frontMatterHandler {
-	return &frontMatterHandlerImpl{originalFilename, "", ""}
+	return &frontMatterHandlerImpl{originalFilename, "", nil}
 }
 
 func (f *frontMatterHandlerImpl) GetYamlFrontMatterFilename() string {
 	return f.yamlFrontMatterFilename
 }
 
-func (f *frontMatterHandlerImpl) GetContentFilename() string {
+func (f *frontMatterHandlerImpl) GetContentReader() io.Reader {
-	return f.contentFilename
+	return f.contentReader
 }
 
 func (f *frontMatterHandlerImpl) CleanUp() {
 	tryRemoveFile(f.yamlFrontMatterFilename)
-	tryRemoveFile(f.contentFilename)
 }
 
 // Splits the given file by yaml front matter
 // yaml content will be saved to first temporary file
 // remaining content will be saved to second temporary file
 func (f *frontMatterHandlerImpl) Split() error {
-	var reader io.Reader
+	var reader *bufio.Reader
 	var err error
 	if f.originalFilename == "-" {
 		reader = bufio.NewReader(os.Stdin)
 	} else {
-		reader, err = os.Open(f.originalFilename) // #nosec
+		file, err := os.Open(f.originalFilename) // #nosec
 		if err != nil {
 			return err
 		}
+		reader = bufio.NewReader(file)
 	}
+	f.contentReader = reader
 
 	yamlTempFile, err := createTempFile()
 	if err != nil {
@@ -58,39 +59,35 @@ func (f *frontMatterHandlerImpl) Split() error {
 	f.yamlFrontMatterFilename = yamlTempFile.Name()
 	log.Debug("yamlTempFile: %v", yamlTempFile.Name())
 
-	contentTempFile, err := createTempFile()
-	if err != nil {
-		return err
-	}
-	f.contentFilename = contentTempFile.Name()
-	log.Debug("contentTempFile: %v", contentTempFile.Name())
-
-	scanner := bufio.NewScanner(reader)
-
 	lineCount := 0
-	yamlContentBlock := true
 
-	for scanner.Scan() {
+	for {
-		line := scanner.Text()
+		peekBytes, err := reader.Peek(3)
-
+		if err == io.EOF {
-		if lineCount > 0 && line == "---" {
+			// we've finished reading the yaml content..I guess
-			//we've finished reading the yaml content
+			break
-			yamlContentBlock = false
+		} else if err != nil {
-		}
-		if yamlContentBlock {
-			_, err = yamlTempFile.Write([]byte(line + "\n"))
-		} else {
-			_, err = contentTempFile.Write([]byte(line + "\n"))
-		}
-		if err != nil {
 			return err
 		}
+		if lineCount > 0 && string(peekBytes) == "---" {
+			// we've finished reading the yaml content..
+			break
+		}
+		line, errReading := reader.ReadString('\n')
 		lineCount = lineCount + 1
+		if errReading != nil && errReading != io.EOF {
+			return errReading
+		}
+
+		_, errWriting := yamlTempFile.Write([]byte(line))
+
+		if errWriting != nil {
+			return errWriting
+		}
 	}
 
 	safelyCloseFile(yamlTempFile)
-	safelyCloseFile(contentTempFile)
 
-	return scanner.Err()
+	return nil
 
 }

pkg/yqlib/front_matter_test.go

@@ -60,8 +60,11 @@ yaml: doc
 
 	test.AssertResult(t, expectedYamlFm, yamlFm)
 
-	content := readFile(fmHandler.GetContentFilename())
+	contentBytes, err := ioutil.ReadAll(fmHandler.GetContentReader())
-	test.AssertResult(t, expectedContent, content)
+	if err != nil {
+		panic(err)
+	}
+	test.AssertResult(t, expectedContent, string(contentBytes))
 
 	tryRemoveFile(file)
 	fmHandler.CleanUp()
@@ -94,8 +97,11 @@ yaml: doc
 
 	test.AssertResult(t, expectedYamlFm, yamlFm)
 
-	content := readFile(fmHandler.GetContentFilename())
+	contentBytes, err := ioutil.ReadAll(fmHandler.GetContentReader())
-	test.AssertResult(t, expectedContent, content)
+	if err != nil {
+		panic(err)
+	}
+	test.AssertResult(t, expectedContent, string(contentBytes))
 
 	tryRemoveFile(file)
 	fmHandler.CleanUp()
@@ -125,8 +131,11 @@ yaml: doc
 
 	test.AssertResult(t, expectedYamlFm, yamlFm)
 
-	content := readFile(fmHandler.GetContentFilename())
+	contentBytes, err := ioutil.ReadAll(fmHandler.GetContentReader())
-	test.AssertResult(t, expectedContent, content)
+	if err != nil {
+		panic(err)
+	}
+	test.AssertResult(t, expectedContent, string(contentBytes))
 
 	tryRemoveFile(file)
 	fmHandler.CleanUp()

pkg/yqlib/printer.go

@@ -28,7 +28,6 @@ type resultsPrinter struct {
 	previousFileIndex  int
 	printedMatches     bool
 	treeNavigator      DataTreeNavigator
-	preambleReader     io.Reader
 	appendixReader     io.Reader
 }
 
@@ -45,10 +44,6 @@ func NewPrinter(writer io.Writer, outputToJSON bool, unwrapScalar bool, colorsEn
 	}
 }
 
-func (p *resultsPrinter) SetPreamble(reader io.Reader) {
-	p.preambleReader = reader
-}
-
 func (p *resultsPrinter) SetAppendix(reader io.Reader) {
 	p.appendixReader = reader
 }